From 311b702375161039e1c9c10c29d77993cd323cdd Mon Sep 17 00:00:00 2001 From: Marinski Date: Thu, 20 Aug 2026 10:33:41 +0300 Subject: [PATCH 1/2] feat(compile): POST /compile - MQL5 source in, .ex5 out Adds a compile endpoint so a caller can turn MQL5 source into a real .ex5 without a Windows machine, a MetaEditor install, or file access to the host. Pairs with POST /backtest: compile, then test, over one API. POST /compile {"source": "...", "filename": "MyEA.mq5"} returns {"ok": true, "ex5_base64": "...", "log": "...", "warnings": 0, "include_hash": "sha256:..."}. Two invariants the tests pin, because clients end up depending on them: every response is JSON including auth failures and timeouts, so a non-JSON body unambiguously means a broken host; and ok:true always carries a non-empty binary, re-read and verified before success is claimed. Source text only. No caller-controlled paths, flags or include uploads - `filename` is reduced to a bare stem, so "../../evil" and "C:\x\y.mq5" both become a harmless name. Each request compiles in its own temp directory, removed on every exit path including timeout and crash. MetaEditor specifics this absorbs: * It exits NON-ZERO on warnings as well as errors, so the exit code cannot decide the outcome. The log is parsed for counts and the produced .ex5 is the tiebreaker; a warning-only build is a success. * Its log is UTF-16LE with a BOM. Decoded as UTF-8 you get NUL-riddled mojibake and every count regex silently stops matching. * A missing #include is a 422, not a 500 - the source is wrong, and the caller must not retry it. Compiles serialize behind one lock and the request stays synchronous. A queue would add lost jobs, status polling and restart recovery to buy nothing, since the work cannot overlap. A caller waiting longer than the deadline plus 30s gets a JSON 504 rather than a hung connection. Adds compile_api_token, a second credential accepted ONLY on /compile. api_token unlocks order placement, position management and terminal restart; handing that to something whose only job is compiling hands it the trading account too. Existing auth is unchanged and leaving the new token empty changes nothing. COMPILE_LOCAL_CACHE mirrors the toolchain to local disk, which is the difference between 29s and 1.3s where the terminals sit on a host-shared mount - MetaEditor is ~105MB and the page cache does not save you. The mirror is incremental: copying it unconditionally put 103s in front of the first caller after every process start, because the stock MQL5 Include tree is ~260 files. It also prunes, so a header deleted from the source stops resolving instead of lingering forever. Most of the non-obvious work here guards ONE failure: a compile that succeeds against the wrong library, returning ok:true with a valid binary that nothing downstream can tell apart from a correct one. * The include tree is re-validated while running (INCLUDE_REFRESH_SECONDS, 60s). Resolving it once per process meant an edited .mqh was invisible until the next restart while compiles kept reporting success. * include_hash identifies the library a binary was built against, computed from the tree the compiler actually read rather than from the source, so drift stays detectable instead of being asserted away. * include_files (opt-in via COMPILE_INCLUDE_DIGESTS) adds a digest per named header, because the tree hash cannot say WHAT moved: upgrading the stock library and editing a caller's own shared header both move it, and the correct responses are opposites - no rebuild versus rebuild everything. With a local cache configured the server compiles a throwaway EA 180s after start, so a real caller does not pay MetaEditor's 30-55s cold load. It is gated: delayed until the VM has finished launching terminals, claimed once per host via an exclusive file (every API process exposes /compile and they share the cache, so ungated this starts one MetaEditor per terminal), and it yields to real work rather than making a caller queue behind it. scripts/config_helper.py raises the generated proxy timeouts: a queue of compiles outlives nginx's 60s default, and then the caller gets an HTML error page instead of the JSON this endpoint documents. Docs in docs/compiling.md, linked from README and docs/rest-api.md, with a CHANGELOG entry and a config.yaml.example block. --- CHANGELOG.md | 62 +++ README.md | 1 + config/config.yaml.example | 15 + docs/compiling.md | 295 +++++++++++ docs/rest-api.md | 1 + mt5api/config.py | 90 ++++ mt5api/handlers/compile.py | 766 ++++++++++++++++++++++++++++ mt5api/main.py | 7 + mt5api/server.py | 37 +- scripts/config_helper.py | 8 + scripts/prune-terminal-logs.sh | 110 ++++ tests/test_compile.py | 895 +++++++++++++++++++++++++++++++++ 12 files changed, 2283 insertions(+), 4 deletions(-) create mode 100644 docs/compiling.md create mode 100644 mt5api/handlers/compile.py create mode 100755 scripts/prune-terminal-logs.sh create mode 100644 tests/test_compile.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a144eb4..1b4b8d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,68 @@ The project follows [Semantic Versioning](https://semver.org/): patch = bug fixe ## [Unreleased] +### Added + +- **`POST /compile` — MQL5 source in, `.ex5` out.** MetaEditor is the only + thing that can produce an `.ex5` and it only runs on Windows, which this + stack already has. Anything that generates or patches EA source elsewhere — + CI, a code generator, a web app, an agent — can now get a binary back over + HTTP instead of putting a human on an RDP session. + + Source text only: no caller-supplied path, no file reads, and no control over + MetaEditor's arguments. `filename` is cosmetic and gets stripped to a bare + stem, so it cannot escape the per-request temp directory, which is removed on + every exit path including timeouts. The handler never touches the MT5 SDK, so + it cannot trade or restart a terminal. + + Three MetaEditor behaviours the implementation absorbs, because each one + produces a wrong answer if you take the obvious path: + + - It exits NON-ZERO on warnings as well as errors, so the exit code cannot + decide success. The log is parsed for counts and the produced binary is the + tiebreaker. A warning-only build is a success. + - It writes its log as UTF-16LE with a BOM. Decoded as UTF-8 you get + NUL-riddled mojibake and every count regex silently stops matching. + - It can report success and produce no file. That is returned as a failure + with a non-zero error count, never as `ok: true` — a caller that trusts + `ok` and finds no binary has nothing to fall back to. + + Compiles serialize behind one lock and the request stays synchronous. + MetaEditor compiles are sub-second for normal EAs, so a job queue would add + lost jobs, status polling and restart recovery to buy nothing. A caller that + waits longer than the deadline plus 30s gets a 504 rather than a hung + connection. + + `compile_local_cache` mirrors the toolchain onto local disk on first use, for + installs whose terminals sit on a network or host-shared mount. Measured on a + docker-hosted Windows VM with terminals on a 9p share, a compile MetaEditor + timed at 4.1s took 29s wall-clock on every request — the cost is loading a + 105MB binary across the mount, not compiling. Mirroring takes it to local + disk once per process, and a mirror that cannot be built falls back to the + shared copy rather than failing the request. + + Documented in [docs/compiling.md](docs/compiling.md). + +- **`compile_api_token` — a second, compile-only credential.** `api_token` + unlocks order placement, position management and terminal restart. Handing + that to something whose only job is compiling hands it the trading account + too. `compile_api_token` is accepted on `/compile` and rejected on every + other route; `api_token` keeps working everywhere including `/compile`. + Existing auth behaviour for existing routes is unchanged, and leaving the new + token empty changes nothing. + + Compile target, include directory, work directory and timeout are overridable + via `COMPILE_TERMINAL_DIR`, `COMPILE_INCLUDE_DIR`, `COMPILE_WORK_DIR` and + `COMPILE_TIMEOUT` (default `30s`, hard ceiling `60s`). + +- **30 tests** (`tests/test_compile.py`) covering the response contract and the + MetaEditor quirks above: real UTF-16LE log bytes decoded and parsed, warnings + not failing a build, `ok: true` never returned without a binary, temp + directories removed after success, failure and timeout, `/log:` and `/inc:` + unreachable from the request body, path traversal in `filename` neutralised, + and the compile token refused on `/account`, `/positions`, `/orders`, + `POST /orders`, `/terminal` and `/terminal/restart`. + ## [v4.12.2] — 2026-08-20 ### Fixed diff --git a/README.md b/README.md index 7e70747..a2fc515 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,7 @@ The old README became a massive wall of API shit, so the details now live in sep | Place/close shit and inspect orders, positions, and history | [Trading and history API](docs/trading-and-history.md) | | Run Strategy Tester jobs and get the artifacts back | [Backtesting](docs/backtesting.md) | | Throw giant parameter sweeps at the Strategy Tester | [Backtest optimization](docs/backtest-optimization.md) | +| Turn MQL5 source into an `.ex5` without touching MetaEditor | [Compiling MQL5](docs/compiling.md) | | Wire the MCP endpoints into your agent of choice | [MCP and agent integrations](docs/mcp-and-agents.md) | | Copy working curl and Go examples instead of guessing | [Clients and examples](docs/clients-and-examples.md) | | Operate the bastard: Make targets, ports, remote access, concurrency, and logs | [Operations](docs/operations.md) | diff --git a/config/config.yaml.example b/config/config.yaml.example index 79bddb6..37552c4 100644 --- a/config/config.yaml.example +++ b/config/config.yaml.example @@ -5,6 +5,21 @@ # Bearer token for API auth. Empty string = no auth (open to anyone on the network). api_token: "" +# Optional SECOND bearer token, accepted ONLY on POST /compile. +# +# api_token unlocks order placement, position management and terminal restart. +# Anything that only needs to compile MQL5 - a build pipeline, a code +# generator, a third-party service - should not be holding that. Set this and +# hand out this one instead; it is rejected on every other route. +# Empty string = /compile accepts api_token only. +compile_api_token: "" + +# Mirror the compile toolchain (MetaEditor + includes, ~105MB) onto local disk +# on first use. Only worth setting when the terminals live on a network or +# host-shared mount, where MetaEditor's load time dominates the compile - see +# docs/compiling.md. Empty = compile straight from the terminal directory. +compile_local_cache: "" + # Automatically reboot the VM every N minutes to flush DWM/VirtIO-GPU state # before sustained-load crashes wedge the MT5 SDK pipe. 0 = disabled. reboot_interval: 30 diff --git a/docs/compiling.md b/docs/compiling.md new file mode 100644 index 0000000..0ac9dda --- /dev/null +++ b/docs/compiling.md @@ -0,0 +1,295 @@ +# Compiling MQL5 + +`POST /compile` takes MQL5 source text and hands back a compiled `.ex5`. No +Windows machine, no MetaEditor GUI, no RDP session. + +This exists because MetaEditor is the only thing that can produce an `.ex5`, and +it only runs on Windows — which this stack already has. If you are generating, +templating, or patching EA source anywhere else (CI, a code generator, a web +app, an agent), this is how you get a binary out of it without a human in the +loop. + +The endpoint is source-text-only by design. It takes no path, reads nothing off +disk on your behalf, and gives you no control over MetaEditor's arguments. See +[Security](#security) for why that matters. + +## Quick start + +```bash +curl -sS -X POST -H "Authorization: Bearer $MT5_API_TOKEN" \ + -H 'Content-Type: application/json' \ + "$MT5_API_URL/compile" \ + -d '{ + "source": "void OnTick() {}", + "filename": "MyEA.mq5" + }' | jq -r 'if .ok then .ex5_base64 else .log end' +``` + +Decode the binary: + +```bash +curl -sS -X POST -H "Authorization: Bearer $MT5_API_TOKEN" \ + -H 'Content-Type: application/json' \ + "$MT5_API_URL/compile" -d @payload.json \ + | jq -r .ex5_base64 | base64 -d > MyEA.ex5 +``` + +## Request + +| Field | Type | Required | Notes | +| --- | --- | --- | --- | +| `source` | string | yes | The complete `.mq5` text. | +| `filename` | string | no | Cosmetic. Reduced to a bare stem — see [Security](#security). Defaults to `ea.mq5`. | +| `ea_version` | string | no | Recorded in the server log to correlate a compile with your build. Not passed to the compiler. | + +## Responses + +Every response is JSON, including failures. `log` is always a string, never +`null`, and is capped at the last 8 KB. + +**200 — compiled.** + +```json +{ + "ok": true, + "ex5_base64": "AAEC...", + "log": "Result: 0 errors, 1 warnings, 143 msec elapsed", + "warnings": 1, + "include_hash": "sha256:a8694a3b..." +} +``` + +`ok: true` always comes with a non-empty `ex5_base64`. If the compiler reports +success but produces no binary, that is reported as a failure, not a success. + +`include_files` is present only when `COMPILE_INCLUDE_DIGESTS` names something, +and carries a digest per matching header: + +```json +"include_files": {"MyLib.mqh": "sha256:17d64580..."} +``` + +It exists because `include_hash` cannot say *what* moved. Upgrading the stock +MQL5 library and editing your own shared header both change it, and the correct +responses are opposites — the first needs no rebuild, the second needs every +dependent artifact rebuilt. Without per-header digests a caller has to assume +the expensive one. Keys are paths relative to the include root. + +A configured header that is **absent from the tree** is absent from this map — +never null. That is a distinct third case, and it is worth handling as one: + +| Observed | Means | Response | +| --- | --- | --- | +| digest unchanged | that header did not move | nothing, even if `include_hash` moved | +| digest changed | that header was edited | rebuild what depends on it | +| **key gone** | the header is not in the tree the compiler read | **do not rebuild — investigate** | +| whole field absent | nothing is configured, or the tree was unreadable | unknown; treat as rebuild | + +The third row is the one that bites. A missing header is a provisioning fault on +*this* host, not drift on the caller's side, and rebuilding against it cannot +succeed — every dependent source fails with `error 106: file not found`. A +caller that treats a vanished key as "rebuild" queues a run of guaranteed 422s +instead of alerting someone. + +`include_hash` identifies the include tree this binary was built against — +sha256 over the relative paths and contents of everything under the `/inc:` +root. Record it with the build and "which artifacts used a library that has +since changed" becomes a comparison instead of an assumption. It is computed +from the tree the compiler actually read, so if the mirror were stale the hash +reports the stale tree rather than claiming the current one. Omitted if the +tree cannot be read; never guessed. + +**422 — the source did not compile.** This is your code being wrong, not the +server. `log` carries MetaEditor's own diagnostics. + +```json +{ + "ok": false, + "log": "ea.mq5(12,5) : error 160: expression of 'void' type is illegal\nResult: 3 errors, 0 warnings", + "errors": 3 +} +``` + +A missing `#include` lands here too — it is a compile error, not a server fault. + +**504 — the compile exceeded its deadline**, or waited too long for the +compiler lock. + +```json +{"ok": false, "log": "compile timeout after 30s"} +``` + +**400 — malformed request** (no `source`, or not a string). +**401 — bad or missing credentials.** +**500 — the host cannot compile** (MetaEditor missing, unreadable, unlaunchable). + +Warnings never fail a compile. MetaEditor exits non-zero on warnings as well as +errors, so the exit code is not used to decide the outcome — the log is parsed +and the produced binary is the tiebreaker. + +## Configuration + +```yaml +# config/config.yaml + +# Optional second credential, accepted ONLY on /compile. api_token keeps +# working everywhere including here. Leave empty if you do not need it. +compile_api_token: "" +``` + +Environment overrides, all optional: + +| Variable | Default | Purpose | +| --- | --- | --- | +| `COMPILE_API_TOKEN` | unset | Compile-only bearer token. | +| `COMPILE_TERMINAL_DIR` | `terminals/metaquotes/base` | Terminal directory whose `MetaEditor64.exe` is used. | +| `COMPILE_INCLUDE_DIR` | `/MQL5` | Passed to MetaEditor as `/inc:`. `#include ` resolves under `/Include/`. | +| `COMPILE_WORK_DIR` | `logs/compile-work` | Parent of the per-request temp directories. | +| `COMPILE_LOCAL_CACHE` | unset | Mirror the toolchain onto local disk — see [Performance](#performance). | +| `COMPILE_INCLUDE_DIGESTS` | unset | Comma-separated globs (relative to the include root) whose per-file digests are reported as `include_files`. | +| `COMPILE_TIMEOUT` | `30s` | Per-compile deadline. Hard ceiling 60s. | + +### Which terminal compiles + +Compiles run in a dedicated terminal directory, not a broker terminal. +MetaEditor is a separate executable from `terminal64.exe` and does not contend +with a running terminal for the SDK — but it does write into the directory it +compiles under, and sharing that with a live terminal or a running Strategy +Tester buys nothing. `metaquotes/base` is the default because it ships +MetaEditor and no terminal runs out of it. + +### Custom includes + +Drop `.mqh` files into `/Include/` and `#include ` +resolves. If you are generating source against a library you maintain +elsewhere, sync it into that directory as part of your deploy — a stale `.mqh` +compiles clean and then misbehaves at runtime, which is the worst failure shape +available. + +Edits are picked up by a running server without a restart. With +`COMPILE_LOCAL_CACHE` set the include tree is re-validated against the source at +most once every `INCLUDE_REFRESH_SECONDS` (60s), so an updated header reaches +builds within that window rather than at the next restart. If you have just +changed a shared library and are about to rebuild everything that depends on +it, let that window pass first — otherwise the first builds of the batch can +still use the previous copy, and they will report success while doing it. + +## Performance + +If your terminals live on a network or host-shared mount, the compile is not +what costs you — dragging MetaEditor across the mount is. + +Measured on a docker-hosted Windows VM with the terminals on a 9p share: a +compile MetaEditor's own log timed at **4.1s** took **29s** wall-clock, on every +request. The page cache does not save you, and three concurrent requests then +queue past a reverse proxy's default 60s read timeout. + +Set `COMPILE_LOCAL_CACHE` to a path on the VM's own disk: + +```yaml +compile_local_cache: "C:\\mt5-compile" +``` + +On the first compile the server copies MetaEditor, its `Config`, and the include +tree there — about 105 MB, once per process — and compiles from local disk +afterwards. `terminal64.exe`, `metatester64.exe` and `Bases` are not copied; a +compile never reads them. + +Only files that are missing or changed are copied, compared on size and +whole-second mtime, so a mirror that survived a restart costs nothing to +re-validate. This matters more than it looks: the stock MQL5 `Include` tree is +~260 files, and copying it unconditionally took **103s** on a 9p share — +directly in front of the first caller after every process start. Populating it +the first time still costs that once. + +Even with the mirror warm, the first compile after a VM restart pays +MetaEditor's own cold load: measured at **30–55s** on a busy host, against ~2–3s +warm. With `COMPILE_LOCAL_CACHE` set, the server now absorbs that itself — it +compiles a throwaway EA in the background 180s after start, so a real caller +finds MetaEditor warm. + +That warm-up is deliberately gated, and the gates matter more than the compile: + +- **Delayed**, because the VM launches every terminal at boot and a MetaEditor + run added to that contention slows the guest exactly when its health probe is + most marginal. +- **Claimed once per VM**, via an exclusive file in the cache directory. Every + API process exposes `/compile` and they share that directory, so an ungated + warm-up starts one MetaEditor per terminal — twenty at once on a busy host. + The claim expires after an hour so a process killed mid-warm-up cannot + disable warm-up permanently. +- **Yields to real work.** It takes the compile lock non-blocking and gives up + if a compile is running; a caller queuing behind a warm-up would defeat it. + +Failures are logged and ignored — the worst case is that the next real compile +pays the cold load, exactly as before. + +If the mirror cannot be built (unwritable path, no space) the server logs a +warning and falls back to the configured paths. A slow compile beats a broken +one. + +Leave it unset if your terminals are already on local disk — then there is +nothing to win. + +## Concurrency + +Compiles are serialized behind one lock and the request is synchronous. A queue +would add failure modes — lost jobs, status polling, restart recovery — to buy +nothing, because the work itself cannot overlap. Concurrent callers wait; a +caller that waits longer than the compile deadline plus 30s gets a 504 rather +than a hung connection. + +**Tell callers to compile one at a time.** Since the lock serializes them +anyway, concurrency buys no throughput — but it does stack waits on top of a +fixed deadline, so the callers at the back of the queue start returning 504 +while the same requests sent sequentially would all have succeeded. Measured on +a loaded host: five concurrent compiles of an EA including `` +took 92s in total, with individual waits of 24/47/72/90/92s and one 504. + +Sustained parallel compiles are also enough to saturate a VM's CPU. If +something supervises that VM on a health probe with a short timeout, the probe +starts failing under compile load and the supervisor restarts a VM that was +merely busy — converting a slow batch into an outage. Give the probe enough +timeout to survive a CPU-saturated host. + +If you put a reverse proxy in front of this, give `/compile` a read +timeout longer than your compile deadline. nginx defaults to 60s, which a +queue of slow compiles will cross — and then the caller gets the proxy's +HTML error page instead of the JSON documented here. + +Each request compiles inside its own temp directory, which is removed on every +exit path including timeouts and crashes. Two callers compiling different +sources under the same `filename` cannot see each other's output. + +## Security + +The endpoint accepts source text and nothing else. Specifically: + +- **No caller-supplied paths.** `filename` is stripped to a bare stem — + directories, drive letters and `..` are discarded — then re-suffixed. It + cannot escape the temp directory. +- **No caller-controlled compiler arguments.** `/compile:`, `/log:` and `/inc:` + are all computed server-side. Sending `log` or `include` in the body does + nothing. +- **No file reads on your behalf.** The only files touched are the ones written + into the request's own temp directory. +- **No trading surface.** The handler never touches the MT5 SDK. It cannot + place an order, modify a position, or restart a terminal. + +### A compile-only credential + +`compile_api_token` exists so a system that only needs to compile does not have +to hold a token that can also trade. + +`api_token` unlocks order placement, position management and terminal restart. +If you are handing a build pipeline, a code generator, or a third-party service +the ability to compile, giving it `api_token` gives it your account too. Set +`compile_api_token` and hand out that instead: it is accepted on `/compile` and +rejected everywhere else. + +Both tokens work on `/compile`. Only `api_token` works anywhere else. + +> Compiling arbitrary source is not a sandbox. MetaEditor parses attacker- +> controlled text, and MQL5 has `#import` for DLLs. Treat `/compile` as trusted +> input regardless of which token opens it, and do not expose it to the open +> internet. diff --git a/docs/rest-api.md b/docs/rest-api.md index de935f8..ed6d871 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -12,6 +12,7 @@ One HTTP surface for routing, auth, health, terminal control, account state, and - [Market data](market-data.md) - [Trading and history](trading-and-history.md) - [Backtesting](backtesting.md) +- [Compiling MQL5](compiling.md) ## API diff --git a/mt5api/config.py b/mt5api/config.py index 1338f4c..cfdf921 100644 --- a/mt5api/config.py +++ b/mt5api/config.py @@ -211,6 +211,96 @@ def load_terminal_config(): INSTANCE = normalize_instance(_args.instance or _terminal_config.get("instance")) PORT = _args.port or _terminal_config.get("port") or 6542 API_TOKEN = _args.token or os.environ.get("API_TOKEN", "") + +# ── Compile endpoint (POST /compile) ───────────────────────────── +# A SECOND credential that opens /compile and nothing else. The existing +# API_TOKEN unlocks order placement, position management and terminal restart; +# a caller that only compiles must not hold that. server.py +# enforces the split: API_TOKEN works everywhere, COMPILE_API_TOKEN works only +# on /compile. +_compile_cfg = load_yaml_config() + + +def _compile_setting(env_name, yaml_name, default=""): + """Env wins, then config.yaml, then the default -- the same precedence the + rest of this file uses for tokens and timeouts.""" + value = os.environ.get(env_name) + if value not in (None, ""): + return value + value = _compile_cfg.get(yaml_name) + if value not in (None, ""): + return str(value) + return default + + +COMPILE_API_TOKEN = _compile_setting("COMPILE_API_TOKEN", "compile_api_token") + +# Compiles run in a DEDICATED terminal directory, never a broker terminal. +# MetaEditor is independent of terminal64.exe, so this is not about the +# compiler colliding with a running terminal - it is about not putting write +# traffic and a temp tree inside a directory a live terminal or a running +# tester owns. metaquotes/base is the natural pick: it ships MetaEditor64.exe +# and no instance runs a terminal out of it. +COMPILE_TERMINAL_DIR = _compile_setting( + "COMPILE_TERMINAL_DIR", "compile_terminal_dir" +) or os.path.join(BROKERS_DIR, "metaquotes", "base") +COMPILE_METAEDITOR = os.path.join(COMPILE_TERMINAL_DIR, "MetaEditor64.exe") + +# Passed to MetaEditor as /inc:. This is the MQL5 directory (the PARENT of +# Include), because that is what /inc: expects - `#include ` resolves +# to /Include/Foo.mqh. +COMPILE_INCLUDE_DIR = _compile_setting( + "COMPILE_INCLUDE_DIR", "compile_include_dir" +) or os.path.join(COMPILE_TERMINAL_DIR, "MQL5") + +# Per-request temp directories live here, not in the Windows user temp, so a +# crashed process leaves its debris somewhere visible and prunable. +COMPILE_WORK_DIR = _compile_setting( + "COMPILE_WORK_DIR", "compile_work_dir" +) or os.path.join(BASE_DIR, "logs", "compile-work") + +# Optional local mirror of the compile toolchain. +# +# When the terminal directory sits on a network or host-shared mount, the cost +# of a compile is dominated by dragging MetaEditor across it, not by compiling: +# measured on a docker-hosted Windows VM with the terminals on a 9p share, a +# compile MetaEditor itself timed at 4.1s took 29s wall-clock, every time -- +# the page cache does not save you. +# +# Point this at a path on the VM's own disk and the toolchain (MetaEditor, its +# Config, and the include tree) is mirrored there once per process, then +# compiled from local disk. Empty = disabled, which is the right default for +# any install whose terminals are already local. +COMPILE_LOCAL_CACHE = _compile_setting("COMPILE_LOCAL_CACHE", "compile_local_cache") + +# Comma-separated globs naming headers whose INDIVIDUAL digests are reported +# alongside the whole-tree include_hash, matched against paths relative to the +# include root (e.g. "MyLib*.mqh, Trade/Trade.mqh"). +# +# The tree hash alone tells a caller that SOMETHING under /inc: moved, which is +# all that is needed to detect drift. It cannot say what: an upgrade of the +# stock MQL5 library looks identical to an edit of the caller's own shared +# header, and those have opposite correct responses - the first needs no +# rebuild at all, the second needs every dependent artifact rebuilt. Naming the +# few headers a caller actually owns lets them tell those apart. +# +# Empty = omit the field entirely. Deliberately not defaulted to the whole +# tree: ~260 digests per response is a payload, not an answer. +COMPILE_INCLUDE_DIGESTS = _compile_setting( + "COMPILE_INCLUDE_DIGESTS", "compile_include_digests" +) + +# Default 30s, hard ceiling 60s. MetaEditor compiles are seconds; anything +# approaching the ceiling means something is wrong, and holding the connection +# longer does not help the caller. +COMPILE_TIMEOUT_CEILING_SECONDS = 60 +COMPILE_TIMEOUT_SECONDS = max( + 1, + min( + COMPILE_TIMEOUT_CEILING_SECONDS, + parse_duration_to_seconds(_compile_setting("COMPILE_TIMEOUT", "compile_timeout", "30s")) or 30, + ), +) UTC_OFFSET_RAW = _args.utc_offset if _args.utc_offset is not None else os.environ.get("UTC_OFFSET", "") UTC_OFFSET_SECONDS = parse_duration_to_seconds(UTC_OFFSET_RAW) UTC_OFFSET_HOURS = UTC_OFFSET_SECONDS / 3600.0 diff --git a/mt5api/handlers/compile.py b/mt5api/handlers/compile.py new file mode 100644 index 0000000..f0256e4 --- /dev/null +++ b/mt5api/handlers/compile.py @@ -0,0 +1,766 @@ +"""Compile MQL5 source to .ex5 via MetaEditor. + +Why this exists: building an EA otherwise requires a human on a Windows machine +with MetaEditor installed. This lets an automated caller compile source it has +generated or assembled, on the host that already has the toolchain. + +Threat model, because this endpoint takes arbitrary text from a caller and hands +it to a compiler: + + * SOURCE TEXT ONLY. There is no caller-supplied path anywhere. `filename` is + reduced to a bare stem and re-suffixed, so "../../terminal64" or + "C:\\Windows\\x" cannot escape the temp directory. + * The caller controls neither /log: nor /inc:. Both are computed here. + * Nothing is read from disk on the caller's behalf. The only files touched are + the ones written into a per-request temp directory, which is removed on + every exit path. + * The handler cannot trade, cannot restart a terminal, and never touches the + MT5 SDK. It shells out to MetaEditor64.exe and reads back two files. + +MetaEditor specifics worth knowing before editing this: + + * It exits NON-ZERO on warnings as well as errors, so the exit code cannot + decide success. The log is the authority, and the produced .ex5 is the + tiebreaker. + * It writes its log as UTF-16LE with a BOM. Decoding it as UTF-8 yields + mojibake and every regex below silently stops matching. + * It emits the .ex5 beside the source file, not into a configurable output + path — which is exactly why compiling inside the temp directory is enough + to keep concurrent requests from colliding over output names. +""" + +import base64 +import fnmatch +import hashlib +import os +import re +import shutil +import subprocess +import tempfile +import threading +import time + +from flask import jsonify, request + +from mt5api.config import ( + COMPILE_INCLUDE_DIGESTS, + COMPILE_INCLUDE_DIR, + COMPILE_LOCAL_CACHE, + COMPILE_METAEDITOR, + COMPILE_TIMEOUT_SECONDS, + COMPILE_WORK_DIR, +) +from mt5api.logger import log + +#: The client treats a non-JSON body as a broken host, and `log` as always a +#: string. Both are enforced at every return in this module. +MAX_LOG_BYTES = 8192 + +#: MetaEditor is single-instance per installation directory and compiles are +#: short. One lock, no job queue — a queue would add failure modes (lost jobs, +#: status polling, restart recovery) to buy nothing at this duration. +_COMPILE_LOCK = threading.Lock() + +#: How long a caller may wait for the lock on top of its own compile budget. +#: Without a bound, a stuck compile turns every later request into a hung +#: connection, which is the one failure the client cannot distinguish from a +#: dead host. +_LOCK_WAIT_MARGIN_SECONDS = 30 + +#: "Result: 0 errors, 2 warnings, 143 msec elapsed" — the summary line. Builds +#: differ on singular/plural and on the "Result:" prefix, so match the counts +#: rather than the whole line. +_ERRORS_RE = re.compile(r"(\d+)\s+error", re.IGNORECASE) +_WARNINGS_RE = re.compile(r"(\d+)\s+warning", re.IGNORECASE) + +#: Fallback when no summary line is present: per-diagnostic lines look like +#: "ea.mq5(12,5) : error 123: ';' - unexpected token". +_ERROR_LINE_RE = re.compile(r":\s*error\s+\d+", re.IGNORECASE) +_WARNING_LINE_RE = re.compile(r":\s*warning\s+\d+", re.IGNORECASE) + + +#: Resolved once per process by _local_toolchain(): (metaeditor, include_dir), +#: or None when no mirror is configured or the mirror could not be built. +_LOCAL_TOOLCHAIN = None +_LOCAL_TOOLCHAIN_RESOLVED = False + +#: When the mirrored include tree was last re-checked against the source. +_INCLUDES_CHECKED_AT = 0.0 + +#: Cached include-tree digest, and the root it was computed for. +_INCLUDE_HASH = None +_INCLUDE_HASH_KEY = None + +#: Cached per-header digests, keyed by the same root. +_INCLUDE_FILES = None + +#: Globs (relative to the include root) whose per-file digests are reported. +_INCLUDE_DIGEST_PATTERNS = tuple( + pattern.strip() for pattern in (COMPILE_INCLUDE_DIGESTS or "").split(",") if pattern.strip() +) + +#: How stale the mirrored include tree may get before it is re-validated. +#: +#: MetaEditor and its Config are effectively immutable between deployments, but +#: the include tree is NOT: a shared library gets edited and every build after +#: that is supposed to pick it up. Resolving the mirror once per process meant +#: an edited .mqh was invisible until the next restart, and the compile that +#: used the old copy still returned ok:true - a silently stale binary, which is +#: the worst failure shape this endpoint has. Bound that window instead. +INCLUDE_REFRESH_SECONDS = 60 + +#: What MetaEditor actually needs to compile. Deliberately NOT the whole +#: terminal directory - that also holds terminal64.exe, metatester64.exe and +#: Bases, roughly 350MB of things a compile never reads. +_TOOLCHAIN_ITEMS = ("MetaEditor64.exe", "Config", "MQL5") + +#: Compile a throwaway EA in the background shortly after start, so the first +#: REAL caller does not pay MetaEditor's cold load - measured at 30-55s on a +#: busy host against ~2-3s warm. Only meaningful with a local mirror, which is +#: also the only configuration where the cold cost is worth eliminating. +WARMUP_ENABLED = bool(COMPILE_LOCAL_CACHE) + +#: Wait this long before warming. The VM launches every terminal at boot, and a +#: MetaEditor run added to that contention makes the guest slower precisely when +#: its health probe is most marginal. Warm once things have settled instead. +WARMUP_DELAY_SECONDS = 180 + +#: Every API process on a VM exposes /compile and they share COMPILE_LOCAL_CACHE, +#: so an ungated warm-up means one MetaEditor per terminal - 20 of them on this +#: host, all at once. A claim file in the shared cache keeps it to one per VM. +_WARMUP_CLAIM = ".warmup-claim" + +#: A claim older than this is treated as abandoned, so a process that died +#: holding it cannot disable warm-up for every future boot. The cache directory +#: survives reboots; the claim inside it must not be permanent. +WARMUP_CLAIM_TTL_SECONDS = 3600 + + +def _is_current(src, dst): + """True when dst already matches src closely enough to skip re-copying. + + Size plus whole-second mtime. Whole seconds because the mirror and the + source can sit on filesystems with different timestamp resolution, and a + sub-second difference there would make every file look stale forever. + """ + try: + s_stat = os.stat(src) + d_stat = os.stat(dst) + except OSError: + return False + return s_stat.st_size == d_stat.st_size and int(s_stat.st_mtime) == int(d_stat.st_mtime) + + +def _mirror_tree(src, dst): + """Copy a directory into the mirror, skipping files already current. + + Returns (files copied, files pruned). + + This is deliberately not shutil.copytree(dirs_exist_ok=True): that re-copies + every file on every call, and this runs at the first compile after each + process start. The stock MQL5 Include tree is ~260 files, so on a slow + host-shared mount an unconditional re-copy puts tens of seconds in front of + the first compile - which is exactly the window where a caller is already + waiting on a reverse-proxy timeout. + """ + copied = 0 + expected = set() + for root, _dirs, files in os.walk(src): + relative = os.path.relpath(root, src) + target = dst if relative == "." else os.path.join(dst, relative) + os.makedirs(target, exist_ok=True) + for name in files: + src_file = os.path.join(root, name) + dst_file = os.path.join(target, name) + expected.add(os.path.relpath(dst_file, dst)) + if _is_current(src_file, dst_file): + continue + # copy2 preserves mtime, which is what makes the next run a no-op. + shutil.copy2(src_file, dst_file) + copied += 1 + + # Copying alone leaves a one-way mirror: a header deleted from the source + # stays here and keeps resolving, so `#include ` still compiles + # and the tree the compiler reads drifts from the tree anyone is + # maintaining. Drop what the source no longer has. + # + # Guarded on a non-empty source: if the mount is unreachable the walk above + # yields nothing, and pruning against that would delete the entire mirror + # over a transient failure. + removed = 0 + if expected: + for root, _dirs, files in os.walk(dst): + for name in files: + stale = os.path.join(root, name) + if os.path.relpath(stale, dst) not in expected: + try: + os.remove(stale) + removed += 1 + except OSError: + pass # in use or already gone; the next pass retries + return copied, removed + + +def _include_hash(include_root): + """sha256 over the include tree the compiler was actually given. + + Returns "sha256:", or None if the tree cannot be read. + + Deliberately hashed from the tree passed as /inc:, NOT from the source it + was mirrored from. The point of this value is to answer "which library is + this binary built against" after the fact, so it has to describe what the + compiler read - if the mirror were stale, a hash of the source would assert + the opposite and be worse than no hash at all. + + Covers relative paths as well as contents so that adding, renaming or + removing a header changes the digest, not just editing one. + + Cached against _INCLUDE_HASH_KEY because the tree only changes when this + module copies into it, and re-reading ~16MB per compile buys nothing. + """ + global _INCLUDE_HASH, _INCLUDE_HASH_KEY + + root = os.path.join(include_root, "Include") + if not os.path.isdir(root): + root = include_root + if not os.path.isdir(root): + return None + if _INCLUDE_HASH_KEY == root and _INCLUDE_HASH: + return _INCLUDE_HASH + + digest = hashlib.sha256() + try: + for dirpath, dirnames, filenames in os.walk(root): + # Sorted so the digest is stable across filesystems that hand back + # directory entries in different orders. + dirnames.sort() + for name in sorted(filenames): + path = os.path.join(dirpath, name) + relative = os.path.relpath(path, root).replace(os.sep, "/") + digest.update(relative.encode("utf-8", "replace") + b"\0") + with open(path, "rb") as handle: + for chunk in iter(lambda h=handle: h.read(1024 * 1024), b""): + digest.update(chunk) + digest.update(b"\0") + except OSError as exc: + log.warning("compile: could not hash include tree (%s)", exc) + return None + + _INCLUDE_HASH_KEY = root + _INCLUDE_HASH = "sha256:" + digest.hexdigest() + return _INCLUDE_HASH + + +def _include_file_digests(include_root): + """Per-header digests for the globs in COMPILE_INCLUDE_DIGESTS. + + Returns {relative path: "sha256:"}, or {} when nothing is configured + or nothing matches. + + Exists because the tree hash cannot say WHAT moved. An upgrade of the stock + MQL5 library and an edit of a caller's own shared header both change it, + and the correct responses are opposites: the first needs no rebuild, the + second needs every dependent artifact rebuilt. Without this a caller has to + assume the expensive one. + + A configured header that is absent from the tree is simply absent here, not + null - "not in the tree the compiler read" is the thing worth knowing + before a rebuild, and a null would blur it with "present but unreadable". + + Same root and cache lifetime as _include_hash. + """ + global _INCLUDE_FILES + + if not _INCLUDE_DIGEST_PATTERNS: + return {} + root = os.path.join(include_root, "Include") + if not os.path.isdir(root): + root = include_root + if not os.path.isdir(root): + return {} + if _INCLUDE_HASH_KEY == root and _INCLUDE_FILES is not None: + return _INCLUDE_FILES + + digests = {} + try: + for dirpath, dirnames, filenames in os.walk(root): + dirnames.sort() + for name in sorted(filenames): + path = os.path.join(dirpath, name) + relative = os.path.relpath(path, root).replace(os.sep, "/") + if not any( + fnmatch.fnmatch(relative, pattern) for pattern in _INCLUDE_DIGEST_PATTERNS + ): + continue + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda h=handle: h.read(1024 * 1024), b""): + digest.update(chunk) + digests[relative] = "sha256:" + digest.hexdigest() + except OSError as exc: + log.warning("compile: could not hash individual includes (%s)", exc) + return {} + + _INCLUDE_FILES = digests + return digests + + +def _invalidate_include_hash(): + """Drop the cached digests. Called whenever this module writes into the tree.""" + global _INCLUDE_HASH, _INCLUDE_HASH_KEY, _INCLUDE_FILES + _INCLUDE_HASH = None + _INCLUDE_HASH_KEY = None + _INCLUDE_FILES = None + + +def _refresh_mirrored_includes(): + """Re-validate the mirrored include tree against the source. + + Runs at most once every INCLUDE_REFRESH_SECONDS, and only walks MQL5 - not + MetaEditor64.exe, which is 105MB and does not change under a running + process. + + This exists because an edited shared header must reach subsequent builds. + Without it the mirror is resolved once per process and an updated .mqh is + invisible until the next restart, while the compile that used the old copy + still returns ok:true. A silently stale binary is worse than a failed + compile: nothing downstream can tell it apart from a correct one. + + Caller must hold _COMPILE_LOCK. + """ + global _INCLUDES_CHECKED_AT + + if not _LOCAL_TOOLCHAIN or not COMPILE_LOCAL_CACHE: + return + if time.monotonic() - _INCLUDES_CHECKED_AT < INCLUDE_REFRESH_SECONDS: + return + + _INCLUDES_CHECKED_AT = time.monotonic() + src = os.path.join(os.path.dirname(COMPILE_METAEDITOR), "MQL5") + dst = os.path.join(COMPILE_LOCAL_CACHE, "MQL5") + if not os.path.isdir(src): + return + try: + copied, removed = _mirror_tree(src, dst) + if copied or removed: + _invalidate_include_hash() + log.info( + "compile: include tree refreshed (%d changed, %d pruned)", copied, removed + ) + except Exception as exc: # noqa: BLE001 - keep compiling with what we have + log.warning("compile: could not refresh includes (%s), using mirrored copy", exc) + + +def _local_toolchain(): + """Mirror the compile toolchain onto local disk, once per process. + + Returns (metaeditor_path, include_dir) to compile with, or None to use the + configured paths as-is. + + Failure here is never fatal: if the mirror cannot be built we log it and + fall back to the shared copy, which is slower but correct. A compile that + works slowly beats a compile that stops working because a cache directory + was not writable. + + Callers must hold _COMPILE_LOCK - this writes ~100MB and must not run twice + concurrently. + """ + global _LOCAL_TOOLCHAIN, _LOCAL_TOOLCHAIN_RESOLVED, _INCLUDES_CHECKED_AT + + if _LOCAL_TOOLCHAIN_RESOLVED: + _refresh_mirrored_includes() + return _LOCAL_TOOLCHAIN + + _LOCAL_TOOLCHAIN_RESOLVED = True + _INCLUDES_CHECKED_AT = time.monotonic() + if not COMPILE_LOCAL_CACHE: + return None + + source_dir = os.path.dirname(COMPILE_METAEDITOR) + try: + os.makedirs(COMPILE_LOCAL_CACHE, exist_ok=True) + for item in _TOOLCHAIN_ITEMS: + src = os.path.join(source_dir, item) + dst = os.path.join(COMPILE_LOCAL_CACHE, item) + if not os.path.exists(src): + continue + if os.path.isdir(src): + copied, removed = _mirror_tree(src, dst) + if copied or removed: + _invalidate_include_hash() + log.info( + "compile: mirrored %s (%d refreshed, %d pruned)", item, copied, removed + ) + elif not _is_current(src, dst): + shutil.copy2(src, dst) + + editor = os.path.join(COMPILE_LOCAL_CACHE, "MetaEditor64.exe") + if not os.path.exists(editor): + log.warning("compile: local cache built without MetaEditor, using shared copy") + return None + + # Includes come from the mirrored tree so a compile never reaches back + # across the slow mount for a .mqh either. + include_dir = os.path.join(COMPILE_LOCAL_CACHE, "MQL5") + if not os.path.isdir(include_dir): + include_dir = COMPILE_INCLUDE_DIR + + _LOCAL_TOOLCHAIN = (editor, include_dir) + log.info("compile: using local toolchain at %s", COMPILE_LOCAL_CACHE) + except Exception as exc: # noqa: BLE001 - fall back, never fail the request + log.warning("compile: could not build local toolchain (%s), using shared copy", exc) + _LOCAL_TOOLCHAIN = None + + return _LOCAL_TOOLCHAIN + + +def _tail(text, limit=MAX_LOG_BYTES): + """Last `limit` bytes of a log, as a string. Never None.""" + if not text: + return "" + encoded = text.encode("utf-8", errors="replace") + if len(encoded) <= limit: + return text + # Cut on a character boundary so the tail is still valid UTF-8. + return encoded[-limit:].decode("utf-8", errors="replace") + + +def _read_metaeditor_log(path): + """Decode MetaEditor's log file. Missing or unreadable reads as empty.""" + try: + with open(path, "rb") as handle: + raw = handle.read() + except OSError: + return "" + return _read_metaeditor_log_bytes(raw) + + +def _read_metaeditor_log_bytes(raw): + """Decode MetaEditor's UTF-16LE log bytes. + + Split out from the file read so the decoding — the part that actually goes + wrong — is testable against real bytes without a filesystem. + + Falls back through UTF-8 and latin-1 rather than raising: a log we cannot + decode must not turn a real compile result into a 500. + """ + if not raw: + return "" + + for encoding in ("utf-16", "utf-8-sig", "utf-8", "latin-1"): + try: + text = raw.decode(encoding) + except (UnicodeDecodeError, LookupError): + continue + # A UTF-16 log decoded as UTF-8 comes back riddled with NULs; treat that + # as a failed decode rather than returning shredded text. + if "\x00" in text: + continue + return text.replace("\r\n", "\n") + + return raw.decode("utf-8", errors="replace").replace("\x00", "") + + +def _parse_counts(log_text): + """(errors, warnings) from a MetaEditor log. + + Prefers the trailing summary line; falls back to counting diagnostics. When + neither is present the counts are 0 and the caller decides on the .ex5. + """ + errors = warnings = None + + # Walk backwards: the summary is the last line that carries both counts. + for line in reversed(log_text.splitlines()): + if _ERRORS_RE.search(line) and _WARNINGS_RE.search(line): + errors = int(_ERRORS_RE.search(line).group(1)) + warnings = int(_WARNINGS_RE.search(line).group(1)) + break + + if errors is None: + errors = len(_ERROR_LINE_RE.findall(log_text)) + if warnings is None: + warnings = len(_WARNING_LINE_RE.findall(log_text)) + + return errors, warnings + + +def _safe_stem(filename): + """Reduce a caller-supplied filename to a harmless stem. + + Everything structural is discarded: directories, drive letters, traversal. + What survives is a conservative character class, because this string becomes + a filename inside the temp directory AND the .ex5 name we read back. + """ + if not isinstance(filename, str): + filename = "" + # ntpath-style and posix separators, plus drive colons. + stem = re.split(r"[\\/]", filename)[-1] + stem = stem.split(":")[-1] + if stem.lower().endswith(".mq5"): + stem = stem[:-4] + stem = re.sub(r"[^A-Za-z0-9._-]", "_", stem).strip("._-") + return stem or "ea" + + +def _json(payload, status): + """Every exit from this handler goes through here, so the client never sees + a non-JSON body — which it is documented to treat as a broken host.""" + payload.setdefault("log", "") + if payload.get("log") is None: + payload["log"] = "" + return jsonify(payload), status + + +def _claim_warmup(): + """Claim the once-per-VM warm-up. True only for the process that wins it. + + O_CREAT|O_EXCL against a file in the shared cache, because the processes + racing for this are separate PIDs - a threading.Lock would gate one process + while the other nineteen went ahead and launched MetaEditor anyway. + """ + if not COMPILE_LOCAL_CACHE: + return False + path = os.path.join(COMPILE_LOCAL_CACHE, _WARMUP_CLAIM) + try: + os.makedirs(COMPILE_LOCAL_CACHE, exist_ok=True) + try: + if time.time() - os.path.getmtime(path) > WARMUP_CLAIM_TTL_SECONDS: + os.remove(path) + except OSError: + pass # absent, or someone else just removed it - both fine + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644) + try: + os.write(fd, time.strftime("%Y-%m-%dT%H:%M:%S").encode()) + finally: + os.close(fd) + return True + except OSError: + return False # another process holds the claim + + +def _warmup(): + """Run one throwaway compile so the first real caller finds MetaEditor warm.""" + time.sleep(WARMUP_DELAY_SECONDS) + if not _claim_warmup(): + return + + # Non-blocking: if a real compile is already running the toolchain is being + # warmed by it, and making a caller queue behind a warm-up would defeat the + # point of having one. + if not _COMPILE_LOCK.acquire(blocking=False): + log.info("compile warm-up: skipped, a compile is already running") + return + + work_dir = None + try: + local = _local_toolchain() + metaeditor, include_dir = local if local else (COMPILE_METAEDITOR, COMPILE_INCLUDE_DIR) + if not os.path.exists(metaeditor): + log.warning("compile warm-up: MetaEditor missing at %s", metaeditor) + return + + os.makedirs(COMPILE_WORK_DIR, exist_ok=True) + work_dir = tempfile.mkdtemp(prefix="warmup-", dir=COMPILE_WORK_DIR) + source_path = os.path.join(work_dir, "warmup.mq5") + with open(source_path, "w", encoding="utf-8-sig", newline="\r\n") as handle: + handle.write("int OnInit(){return(INIT_SUCCEEDED);}\nvoid OnTick(){}\n") + + started = time.monotonic() + subprocess.run( + [ + metaeditor, + f"/compile:{source_path}", + f"/log:{os.path.join(work_dir, 'warmup.log')}", + f"/inc:{include_dir}", + ], + capture_output=True, + # Generous: the whole point is absorbing the cold load, which is + # slower than any warm compile this deadline normally covers. + timeout=COMPILE_TIMEOUT_SECONDS + 120, + ) + log.info("compile warm-up done in %.1fs", time.monotonic() - started) + except Exception as exc: # noqa: BLE001 - a failed warm-up must not matter + log.warning( + "compile warm-up failed (%s); the first real compile pays the cold load", exc + ) + finally: + _COMPILE_LOCK.release() + if work_dir: + shutil.rmtree(work_dir, ignore_errors=True) + + +def start_warmup(): + """Kick the warm-up off in the background. Never blocks or fails startup.""" + if not WARMUP_ENABLED: + return + threading.Thread(target=_warmup, daemon=True, name="compile-warmup").start() + + +def compile_source(): + """POST /compile — synchronous compile of one .mq5 to one .ex5.""" + started = time.monotonic() + try: + return _compile_source_inner(started) + except Exception as exc: # noqa: BLE001 - the handler must never raise + log.exception("compile: unhandled error") + return _json( + {"ok": False, "log": f"internal error: {exc.__class__.__name__}: {exc}"}, + 500, + ) + + +def _compile_source_inner(started): + body = request.get_json(silent=True) + if not isinstance(body, dict): + return _json({"ok": False, "log": "body must be a JSON object"}, 400) + + source = body.get("source") + if not isinstance(source, str) or not source.strip(): + return _json({"ok": False, "log": "'source' must be a non-empty string"}, 400) + + stem = _safe_stem(body.get("filename") or "ea.mq5") + ea_version = body.get("ea_version") + + if not os.path.exists(COMPILE_METAEDITOR): + log.error("compile: MetaEditor missing at %s", COMPILE_METAEDITOR) + return _json( + {"ok": False, "log": f"MetaEditor not found at {COMPILE_METAEDITOR}"}, + 500, + ) + + # Bound the wait so a stuck compile fails fast for everyone behind it + # instead of holding connections open. + lock_deadline = COMPILE_TIMEOUT_SECONDS + _LOCK_WAIT_MARGIN_SECONDS + if not _COMPILE_LOCK.acquire(timeout=lock_deadline): + log.warning("compile: lock wait exceeded %ss", lock_deadline) + return _json( + { + "ok": False, + "log": f"compile busy: waited {lock_deadline}s for the compiler lock", + }, + 504, + ) + + try: + return _run_compile(stem, source, ea_version, started) + finally: + _COMPILE_LOCK.release() + + +def _run_compile(stem, source, ea_version, started): + # Under the lock, so the one-time ~100MB mirror cannot race itself. + local = _local_toolchain() + metaeditor, include_dir = local if local else (COMPILE_METAEDITOR, COMPILE_INCLUDE_DIR) + + os.makedirs(COMPILE_WORK_DIR, exist_ok=True) + work_dir = tempfile.mkdtemp(prefix="compile-", dir=COMPILE_WORK_DIR) + src_path = os.path.join(work_dir, f"{stem}.mq5") + ex5_path = os.path.join(work_dir, f"{stem}.ex5") + log_path = os.path.join(work_dir, "compile.log") + + try: + # utf-8-sig: MetaEditor honours a BOM and will otherwise guess the + # codepage, which mangles non-ASCII string literals in the source. + with open(src_path, "w", encoding="utf-8-sig", newline="\r\n") as handle: + handle.write(source) + + cmd = [ + metaeditor, + f"/compile:{src_path}", + f"/log:{log_path}", + f"/inc:{include_dir}", + ] + + log.info( + "compile start stem=%s ea_version=%s bytes=%d dir=%s", + stem, ea_version, len(source), work_dir, + ) + + timed_out = False + returncode = None + try: + completed = subprocess.run( + cmd, + cwd=work_dir, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=COMPILE_TIMEOUT_SECONDS, + check=False, + ) + returncode = completed.returncode + except subprocess.TimeoutExpired: + timed_out = True + except OSError as exc: + log.error("compile: could not launch MetaEditor: %s", exc) + return _json({"ok": False, "log": f"could not launch MetaEditor: {exc}"}, 500) + + if timed_out: + log.warning("compile timeout after %ss stem=%s", COMPILE_TIMEOUT_SECONDS, stem) + return _json( + {"ok": False, "log": f"compile timeout after {COMPILE_TIMEOUT_SECONDS}s"}, + 504, + ) + + log_text = _read_metaeditor_log(log_path) + errors, warnings = _parse_counts(log_text) + + ex5_bytes = b"" + if os.path.exists(ex5_path): + try: + with open(ex5_path, "rb") as handle: + ex5_bytes = handle.read() + except OSError as exc: + log.error("compile: could not read .ex5: %s", exc) + + elapsed = round(time.monotonic() - started, 3) + + # Success needs BOTH a clean log and an actual binary. Reporting ok:true + # without a binary is the one thing the client cannot recover from, and + # MetaEditor's exit code is not usable here because warnings make it + # non-zero too. + if errors == 0 and ex5_bytes: + log.info( + "compile ok stem=%s bytes=%d warnings=%d rc=%s dur=%.3fs", + stem, len(ex5_bytes), warnings, returncode, elapsed, + ) + body = { + "ok": True, + "ex5_base64": base64.b64encode(ex5_bytes).decode("ascii"), + "log": _tail(log_text), + "warnings": warnings, + } + # Identifies the library this binary was built against, so a caller + # can answer "is this still current?" later without having to trust + # that a sync had landed at the time. Omitted rather than guessed if + # the tree cannot be read. + digest = _include_hash(include_dir) + if digest: + body["include_hash"] = digest + # Only alongside the tree hash: on its own it would say which of a + # caller's headers changed without saying whether anything else did. + per_file = _include_file_digests(include_dir) if digest else {} + if per_file: + body["include_files"] = per_file + return _json(body, 200) + + # No binary but a clean log means MetaEditor failed in a way it did not + # report as a diagnostic. That is still the caller's compile failing, so + # it belongs on the 422 path with at least one error rather than a 500 — + # but say so, because an empty log would otherwise look like success. + if errors == 0 and not ex5_bytes: + errors = 1 + log_text = ( + (log_text.rstrip() + "\n" if log_text.strip() else "") + + f"compile produced no .ex5 (MetaEditor exit code {returncode})" + ) + + log.info( + "compile failed stem=%s errors=%d warnings=%d rc=%s dur=%.3fs", + stem, errors, warnings, returncode, elapsed, + ) + return _json( + {"ok": False, "log": _tail(log_text), "errors": errors}, + 422, + ) + finally: + shutil.rmtree(work_dir, ignore_errors=True) diff --git a/mt5api/main.py b/mt5api/main.py index ac71e0d..19a7a5e 100644 --- a/mt5api/main.py +++ b/mt5api/main.py @@ -10,6 +10,7 @@ from mt5api.backtest import jobs as backtest_jobs from mt5api.config import ACCOUNT, API_TOKEN, BROKER, HOST, INSTANCE, MODE, PORT +from mt5api.handlers import compile as compile_handler from mt5api.logger import log from mt5api.mcp_server import build_mcp_server from mt5api.monitor import start_monitor @@ -222,6 +223,12 @@ def main(): daemon=True, ).start() + # Absorbs MetaEditor's cold load before a real caller meets it. Self-gating: + # no-op unless a local compile cache is configured, delayed until the VM has + # finished launching terminals, and claimed once per VM so N API processes + # do not each launch their own MetaEditor. + compile_handler.start_warmup() + log.info( "HTTP API listening on %s:%d (waitress, threads=%d, conn_limit=%d, max_queue_depth=%d)", HOST, PORT, WSGI_THREADS, WSGI_CONNECTION_LIMIT, MAX_QUEUE_DEPTH, diff --git a/mt5api/server.py b/mt5api/server.py index 1aca624..127394a 100644 --- a/mt5api/server.py +++ b/mt5api/server.py @@ -1,11 +1,19 @@ import os import time -from flask import Flask, abort, g, request +from flask import Flask, abort, g, jsonify, request from flask_compress import Compress from mt5api.backtest import handler as backtest_handler -from mt5api.config import API_TOKEN -from mt5api.handlers import account, history, orders, positions, symbols, terminal +from mt5api.config import API_TOKEN, COMPILE_API_TOKEN +from mt5api.handlers import ( + account, + compile as compile_handler, + history, + orders, + positions, + symbols, + terminal, +) from mt5api.logger import log app = Flask(__name__) @@ -27,9 +35,25 @@ def _start_request(): g.req_id, request.method, request.full_path, _client_ip(), request.headers.get("User-Agent", "-"), ) + auth = request.headers.get("Authorization", "") + + # /compile carries a SECOND, compile-only credential. + # + # A caller that only needs to compile must not hold a token that can also + # place orders, close positions or restart a terminal. So API_TOKEN + # is accepted everywhere including here, while COMPILE_API_TOKEN is accepted + # ONLY on this path - every other route falls through to the original check + # below, which compares against API_TOKEN alone and therefore rejects it. + if request.path == "/compile": + accepted = [f"Bearer {t}" for t in (API_TOKEN, COMPILE_API_TOKEN) if t] + if accepted and auth not in accepted: + # JSON rather than Flask's HTML error page: the client is documented + # to treat a non-JSON body as a broken host. + return jsonify({"ok": False, "log": "unauthorized"}), 401 + return + if not API_TOKEN: return - auth = request.headers.get("Authorization", "") if auth != f"Bearer {API_TOKEN}": abort(401) @@ -94,6 +118,11 @@ def _end_request(response): app.get("/history/orders")(history.get_orders) app.get("/history/deals")(history.get_deals) +# ── Compile ────────────────────────────────────────────────────── +# Source text in, .ex5 out. Auth for this one route is handled in +# _start_request above; it accepts COMPILE_API_TOKEN as well as API_TOKEN. +app.post("/compile")(compile_handler.compile_source) + # ── Backtest ───────────────────────────────────────────────────── app.post("/backtest/build-ini")(backtest_handler.build_ini_route) app.post("/backtest/build-set")(backtest_handler.build_set_route) diff --git a/scripts/config_helper.py b/scripts/config_helper.py index 92c360b..db11cec 100644 --- a/scripts/config_helper.py +++ b/scripts/config_helper.py @@ -254,6 +254,14 @@ def main(): f" proxy_pass $vm_upstream;\n" f" proxy_set_header Host $host;\n" f" proxy_set_header X-Forwarded-For $remote_addr;\n" + # POST /compile is synchronous and serialized behind one + # lock, so a queue of compiles can outlive nginx's 60s + # default. When it does, the caller gets nginx's HTML error + # page instead of the JSON the endpoint documents - and the + # API's own 504, which IS JSON, never gets to be sent. + # Long enough for a queue; the handler still bounds itself. + f" proxy_read_timeout 300s;\n" + f" proxy_send_timeout 300s;\n" f" }}" ) diff --git a/scripts/prune-terminal-logs.sh b/scripts/prune-terminal-logs.sh new file mode 100755 index 0000000..733ac4c --- /dev/null +++ b/scripts/prune-terminal-logs.sh @@ -0,0 +1,110 @@ +#!/bin/sh +# Retention for MT5 terminal and tester-agent logs. +# +# These are not the API's own logs (rotate-logs.sh handles those). They are +# written by terminal64.exe and metatester64.exe inside each terminal +# directory, in UTF-16, and nothing prunes them: MT5 has no retention setting +# and the log-rotator sidecar only ever saw data/shared/logs. +# +# Two controls, because age alone is not enough. A high-frequency strategy +# logs every order placement, cancellation and modification, so one backtest +# can produce tens of gigabytes in a single day — a 7-day window would let a +# disk fill long before the first prune fired. Age bounds slow creep; the size +# cap bounds the actual hazard. +# +# RETAIN_DAYS delete *.log older than this +# MAX_LOG_BYTES truncate any single *.log larger than this +# IDLE_MINUTES never touch a file written more recently than this, so a +# running backtest never loses its own diagnostics +# +# Truncate-in-place (: >) rather than delete for the size cap: the terminal +# holds these files open, so removing the inode would leave the writer pointed +# at a deleted file and the space would not come back until it exited. + +set -eu + +TERMINAL_ROOTS="${TERMINAL_ROOTS:-/terminals}" +RETAIN_DAYS="${RETAIN_DAYS:-7}" +MAX_LOG_BYTES="${MAX_LOG_BYTES:-2147483648}" +IDLE_MINUTES="${IDLE_MINUTES:-30}" +INTERVAL="${INTERVAL:-3600}" +DRY_RUN="${DRY_RUN:-0}" + +log() { + printf '[%s] [logprune] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" +} + +human() { + awk -v b="$1" 'BEGIN { + split("B KB MB GB TB", u, " "); i = 1 + while (b >= 1024 && i < 5) { b /= 1024; i++ } + printf "%.1f%s", b, u[i] + }' +} + +# Every terminal log directory: /logs, /Tester/logs and +# /Tester/Agent-*/logs. Matching on the path keeps us off the rest +# of the terminal tree (Bases, MQL5, Reports) entirely. +find_logs() { + root="$1" + shift + [ -d "$root" ] || return 0 + find "$root" \ + \( -path '*/Tester/logs/*.log' -o -path '*/Tester/Agent-*/logs/*.log' \) \ + -type f "$@" 2>/dev/null || true +} + +prune_once() { + for root in $(echo "$TERMINAL_ROOTS" | tr ':' ' '); do + [ -d "$root" ] || { + log "root missing, skipping: $root" + continue + } + + # 1. Age: delete whole files past the retention window. + find_logs "$root" -mtime "+${RETAIN_DAYS}" | while read -r f; do + size=$(wc -c <"$f" 2>/dev/null || echo 0) + if [ "$DRY_RUN" = "1" ]; then + log "would delete (age) $(human "$size") $f" + elif rm -f "$f"; then + log "deleted (age) $(human "$size") $f" + fi + done + + # 2. Size: truncate oversized files that nothing is actively writing. + find_logs "$root" -size "+$((MAX_LOG_BYTES / 1024))k" | while read -r f; do + size=$(wc -c <"$f" 2>/dev/null || echo 0) + [ "$size" -gt "$MAX_LOG_BYTES" ] || continue + + # Skip anything touched inside the idle window — that is a live run. + if [ -n "$(find "$f" -mmin "-${IDLE_MINUTES}" 2>/dev/null)" ]; then + log "skipped (active, $(human "$size")) $f" + continue + fi + + if [ "$DRY_RUN" = "1" ]; then + log "would truncate $(human "$size") $f" + elif : >"$f"; then + log "truncated $(human "$size") $f" + fi + done + done + + # Per-file actions are logged above; each `find | while read` runs in its + # own subshell, so totals cannot be accumulated here without a temp file. + log "pass complete" +} + +log "starting (roots=$TERMINAL_ROOTS retain_days=$RETAIN_DAYS max_log=$(human "$MAX_LOG_BYTES") idle_min=$IDLE_MINUTES interval=${INTERVAL}s dry_run=$DRY_RUN)" + +if [ "${RUN_ONCE:-0}" = "1" ]; then + prune_once + exit 0 +fi + +while true; do + if ! prune_once; then + log "prune_once failed (continuing)" + fi + sleep "$INTERVAL" +done diff --git a/tests/test_compile.py b/tests/test_compile.py new file mode 100644 index 0000000..621c1c8 --- /dev/null +++ b/tests/test_compile.py @@ -0,0 +1,895 @@ +"""Contract tests for POST /compile. + +The endpoint takes caller-supplied MQL5 text, hands it to MetaEditor, and +returns a .ex5. Two classes of thing are worth pinning here: + + * The contract itself. Clients are written against it, and its two + hard rules — `ok: true` implies a non-empty binary, and `log` is always a + string — are the ones that break the caller silently if they regress. + * The MetaEditor quirks the implementation exists to absorb: a UTF-16LE log, + an exit code that goes non-zero on warnings, and a compiler that reports + success in the log while producing no file. + +MetaEditor itself is a Windows binary, so subprocess.run is patched. What is +NOT patched is the log decoding or the count parsing — those run against real +UTF-16LE bytes, because that is where the bugs live. +""" + +import base64 +import json +import os +import time + +import pytest + +from mt5api.handlers import compile as compile_handler + + +# ── Helpers ────────────────────────────────────────────────────────────────── + +def _utf16_log(text): + """Bytes exactly as MetaEditor writes them: UTF-16LE with a BOM, CRLF.""" + return text.replace("\n", "\r\n").encode("utf-16") + + +SUCCESS_LOG = ( + "MetaEditor 5 build 4885 started\n" + "ea.mq5 : information: compiling 'ea.mq5'\n" + "Result: 0 errors, 0 warnings, 121 msec elapsed\n" +) + +WARNING_LOG = ( + "ea.mq5(14,7) : warning 43: possible loss of data due to type conversion\n" + "Result: 0 errors, 2 warnings, 138 msec elapsed\n" +) + +ERROR_LOG = ( + "ea.mq5(12,5) : error 160: expression of 'void' type is illegal\n" + "ea.mq5(13,1) : error 145: '}' - unexpected end of program\n" + "ea.mq5(13,1) : error 100: ';' - semicolon expected\n" + "Result: 3 errors, 0 warnings, 96 msec elapsed\n" +) + +MISSING_INCLUDE_LOG = ( + "ea.mq5(3,11) : error 133: cannot open include file " + "'SomeLibrary.mqh'\n" + "Result: 1 errors, 0 warnings, 44 msec elapsed\n" +) + + +class FakeCompleted: + def __init__(self, returncode=0): + self.returncode = returncode + + +@pytest.fixture +def compile_env(monkeypatch, tmp_path): + """Point the handler at a temp workspace and a MetaEditor that 'exists'.""" + fake_editor = tmp_path / "MetaEditor64.exe" + fake_editor.write_bytes(b"MZ") + work = tmp_path / "work" + monkeypatch.setattr(compile_handler, "COMPILE_METAEDITOR", str(fake_editor)) + monkeypatch.setattr(compile_handler, "COMPILE_WORK_DIR", str(work)) + monkeypatch.setattr(compile_handler, "COMPILE_INCLUDE_DIR", str(tmp_path / "MQL5")) + monkeypatch.setattr(compile_handler, "COMPILE_TIMEOUT_SECONDS", 30) + return {"work": work, "editor": str(fake_editor)} + + +def _fake_metaeditor(log_bytes, ex5_bytes=None, returncode=0, record=None): + """Stand in for MetaEditor: writes the log it was given, and an .ex5 when + the compile is meant to have produced one.""" + def _run(cmd, **kwargs): + if record is not None: + record.append(cmd) + log_path = next(a.split(":", 1)[1] for a in cmd if a.startswith("/log:")) + src_path = next(a.split(":", 1)[1] for a in cmd if a.startswith("/compile:")) + with open(log_path, "wb") as fh: + fh.write(log_bytes) + if ex5_bytes is not None: + with open(os.path.splitext(src_path)[0] + ".ex5", "wb") as fh: + fh.write(ex5_bytes) + return FakeCompleted(returncode) + return _run + + +API_TOKEN = "full-api-token" +COMPILE_TOKEN = "compile-only-token" + + +@pytest.fixture +def client(monkeypatch): + """Flask client with BOTH tokens configured, so the split is exercised.""" + from mt5api import server + + monkeypatch.setattr(server, "API_TOKEN", API_TOKEN) + monkeypatch.setattr(server, "COMPILE_API_TOKEN", COMPILE_TOKEN) + server.app.config["TESTING"] = True + return server.app.test_client() + + +def _post(client, body, token=COMPILE_TOKEN): + headers = {"Content-Type": "application/json"} + if token is not None: + headers["Authorization"] = f"Bearer {token}" + return client.post("/compile", data=json.dumps(body), headers=headers) + + +# ── Log decoding and count parsing (no subprocess involved) ────────────────── + +def test_metaeditor_log_is_decoded_as_utf16(): + # Decoding UTF-16LE as UTF-8 yields NUL-riddled mojibake and every count + # regex silently stops matching, so this is the load-bearing decode. + text = compile_handler._read_metaeditor_log_bytes(_utf16_log(SUCCESS_LOG)) + assert "Result: 0 errors, 0 warnings" in text + assert "\x00" not in text + + +def test_counts_come_from_the_summary_line(): + assert compile_handler._parse_counts(ERROR_LOG) == (3, 0) + assert compile_handler._parse_counts(WARNING_LOG) == (0, 2) + assert compile_handler._parse_counts(SUCCESS_LOG) == (0, 0) + + +def test_counts_fall_back_to_counting_diagnostics_without_a_summary(): + no_summary = ( + "ea.mq5(12,5) : error 160: bad\n" + "ea.mq5(13,1) : error 145: worse\n" + "ea.mq5(14,2) : warning 43: meh\n" + ) + assert compile_handler._parse_counts(no_summary) == (2, 1) + + +def test_log_tail_is_capped_at_8kb_and_never_none(): + assert compile_handler._tail(None) == "" + assert compile_handler._tail("") == "" + big = "x" * 20000 + assert len(compile_handler._tail(big).encode("utf-8")) <= 8192 + # Multi-byte content must not be cut mid-character. + assert compile_handler._tail("é" * 20000).encode("utf-8") + + +# ── Filename handling: source text only, never a caller-supplied path ──────── + +@pytest.mark.parametrize("supplied,expected", [ + ("ea.mq5", "ea"), + ("MyEA.mq5", "MyEA"), + ("../../terminal64", "terminal64"), + ("..\\..\\Windows\\system32\\evil.mq5", "evil"), + ("C:\\Windows\\x.mq5", "x"), + ("/etc/passwd", "passwd"), + ("", "ea"), + (None, "ea"), + ("...", "ea"), + ("a b;c&d.mq5", "a_b_c_d"), +]) +def test_filename_is_reduced_to_a_harmless_stem(supplied, expected): + assert compile_handler._safe_stem(supplied) == expected + + +# ── The contract ───────────────────────────────────────────────────────────── + +def test_success_returns_the_binary_and_a_zero_warning_count(client, compile_env, monkeypatch): + ex5 = b"\x00ex5-binary-content" * 40 + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), ex5), + ) + resp = _post(client, {"source": "void OnTick(){}", "filename": "ea.mq5"}) + assert resp.status_code == 200 + body = resp.get_json() + assert body["ok"] is True + assert base64.b64decode(body["ex5_base64"]) == ex5 + assert body["warnings"] == 0 + assert isinstance(body["log"], str) + + +def test_warnings_do_not_fail_the_compile(client, compile_env, monkeypatch): + # MetaEditor exits NON-ZERO on warnings. Trusting the exit code here would + # turn every warning into a failed compile. + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(WARNING_LOG), b"ex5", returncode=1), + ) + resp = _post(client, {"source": "void OnTick(){}"}) + assert resp.status_code == 200 + assert resp.get_json()["ok"] is True + assert resp.get_json()["warnings"] == 2 + + +def test_compile_errors_return_422_with_the_compiler_message(client, compile_env, monkeypatch): + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(ERROR_LOG), None, returncode=1), + ) + resp = _post(client, {"source": "garbage"}) + assert resp.status_code == 422 + body = resp.get_json() + assert body["ok"] is False + assert body["errors"] == 3 + assert "expression of 'void' type is illegal" in body["log"] + assert "ex5_base64" not in body + + +def test_missing_include_is_a_compile_error_not_a_500(client, compile_env, monkeypatch): + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(MISSING_INCLUDE_LOG), None, returncode=1), + ) + resp = _post(client, {"source": "#include "}) + assert resp.status_code == 422 + body = resp.get_json() + assert body["errors"] == 1 + assert "cannot open include file" in body["log"] + + +def test_clean_log_without_a_binary_is_never_reported_as_success(client, compile_env, monkeypatch): + # The one outcome the client cannot recover from: ok:true with no .ex5. + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), None, returncode=0), + ) + resp = _post(client, {"source": "void OnTick(){}"}) + assert resp.status_code == 422 + body = resp.get_json() + assert body["ok"] is False + assert body["errors"] >= 1 + assert "produced no .ex5" in body["log"] + + +def test_empty_binary_is_not_success(client, compile_env, monkeypatch): + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), b""), + ) + resp = _post(client, {"source": "void OnTick(){}"}) + assert resp.status_code == 422 + + +def test_timeout_returns_504_with_a_string_log(client, compile_env, monkeypatch): + def _timeout(cmd, **kwargs): + raise compile_handler.subprocess.TimeoutExpired(cmd, 30) + monkeypatch.setattr(compile_handler.subprocess, "run", _timeout) + resp = _post(client, {"source": "void OnTick(){}"}) + assert resp.status_code == 504 + body = resp.get_json() + assert body["ok"] is False + assert body["log"] == "compile timeout after 30s" + + +def test_missing_source_is_rejected_as_json(client, compile_env): + for body in [{}, {"source": ""}, {"source": " "}, {"source": 5}]: + resp = _post(client, body) + assert resp.status_code == 400 + assert isinstance(resp.get_json()["log"], str) + + +def test_handler_never_raises_and_always_answers_json(client, compile_env, monkeypatch): + def _boom(cmd, **kwargs): + raise RuntimeError("kaboom") + monkeypatch.setattr(compile_handler.subprocess, "run", _boom) + resp = _post(client, {"source": "void OnTick(){}"}) + assert resp.status_code == 500 + assert resp.is_json + assert isinstance(resp.get_json()["log"], str) + + +# ── Temp directory hygiene ─────────────────────────────────────────────────── + +@pytest.mark.parametrize("log_text,ex5,expect_status", [ + (SUCCESS_LOG, b"ex5", 200), + (ERROR_LOG, None, 422), +]) +def test_temp_directory_is_removed_after_success_and_failure( + client, compile_env, monkeypatch, log_text, ex5, expect_status +): + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(log_text), ex5), + ) + resp = _post(client, {"source": "void OnTick(){}"}) + assert resp.status_code == expect_status + work = compile_env["work"] + leftovers = list(work.iterdir()) if work.exists() else [] + assert leftovers == [], f"temp dirs left behind: {leftovers}" + + +def test_temp_directory_is_removed_after_a_timeout(client, compile_env, monkeypatch): + def _timeout(cmd, **kwargs): + raise compile_handler.subprocess.TimeoutExpired(cmd, 30) + monkeypatch.setattr(compile_handler.subprocess, "run", _timeout) + _post(client, {"source": "void OnTick(){}"}) + work = compile_env["work"] + assert (list(work.iterdir()) if work.exists() else []) == [] + + +# ── The compiler invocation itself ─────────────────────────────────────────── + +def test_caller_controls_neither_the_log_nor_the_include_argument( + client, compile_env, monkeypatch +): + recorded = [] + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), b"ex5", record=recorded), + ) + _post(client, { + "source": "void OnTick(){}", + "filename": "ea.mq5", + "log": "C:\\evil.log", + "include": "C:\\evil", + }) + cmd = recorded[0] + log_arg = next(a for a in cmd if a.startswith("/log:")) + inc_arg = next(a for a in cmd if a.startswith("/inc:")) + assert "evil" not in log_arg + assert "evil" not in inc_arg + assert inc_arg == f"/inc:{compile_handler.COMPILE_INCLUDE_DIR}" + # The source compiled is the one we wrote, inside the temp dir. + src_arg = next(a for a in cmd if a.startswith("/compile:")) + assert src_arg.endswith("ea.mq5") + assert str(compile_env["work"]) in src_arg + + +# ── Auth: a compile-only credential that opens nothing else ────────────────── + +def test_compile_accepts_the_compile_token(client, compile_env, monkeypatch): + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), b"ex5"), + ) + assert _post(client, {"source": "void OnTick(){}"}, token=COMPILE_TOKEN).status_code == 200 + + +def test_compile_also_accepts_the_existing_api_token(client, compile_env, monkeypatch): + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), b"ex5"), + ) + assert _post(client, {"source": "void OnTick(){}"}, token=API_TOKEN).status_code == 200 + + +@pytest.mark.parametrize("token", [None, "", "wrong-token", "Bearer-ish"]) +def test_compile_rejects_bad_or_missing_credentials(client, compile_env, token): + resp = _post(client, {"source": "void OnTick(){}"}, token=token) + assert resp.status_code in (401, 403) + # Even the auth failure answers JSON: a non-JSON body means "broken host" + # to this client, which would send it down a different recovery path. + assert resp.is_json + + +@pytest.mark.parametrize("method,path", [ + ("get", "/account"), + ("get", "/positions"), + ("get", "/orders"), + ("post", "/orders"), + ("post", "/terminal/restart"), + ("get", "/terminal"), +]) +def test_compile_token_is_refused_on_every_other_route(client, method, path): + """The whole point of the second credential. + + The existing token opens order placement, position management and terminal + restart. A caller that only needs to compile must not be able to reach + any of it, so the compile token has to fail CLOSED everywhere else — and + the trade routes are the ones that matter. + """ + resp = getattr(client, method)( + path, headers={"Authorization": f"Bearer {COMPILE_TOKEN}"} + ) + assert resp.status_code == 401, f"{method.upper()} {path} accepted the compile token" + + +def test_the_full_api_token_still_works_on_other_routes(client, monkeypatch): + """The existing gate must be unchanged for existing routes.""" + resp = client.get("/ping", headers={"Authorization": f"Bearer {API_TOKEN}"}) + assert resp.status_code == 200 + + +def test_other_routes_still_reject_a_missing_token(client): + assert client.get("/ping").status_code == 401 + + +# ── Local toolchain mirror ─────────────────────────────────────────────────── + +@pytest.fixture(autouse=True) +def _reset_toolchain_cache(monkeypatch): + """The mirror resolves once per process; tests must not inherit each + other's resolution.""" + monkeypatch.setattr(compile_handler, "_LOCAL_TOOLCHAIN", None) + monkeypatch.setattr(compile_handler, "_LOCAL_TOOLCHAIN_RESOLVED", False) + + +def test_no_mirror_configured_uses_the_shared_toolchain(client, compile_env, monkeypatch): + monkeypatch.setattr(compile_handler, "COMPILE_LOCAL_CACHE", "") + recorded = [] + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), b"ex5", record=recorded), + ) + _post(client, {"source": "void OnTick(){}"}) + assert recorded[0][0] == compile_env["editor"] + + +def test_mirror_copies_the_toolchain_and_compiles_from_it( + client, compile_env, monkeypatch, tmp_path +): + # The shared "terminal dir" gets an include tree and a Config dir alongside + # MetaEditor, mirroring the real layout. + shared = tmp_path + (shared / "MQL5" / "Include").mkdir(parents=True) + (shared / "MQL5" / "Include" / "Lib.mqh").write_text("// lib") + (shared / "Config").mkdir() + cache = tmp_path / "local-cache" + monkeypatch.setattr(compile_handler, "COMPILE_LOCAL_CACHE", str(cache)) + + recorded = [] + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), b"ex5", record=recorded), + ) + assert _post(client, {"source": "void OnTick(){}"}).status_code == 200 + + # Compiled from the mirror, not the shared copy. + assert recorded[0][0] == str(cache / "MetaEditor64.exe") + inc = next(a for a in recorded[0] if a.startswith("/inc:")) + assert inc == f"/inc:{cache / 'MQL5'}" + # Includes came along, so a compile never reaches back across the mount. + assert (cache / "MQL5" / "Include" / "Lib.mqh").read_text() == "// lib" + # And only what a compile needs: no terminal64.exe, no Bases. + assert not (cache / "terminal64.exe").exists() + + +def test_mirror_is_built_once_per_process(client, compile_env, monkeypatch, tmp_path): + cache = tmp_path / "cache-once" + monkeypatch.setattr(compile_handler, "COMPILE_LOCAL_CACHE", str(cache)) + copies = [] + real_copy = compile_handler.shutil.copy2 + monkeypatch.setattr( + compile_handler.shutil, "copy2", + lambda *a, **k: (copies.append(a[0]), real_copy(*a, **k))[1], + ) + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), b"ex5"), + ) + for _ in range(3): + _post(client, {"source": "void OnTick(){}"}) + # 105MB does not get copied on every request. + assert len(copies) == 1, f"toolchain copied {len(copies)} times" + + +def test_a_warm_mirror_is_not_re_copied_on_the_next_process_start( + client, compile_env, monkeypatch, tmp_path +): + """The regression that matters after a restart. + + The mirror is built once per PROCESS, but the VM restarts several times a + day and the mirror survives on local disk. Re-copying the whole tree each + time puts that cost in front of the first compile after every restart, and + the stock MQL5 Include tree is ~260 files - big enough on a slow mount to + push that first caller past a reverse-proxy timeout. + """ + shared = tmp_path + include = shared / "MQL5" / "Include" / "Trade" + include.mkdir(parents=True) + for name in ("Trade.mqh", "SymbolInfo.mqh", "PositionInfo.mqh"): + (include / name).write_text(f"// {name}") + (shared / "Config").mkdir() + cache = tmp_path / "warm-cache" + monkeypatch.setattr(compile_handler, "COMPILE_LOCAL_CACHE", str(cache)) + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), b"ex5"), + ) + + # First process: builds the mirror. + assert _post(client, {"source": "void OnTick(){}"}).status_code == 200 + assert (cache / "MQL5" / "Include" / "Trade" / "Trade.mqh").exists() + + # Second process, same on-disk mirror: nothing should be copied again. + compile_handler._LOCAL_TOOLCHAIN = None + compile_handler._LOCAL_TOOLCHAIN_RESOLVED = False + copies = [] + real_copy = compile_handler.shutil.copy2 + monkeypatch.setattr( + compile_handler.shutil, "copy2", + lambda *a, **k: (copies.append(a[0]), real_copy(*a, **k))[1], + ) + assert _post(client, {"source": "void OnTick(){}"}).status_code == 200 + assert copies == [], f"re-copied {len(copies)} unchanged file(s) on restart" + + +def test_a_changed_include_is_still_picked_up_by_the_mirror( + client, compile_env, monkeypatch, tmp_path +): + """Skipping unchanged files must not mean serving a stale library. + + An edited .mqh that compiles clean but is a version behind is worse than a + slow compile, so the staleness check has to actually notice the edit. + """ + shared = tmp_path + include = shared / "MQL5" / "Include" + include.mkdir(parents=True) + (include / "Lib.mqh").write_text("// v1") + (shared / "Config").mkdir() + cache = tmp_path / "refresh-cache" + monkeypatch.setattr(compile_handler, "COMPILE_LOCAL_CACHE", str(cache)) + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), b"ex5"), + ) + _post(client, {"source": "void OnTick(){}"}) + assert (cache / "MQL5" / "Include" / "Lib.mqh").read_text() == "// v1" + + # Edit the source library, then restart. Size and mtime both move. + (include / "Lib.mqh").write_text("// v2 is longer than v1") + os.utime(include / "Lib.mqh", (time.time() + 10, time.time() + 10)) + compile_handler._LOCAL_TOOLCHAIN = None + compile_handler._LOCAL_TOOLCHAIN_RESOLVED = False + + _post(client, {"source": "void OnTick(){}"}) + assert (cache / "MQL5" / "Include" / "Lib.mqh").read_text() == "// v2 is longer than v1" + + +def test_a_broken_mirror_falls_back_instead_of_failing_the_compile( + client, compile_env, monkeypatch, tmp_path +): + # A compile that works slowly beats one that stops working because a cache + # directory was not writable. + monkeypatch.setattr(compile_handler, "COMPILE_LOCAL_CACHE", str(tmp_path / "nope")) + monkeypatch.setattr( + compile_handler.shutil, "copy2", + lambda *a, **k: (_ for _ in ()).throw(OSError("disk full")), + ) + recorded = [] + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), b"ex5", record=recorded), + ) + resp = _post(client, {"source": "void OnTick(){}"}) + assert resp.status_code == 200 + assert recorded[0][0] == compile_env["editor"] + + +def test_an_edited_include_reaches_later_compiles_without_a_restart( + client, compile_env, monkeypatch, tmp_path +): + """A shared header is edited while the process keeps running. + + The mirror is otherwise resolved once per process, so without a periodic + re-check the old copy is used until the next restart - and the compile that + used it still returns ok:true. A silently stale binary is the worst outcome + available here: nothing downstream can distinguish it from a correct one. + """ + shared = tmp_path + include = shared / "MQL5" / "Include" + include.mkdir(parents=True) + (include / "Lib.mqh").write_text("// v1") + (shared / "Config").mkdir() + cache = tmp_path / "refresh-live" + monkeypatch.setattr(compile_handler, "COMPILE_LOCAL_CACHE", str(cache)) + monkeypatch.setattr(compile_handler, "INCLUDE_REFRESH_SECONDS", 0) + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), b"ex5"), + ) + + _post(client, {"source": "void OnTick(){}"}) + assert (cache / "MQL5" / "Include" / "Lib.mqh").read_text() == "// v1" + + # Edit the shared header. No restart, no cache reset. + (include / "Lib.mqh").write_text("// v2 and longer") + os.utime(include / "Lib.mqh", (time.time() + 5, time.time() + 5)) + + _post(client, {"source": "void OnTick(){}"}) + assert (cache / "MQL5" / "Include" / "Lib.mqh").read_text() == "// v2 and longer" + + +def test_the_include_recheck_is_rate_limited(client, compile_env, monkeypatch, tmp_path): + """The re-check walks the source tree, which sits on the slow mount. + + Doing that on every compile would put the walk in front of every caller, so + it is bounded by INCLUDE_REFRESH_SECONDS rather than run each time. + """ + shared = tmp_path + (shared / "MQL5" / "Include").mkdir(parents=True) + (shared / "MQL5" / "Include" / "Lib.mqh").write_text("// v1") + (shared / "Config").mkdir() + cache = tmp_path / "ratelimit" + monkeypatch.setattr(compile_handler, "COMPILE_LOCAL_CACHE", str(cache)) + monkeypatch.setattr(compile_handler, "INCLUDE_REFRESH_SECONDS", 3600) + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), b"ex5"), + ) + _post(client, {"source": "void OnTick(){}"}) + + walks = [] + real_walk = compile_handler.os.walk + monkeypatch.setattr( + compile_handler.os, "walk", + lambda *a, **k: (walks.append(a[0]), real_walk(*a, **k))[1], + ) + for _ in range(5): + _post(client, {"source": "void OnTick(){}"}) + assert walks == [], f"re-walked the include tree {len(walks)} time(s) inside the window" + + +# ── include_hash: which library was this binary built against? ─────────────── + +def _hash_env(monkeypatch, tmp_path, name="hash-cache"): + """Shared toolchain + mirror, with one header in the include tree.""" + shared = tmp_path + include = shared / "MQL5" / "Include" + include.mkdir(parents=True, exist_ok=True) + (include / "Lib.mqh").write_text("// v1") + (shared / "Config").mkdir(exist_ok=True) + cache = tmp_path / name + monkeypatch.setattr(compile_handler, "COMPILE_LOCAL_CACHE", str(cache)) + monkeypatch.setattr(compile_handler, "INCLUDE_REFRESH_SECONDS", 0) + monkeypatch.setattr(compile_handler, "_INCLUDE_HASH", None) + monkeypatch.setattr(compile_handler, "_INCLUDE_HASH_KEY", None) + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), b"ex5"), + ) + return include + + +def test_success_reports_the_include_hash(client, compile_env, monkeypatch, tmp_path): + _hash_env(monkeypatch, tmp_path) + body = _post(client, {"source": "void OnTick(){}"}).get_json() + assert body["include_hash"].startswith("sha256:") + assert len(body["include_hash"]) == len("sha256:") + 64 + + +def test_the_include_hash_is_stable_across_compiles(client, compile_env, monkeypatch, tmp_path): + """Two builds against an unchanged library must be comparable.""" + _hash_env(monkeypatch, tmp_path) + first = _post(client, {"source": "void OnTick(){}"}).get_json()["include_hash"] + second = _post(client, {"source": "void OnTick(){}"}).get_json()["include_hash"] + assert first == second + + +def test_editing_a_header_changes_the_include_hash(client, compile_env, monkeypatch, tmp_path): + """The whole point: drift has to be detectable after the fact.""" + include = _hash_env(monkeypatch, tmp_path) + before = _post(client, {"source": "void OnTick(){}"}).get_json()["include_hash"] + + (include / "Lib.mqh").write_text("// v2 is different") + os.utime(include / "Lib.mqh", (time.time() + 5, time.time() + 5)) + + after = _post(client, {"source": "void OnTick(){}"}).get_json()["include_hash"] + assert after != before, "an edited header left the include hash unchanged" + + +def test_adding_a_header_changes_the_include_hash(client, compile_env, monkeypatch, tmp_path): + """Contents alone would miss this - the digest covers paths too.""" + include = _hash_env(monkeypatch, tmp_path) + before = _post(client, {"source": "void OnTick(){}"}).get_json()["include_hash"] + + (include / "Extra.mqh").write_text("// new") + after = _post(client, {"source": "void OnTick(){}"}).get_json()["include_hash"] + assert after != before, "a new header left the include hash unchanged" + + +def test_the_hash_describes_the_mirror_not_the_source(client, compile_env, monkeypatch, tmp_path): + """If the mirror is stale, the hash must report what was COMPILED. + + Hashing the source instead would assert the build used a library it did not, + which is worse than reporting no hash at all. + """ + include = _hash_env(monkeypatch, tmp_path, name="mirror-truth") + body = _post(client, {"source": "void OnTick(){}"}).get_json() + mirrored = body["include_hash"] + + # Change the source but freeze the mirror by disabling any further refresh. + monkeypatch.setattr(compile_handler, "INCLUDE_REFRESH_SECONDS", 3600) + monkeypatch.setattr(compile_handler, "_INCLUDES_CHECKED_AT", time.monotonic()) + (include / "Lib.mqh").write_text("// source moved on without the mirror") + os.utime(include / "Lib.mqh", (time.time() + 9, time.time() + 9)) + + again = _post(client, {"source": "void OnTick(){}"}).get_json()["include_hash"] + assert again == mirrored, "hash followed the source instead of the compiled tree" + + +# ── Boot warm-up ───────────────────────────────────────────────────────────── +# +# The warm-up exists to move MetaEditor's cold load off the first real caller. +# Its gates matter more than the compile it runs: ungated, every API process on +# the VM launches its own MetaEditor at boot. + +def test_warmup_does_nothing_without_a_local_cache(monkeypatch): + """No mirror means no cold-load problem worth a background compile.""" + monkeypatch.setattr(compile_handler, "WARMUP_ENABLED", False) + started = [] + monkeypatch.setattr( + compile_handler.threading, "Thread", + lambda *a, **k: started.append(k) or pytest.fail("started a thread"), + ) + compile_handler.start_warmup() + assert started == [] + + +def test_only_one_process_claims_the_warmup(monkeypatch, tmp_path): + """The regression that matters. + + Every API process on the VM runs this code and they share the cache dir, so + the claim has to be exclusive ACROSS processes - twenty MetaEditors starting + together would recreate the CPU saturation the warm-up is meant to avoid. + """ + cache = tmp_path / "claim" + monkeypatch.setattr(compile_handler, "COMPILE_LOCAL_CACHE", str(cache)) + winners = [compile_handler._claim_warmup() for _ in range(20)] + assert winners.count(True) == 1, f"{winners.count(True)} processes claimed the warm-up" + + +def test_an_abandoned_claim_does_not_disable_warmup_forever(monkeypatch, tmp_path): + """The cache survives reboots, so the claim inside it must not be permanent. + + A process killed mid-warm-up leaves the file behind; without expiry that + would silently disable warm-up on this VM for good. + """ + cache = tmp_path / "stale" + monkeypatch.setattr(compile_handler, "COMPILE_LOCAL_CACHE", str(cache)) + assert compile_handler._claim_warmup() is True + assert compile_handler._claim_warmup() is False + + claim = cache / compile_handler._WARMUP_CLAIM + old = time.time() - (compile_handler.WARMUP_CLAIM_TTL_SECONDS + 60) + os.utime(claim, (old, old)) + + assert compile_handler._claim_warmup() is True, "an expired claim still blocked warm-up" + + +def test_warmup_yields_to_a_real_compile(monkeypatch, tmp_path): + """A caller must never queue behind a warm-up - that inverts its purpose.""" + cache = tmp_path / "yield" + monkeypatch.setattr(compile_handler, "COMPILE_LOCAL_CACHE", str(cache)) + monkeypatch.setattr(compile_handler, "WARMUP_DELAY_SECONDS", 0) + ran = [] + monkeypatch.setattr( + compile_handler.subprocess, "run", + lambda *a, **k: ran.append(a) or FakeCompleted(0), + ) + + compile_handler._COMPILE_LOCK.acquire() # stand in for a compile in flight + try: + compile_handler._warmup() + finally: + compile_handler._COMPILE_LOCK.release() + + assert ran == [], "warm-up ran MetaEditor while a compile held the lock" + + +def test_a_failing_warmup_is_swallowed(monkeypatch, tmp_path): + """Warm-up is an optimisation. It must never take the process down.""" + cache = tmp_path / "boom" + monkeypatch.setattr(compile_handler, "COMPILE_LOCAL_CACHE", str(cache)) + monkeypatch.setattr(compile_handler, "WARMUP_DELAY_SECONDS", 0) + monkeypatch.setattr(compile_handler, "COMPILE_METAEDITOR", str(tmp_path / "me.exe")) + (tmp_path / "me.exe").write_bytes(b"MZ") + monkeypatch.setattr( + compile_handler.subprocess, "run", + lambda *a, **k: (_ for _ in ()).throw(OSError("cannot launch")), + ) + compile_handler._warmup() # must not raise + # And the lock is released, so real compiles still work afterwards. + assert compile_handler._COMPILE_LOCK.acquire(blocking=False) + compile_handler._COMPILE_LOCK.release() + + +def test_a_header_deleted_from_source_is_pruned_from_the_mirror( + client, compile_env, monkeypatch, tmp_path +): + """Copy-only leaves a one-way mirror. + + A header removed from the source stayed in the mirror and kept resolving, + so `#include ` still compiled against a file nobody maintains. + Found in production: a test header deleted from the source was still being + included by the compiler hours later. + """ + include = _hash_env(monkeypatch, tmp_path, name="prune") + (include / "Doomed.mqh").write_text("// remove me") + cache = tmp_path / "prune" + _post(client, {"source": "void OnTick(){}"}) + assert (cache / "MQL5" / "Include" / "Doomed.mqh").exists() + + os.remove(include / "Doomed.mqh") + _post(client, {"source": "void OnTick(){}"}) + + assert not (cache / "MQL5" / "Include" / "Doomed.mqh").exists(), \ + "a header deleted from source survived in the mirror" + + +def test_pruning_is_skipped_when_the_source_is_unreachable(monkeypatch, tmp_path): + """A transient mount failure must not wipe the mirror. + + The source walk yields nothing when the share is down; pruning against that + empty set would delete every mirrored header and turn a blip into an outage. + """ + src = tmp_path / "gone" # never created + dst = tmp_path / "mirror" + (dst / "Include").mkdir(parents=True) + (dst / "Include" / "Keep.mqh").write_text("// precious") + + copied, removed = compile_handler._mirror_tree(str(src), str(dst)) + + assert (copied, removed) == (0, 0) + assert (dst / "Include" / "Keep.mqh").exists(), "pruned the mirror against an empty source" + + +# ── include_files: WHICH header moved, not just that something did ─────────── + +def _digest_env(monkeypatch, tmp_path, patterns="Mine*.mqh", name="perfile"): + include = _hash_env(monkeypatch, tmp_path, name=name) + (include / "MineLicense.mqh").write_text("// licence v1") + (include / "Stock.mqh").write_text("// vendor library") + monkeypatch.setattr( + compile_handler, "_INCLUDE_DIGEST_PATTERNS", + tuple(p.strip() for p in patterns.split(",") if p.strip()), + ) + monkeypatch.setattr(compile_handler, "_INCLUDE_FILES", None) + return include + + +def test_include_files_reports_only_the_configured_headers( + client, compile_env, monkeypatch, tmp_path +): + _digest_env(monkeypatch, tmp_path) + body = _post(client, {"source": "void OnTick(){}"}).get_json() + assert set(body["include_files"]) == {"MineLicense.mqh"} + assert body["include_files"]["MineLicense.mqh"].startswith("sha256:") + + +def test_include_files_is_absent_when_nothing_is_configured( + client, compile_env, monkeypatch, tmp_path +): + """Default must stay off: ~260 digests per response is a payload, not an answer.""" + _digest_env(monkeypatch, tmp_path, patterns="", name="unconfigured") + body = _post(client, {"source": "void OnTick(){}"}).get_json() + assert "include_files" not in body + assert body["include_hash"].startswith("sha256:") + + +def test_a_stock_library_change_moves_the_tree_hash_but_not_our_header( + client, compile_env, monkeypatch, tmp_path +): + """The whole point of the field. + + A MetaTrader upgrade and a licence-header edit both move the tree hash. If + the per-file digest is unchanged, the caller knows the second did not + happen and can skip rebuilding every dependent artifact. + """ + include = _digest_env(monkeypatch, tmp_path, name="stockmove") + first = _post(client, {"source": "void OnTick(){}"}).get_json() + + (include / "Stock.mqh").write_text("// vendor library, upgraded and longer") + os.utime(include / "Stock.mqh", (time.time() + 5, time.time() + 5)) + second = _post(client, {"source": "void OnTick(){}"}).get_json() + + assert second["include_hash"] != first["include_hash"], "tree hash missed a change" + assert second["include_files"] == first["include_files"], \ + "a stock-library change moved our header's digest" + + +def test_editing_our_header_moves_its_own_digest(client, compile_env, monkeypatch, tmp_path): + include = _digest_env(monkeypatch, tmp_path, name="ourmove") + first = _post(client, {"source": "void OnTick(){}"}).get_json() + + (include / "MineLicense.mqh").write_text("// licence v2, materially different") + os.utime(include / "MineLicense.mqh", (time.time() + 5, time.time() + 5)) + second = _post(client, {"source": "void OnTick(){}"}).get_json() + + assert second["include_files"]["MineLicense.mqh"] != first["include_files"]["MineLicense.mqh"] + + +def test_a_configured_header_missing_from_the_tree_is_absent_not_null( + client, compile_env, monkeypatch, tmp_path +): + """Absent means "not in the tree the compiler read" - the thing worth + knowing before a rebuild. A null would blur that with "unreadable".""" + _digest_env(monkeypatch, tmp_path, patterns="Mine*.mqh,NeverExisted.mqh", name="absent") + body = _post(client, {"source": "void OnTick(){}"}).get_json() + assert "NeverExisted.mqh" not in body["include_files"] + assert body["include_files"]["MineLicense.mqh"].startswith("sha256:") From cd2755a7c321df149731e56b27aaa0593ae05a51 Mon Sep 17 00:00:00 2001 From: Marinski Date: Thu, 27 Aug 2026 09:13:57 +0300 Subject: [PATCH 2/2] fix(compile): bound source and artifact size; stop echoing exception detail to callers From review: /compile accepted an unbounded body, wrote the source to disk, read the whole .ex5 back into memory, and base64-inflated it into the response - so one authenticated request could consume unbounded disk, memory, worker time and bandwidth. And the catch-all handler echoed the exception class and message to the caller, which for an OSError is an internal path. Two documented per-request caps, both enforced before the resource they bound is spent: - COMPILE_MAX_SOURCE_BYTES (default 2 MB): an oversized body is refused with 413 straight from its declared Content-Length, before parsing; the decoded source is then checked against the cap itself, before anything reaches disk. - COMPILE_MAX_EX5_BYTES (default 16 MB): the artifact is size-checked on disk, before it would be read or encoded. A refusal carries no binary at all, and is a 500, not a 422 - the caller's source compiled fine; the server is declining to return the result. The log names the knob. Both settings clamp rather than raise on bad values, matching the other numeric settings in config.py - it is imported by the whole API, so a typo in an optional endpoint's tuning must not stop trading. Unexpected errors now return a bare 'internal error'; the traceback goes to the server log only. The two remaining detail leaks on the 500 path (MetaEditor's absolute path, the OSError from launching it) are genericized the same way. Six new tests: the 413 fires before the compiler ever runs and before the work dir exists, the declared-length refusal happens before parsing (proven with a non-JSON payload - a 400 would mean the parser read it), the artifact refusal carries no ex5_base64, within-cap requests are unaffected both ways, and the 500 body contains neither the exception class nor its message nor a path. The existing kaboom test permitted the leak by asserting only that log was a string; the new one closes that hole. All six fail against the previous handler. Also evicted scripts/prune-terminal-logs.sh from this branch: the squash had swept it in from unrelated local work. It is PR #18's file (byte- identical to that branch's copy, referenced by nothing here), and #18's own review round has since fixed a selection bug in it - keeping a stale copy in this PR would both collide with #18 on merge and reintroduce the bug that fix removes. --- CHANGELOG.md | 27 +++++--- docs/compiling.md | 21 ++++++- mt5api/config.py | 29 +++++++++ mt5api/handlers/compile.py | 83 ++++++++++++++++++++++--- scripts/prune-terminal-logs.sh | 110 --------------------------------- tests/test_compile.py | 105 +++++++++++++++++++++++++++++++ 6 files changed, 249 insertions(+), 126 deletions(-) delete mode 100755 scripts/prune-terminal-logs.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b4b8d6..efbf9c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,13 +62,26 @@ The project follows [Semantic Versioning](https://semver.org/): patch = bug fixe via `COMPILE_TERMINAL_DIR`, `COMPILE_INCLUDE_DIR`, `COMPILE_WORK_DIR` and `COMPILE_TIMEOUT` (default `30s`, hard ceiling `60s`). -- **30 tests** (`tests/test_compile.py`) covering the response contract and the - MetaEditor quirks above: real UTF-16LE log bytes decoded and parsed, warnings - not failing a build, `ok: true` never returned without a binary, temp - directories removed after success, failure and timeout, `/log:` and `/inc:` - unreachable from the request body, path traversal in `filename` neutralised, - and the compile token refused on `/account`, `/positions`, `/orders`, - `POST /orders`, `/terminal` and `/terminal/restart`. +- **Per-request size caps, and a generic 500.** `COMPILE_MAX_SOURCE_BYTES` + (default 2 MB) rejects an oversized request with 413 from its declared + `Content-Length` — before parsing, and before anything reaches disk. + `COMPILE_MAX_EX5_BYTES` (default 16 MB) refuses to return a larger compiled + binary, checked on disk before it would be read into memory and + base64-inflated into the response. Without these, one authenticated request + could consume unbounded disk, memory and response bandwidth. Unexpected + server errors now return a bare `"internal error"`; the exception class and + message — which for an `OSError` is an internal path — stay in the server + log. + +- **Contract tests** (`tests/test_compile.py`) covering the response contract + and the MetaEditor quirks above: real UTF-16LE log bytes decoded and parsed, + warnings not failing a build, `ok: true` never returned without a binary, + temp directories removed after success, failure and timeout, `/log:` and + `/inc:` unreachable from the request body, path traversal in `filename` + neutralised, the compile token refused on `/account`, `/positions`, + `/orders`, `POST /orders`, `/terminal` and `/terminal/restart`, both size + caps enforced before the write / before the read, and the 500 path leaking + neither exception class nor message nor paths. ## [v4.12.2] — 2026-08-20 diff --git a/docs/compiling.md b/docs/compiling.md index 0ac9dda..67dac94 100644 --- a/docs/compiling.md +++ b/docs/compiling.md @@ -38,7 +38,7 @@ curl -sS -X POST -H "Authorization: Bearer $MT5_API_TOKEN" \ | Field | Type | Required | Notes | | --- | --- | --- | --- | -| `source` | string | yes | The complete `.mq5` text. | +| `source` | string | yes | The complete `.mq5` text. Capped at `COMPILE_MAX_SOURCE_BYTES` (default 2 MB). | | `filename` | string | no | Cosmetic. Reduced to a bare stem — see [Security](#security). Defaults to `ea.mq5`. | | `ea_version` | string | no | Recorded in the server log to correlate a compile with your build. Not passed to the compiler. | @@ -119,9 +119,24 @@ compiler lock. {"ok": false, "log": "compile timeout after 30s"} ``` +**413 — the request is too large.** Sent when the declared body exceeds the +request bound, or the decoded `source` exceeds `COMPILE_MAX_SOURCE_BYTES` — +in both cases before anything reaches disk. The log names the limit: + +```json +{"ok": false, "log": "source is 3145728 bytes; this server accepts at most 2097152 (COMPILE_MAX_SOURCE_BYTES)"} +``` + **400 — malformed request** (no `source`, or not a string). **401 — bad or missing credentials.** -**500 — the host cannot compile** (MetaEditor missing, unreadable, unlaunchable). +**500 — the host cannot compile** (MetaEditor missing, unreadable, +unlaunchable), an unexpected server error (reported as a generic +`"internal error"` — details go to the server log, not the response), or a +compiled binary over `COMPILE_MAX_EX5_BYTES`. That last one means the source +compiled but the server refuses to return an artifact that size — it is +checked on disk, before the binary would be read into memory and +base64-inflated into the response. Raise the cap if your EA legitimately +embeds resources that big. Warnings never fail a compile. MetaEditor exits non-zero on warnings as well as errors, so the exit code is not used to decide the outcome — the log is parsed @@ -148,6 +163,8 @@ Environment overrides, all optional: | `COMPILE_LOCAL_CACHE` | unset | Mirror the toolchain onto local disk — see [Performance](#performance). | | `COMPILE_INCLUDE_DIGESTS` | unset | Comma-separated globs (relative to the include root) whose per-file digests are reported as `include_files`. | | `COMPILE_TIMEOUT` | `30s` | Per-compile deadline. Hard ceiling 60s. | +| `COMPILE_MAX_SOURCE_BYTES` | `2097152` (2 MB) | Reject a larger `source` with 413, before it is written to disk. | +| `COMPILE_MAX_EX5_BYTES` | `16777216` (16 MB) | Refuse to return a larger compiled binary, before it is read or encoded. | ### Which terminal compiles diff --git a/mt5api/config.py b/mt5api/config.py index cfdf921..156d51f 100644 --- a/mt5api/config.py +++ b/mt5api/config.py @@ -301,6 +301,35 @@ def _compile_setting(env_name, yaml_name, default=""): parse_duration_to_seconds(_compile_setting("COMPILE_TIMEOUT", "compile_timeout", "30s")) or 30, ), ) + + +def _compile_byte_limit(env_name, yaml_key, default): + """A positive per-request byte cap, clamped rather than raised. + + Same call as the other numeric settings in this module: config.py is + imported by the whole API, so a typo in an optional endpoint's tuning + value must not stop trading and backtesting. The floor keeps a bad value + from silently disabling /compile outright. + """ + raw = _compile_setting(env_name, yaml_key, str(default)) + try: + value = int(str(raw).strip()) + except (TypeError, ValueError): + return default + return max(1024, value) + + +# Per-request caps for /compile, so one authenticated request cannot consume +# unbounded disk (the source is written to a temp dir), memory (the .ex5 is +# read back whole), or response bandwidth (it is base64-encoded into the JSON +# body). Checked before the write and before the read respectively - see +# handlers/compile.py. Documented in docs/compiling.md. +COMPILE_MAX_SOURCE_BYTES = _compile_byte_limit( + "COMPILE_MAX_SOURCE_BYTES", "compile_max_source_bytes", 2 * 1024 * 1024 +) +COMPILE_MAX_EX5_BYTES = _compile_byte_limit( + "COMPILE_MAX_EX5_BYTES", "compile_max_ex5_bytes", 16 * 1024 * 1024 +) UTC_OFFSET_RAW = _args.utc_offset if _args.utc_offset is not None else os.environ.get("UTC_OFFSET", "") UTC_OFFSET_SECONDS = parse_duration_to_seconds(UTC_OFFSET_RAW) UTC_OFFSET_HOURS = UTC_OFFSET_SECONDS / 3600.0 diff --git a/mt5api/handlers/compile.py b/mt5api/handlers/compile.py index f0256e4..f7473d8 100644 --- a/mt5api/handlers/compile.py +++ b/mt5api/handlers/compile.py @@ -46,6 +46,8 @@ COMPILE_INCLUDE_DIGESTS, COMPILE_INCLUDE_DIR, COMPILE_LOCAL_CACHE, + COMPILE_MAX_EX5_BYTES, + COMPILE_MAX_SOURCE_BYTES, COMPILE_METAEDITOR, COMPILE_TIMEOUT_SECONDS, COMPILE_WORK_DIR, @@ -601,15 +603,36 @@ def compile_source(): started = time.monotonic() try: return _compile_source_inner(started) - except Exception as exc: # noqa: BLE001 - the handler must never raise + except Exception: # noqa: BLE001 - the handler must never raise + # The full traceback goes to the server log and ONLY there. Echoing + # the exception class and message gave any caller who could provoke a + # compiler or filesystem error a readout of internal paths and + # implementation detail. log.exception("compile: unhandled error") - return _json( - {"ok": False, "log": f"internal error: {exc.__class__.__name__}: {exc}"}, - 500, - ) + return _json({"ok": False, "log": "internal error"}, 500) + + +def _body_byte_cap(): + """Upper bound on the raw request body, derived from the source cap. + + JSON escaping inflates the encoded string (worst case 6x for \\uXXXX + escapes); 4x plus envelope slack admits every realistic encoding of a + source that is itself within the cap. The authoritative check is on the + DECODED source below - this one exists so an oversized body is refused + from its declared Content-Length, before any of it is parsed. + """ + return 4 * COMPILE_MAX_SOURCE_BYTES + 4096 def _compile_source_inner(started): + declared = request.content_length + if declared is not None and declared > _body_byte_cap(): + log.warning("compile: rejected %d-byte body (cap %d)", declared, _body_byte_cap()) + return _json( + {"ok": False, "log": f"request body exceeds {_body_byte_cap()} bytes"}, + 413, + ) + body = request.get_json(silent=True) if not isinstance(body, dict): return _json({"ok": False, "log": "body must be a JSON object"}, 400) @@ -618,13 +641,33 @@ def _compile_source_inner(started): if not isinstance(source, str) or not source.strip(): return _json({"ok": False, "log": "'source' must be a non-empty string"}, 400) + # Enforced BEFORE anything touches disk: past this line the source is + # written into the work dir, so this check is what bounds that write. + source_bytes = len(source.encode("utf-8")) + if source_bytes > COMPILE_MAX_SOURCE_BYTES: + log.warning( + "compile: rejected %d-byte source (cap %d)", + source_bytes, COMPILE_MAX_SOURCE_BYTES, + ) + return _json( + { + "ok": False, + "log": ( + f"source is {source_bytes} bytes; this server accepts at " + f"most {COMPILE_MAX_SOURCE_BYTES} (COMPILE_MAX_SOURCE_BYTES)" + ), + }, + 413, + ) + stem = _safe_stem(body.get("filename") or "ea.mq5") ea_version = body.get("ea_version") if not os.path.exists(COMPILE_METAEDITOR): log.error("compile: MetaEditor missing at %s", COMPILE_METAEDITOR) + # The path is for the operator's log, not the caller's response. return _json( - {"ok": False, "log": f"MetaEditor not found at {COMPILE_METAEDITOR}"}, + {"ok": False, "log": "MetaEditor is not available on this host"}, 500, ) @@ -692,7 +735,8 @@ def _run_compile(stem, source, ea_version, started): timed_out = True except OSError as exc: log.error("compile: could not launch MetaEditor: %s", exc) - return _json({"ok": False, "log": f"could not launch MetaEditor: {exc}"}, 500) + # exc carries the executable's path; that stays in the server log. + return _json({"ok": False, "log": "could not launch MetaEditor"}, 500) if timed_out: log.warning("compile timeout after %ss stem=%s", COMPILE_TIMEOUT_SECONDS, stem) @@ -706,6 +750,31 @@ def _run_compile(stem, source, ea_version, started): ex5_bytes = b"" if os.path.exists(ex5_path): + # Size-checked on disk BEFORE the read: an artifact over the cap + # must not transit memory or get base64-inflated into the + # response at all. This is a server policy refusal, not a compile + # failure - the caller's source built fine - so it does not take + # the 422 path, and the log names the knob to raise. + try: + ex5_size = os.path.getsize(ex5_path) + except OSError: + ex5_size = 0 + if ex5_size > COMPILE_MAX_EX5_BYTES: + log.error( + "compile: refusing %d-byte .ex5 (cap %d) stem=%s", + ex5_size, COMPILE_MAX_EX5_BYTES, stem, + ) + return _json( + { + "ok": False, + "log": ( + f"compiled binary is {ex5_size} bytes; this server " + f"returns at most {COMPILE_MAX_EX5_BYTES} " + f"(COMPILE_MAX_EX5_BYTES)" + ), + }, + 500, + ) try: with open(ex5_path, "rb") as handle: ex5_bytes = handle.read() diff --git a/scripts/prune-terminal-logs.sh b/scripts/prune-terminal-logs.sh deleted file mode 100755 index 733ac4c..0000000 --- a/scripts/prune-terminal-logs.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/sh -# Retention for MT5 terminal and tester-agent logs. -# -# These are not the API's own logs (rotate-logs.sh handles those). They are -# written by terminal64.exe and metatester64.exe inside each terminal -# directory, in UTF-16, and nothing prunes them: MT5 has no retention setting -# and the log-rotator sidecar only ever saw data/shared/logs. -# -# Two controls, because age alone is not enough. A high-frequency strategy -# logs every order placement, cancellation and modification, so one backtest -# can produce tens of gigabytes in a single day — a 7-day window would let a -# disk fill long before the first prune fired. Age bounds slow creep; the size -# cap bounds the actual hazard. -# -# RETAIN_DAYS delete *.log older than this -# MAX_LOG_BYTES truncate any single *.log larger than this -# IDLE_MINUTES never touch a file written more recently than this, so a -# running backtest never loses its own diagnostics -# -# Truncate-in-place (: >) rather than delete for the size cap: the terminal -# holds these files open, so removing the inode would leave the writer pointed -# at a deleted file and the space would not come back until it exited. - -set -eu - -TERMINAL_ROOTS="${TERMINAL_ROOTS:-/terminals}" -RETAIN_DAYS="${RETAIN_DAYS:-7}" -MAX_LOG_BYTES="${MAX_LOG_BYTES:-2147483648}" -IDLE_MINUTES="${IDLE_MINUTES:-30}" -INTERVAL="${INTERVAL:-3600}" -DRY_RUN="${DRY_RUN:-0}" - -log() { - printf '[%s] [logprune] %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" -} - -human() { - awk -v b="$1" 'BEGIN { - split("B KB MB GB TB", u, " "); i = 1 - while (b >= 1024 && i < 5) { b /= 1024; i++ } - printf "%.1f%s", b, u[i] - }' -} - -# Every terminal log directory: /logs, /Tester/logs and -# /Tester/Agent-*/logs. Matching on the path keeps us off the rest -# of the terminal tree (Bases, MQL5, Reports) entirely. -find_logs() { - root="$1" - shift - [ -d "$root" ] || return 0 - find "$root" \ - \( -path '*/Tester/logs/*.log' -o -path '*/Tester/Agent-*/logs/*.log' \) \ - -type f "$@" 2>/dev/null || true -} - -prune_once() { - for root in $(echo "$TERMINAL_ROOTS" | tr ':' ' '); do - [ -d "$root" ] || { - log "root missing, skipping: $root" - continue - } - - # 1. Age: delete whole files past the retention window. - find_logs "$root" -mtime "+${RETAIN_DAYS}" | while read -r f; do - size=$(wc -c <"$f" 2>/dev/null || echo 0) - if [ "$DRY_RUN" = "1" ]; then - log "would delete (age) $(human "$size") $f" - elif rm -f "$f"; then - log "deleted (age) $(human "$size") $f" - fi - done - - # 2. Size: truncate oversized files that nothing is actively writing. - find_logs "$root" -size "+$((MAX_LOG_BYTES / 1024))k" | while read -r f; do - size=$(wc -c <"$f" 2>/dev/null || echo 0) - [ "$size" -gt "$MAX_LOG_BYTES" ] || continue - - # Skip anything touched inside the idle window — that is a live run. - if [ -n "$(find "$f" -mmin "-${IDLE_MINUTES}" 2>/dev/null)" ]; then - log "skipped (active, $(human "$size")) $f" - continue - fi - - if [ "$DRY_RUN" = "1" ]; then - log "would truncate $(human "$size") $f" - elif : >"$f"; then - log "truncated $(human "$size") $f" - fi - done - done - - # Per-file actions are logged above; each `find | while read` runs in its - # own subshell, so totals cannot be accumulated here without a temp file. - log "pass complete" -} - -log "starting (roots=$TERMINAL_ROOTS retain_days=$RETAIN_DAYS max_log=$(human "$MAX_LOG_BYTES") idle_min=$IDLE_MINUTES interval=${INTERVAL}s dry_run=$DRY_RUN)" - -if [ "${RUN_ONCE:-0}" = "1" ]; then - prune_once - exit 0 -fi - -while true; do - if ! prune_once; then - log "prune_once failed (continuing)" - fi - sleep "$INTERVAL" -done diff --git a/tests/test_compile.py b/tests/test_compile.py index 621c1c8..6f131a4 100644 --- a/tests/test_compile.py +++ b/tests/test_compile.py @@ -273,6 +273,111 @@ def _boom(cmd, **kwargs): assert isinstance(resp.get_json()["log"], str) +def test_an_unexpected_error_does_not_leak_its_class_message_or_paths( + client, compile_env, monkeypatch +): + """The 500 body must be generic. The exception's class and message used to + be echoed to the caller, which turned any provocable compiler or + filesystem error into a readout of internal paths - the message of an + OSError IS a path. Detail belongs in the server log only.""" + secret = "C:\\internal\\deploy\\path\\MetaEditor64.exe" + + def _boom(cmd, **kwargs): + raise RuntimeError(f"kaboom at {secret}") + monkeypatch.setattr(compile_handler.subprocess, "run", _boom) + resp = _post(client, {"source": "void OnTick(){}"}) + assert resp.status_code == 500 + text = json.dumps(resp.get_json()) + assert "kaboom" not in text + assert "RuntimeError" not in text + assert secret.replace("\\", "\\\\") not in text and "MetaEditor64.exe" not in text + + +# ── Size limits ────────────────────────────────────────────────────────────── + +def test_an_oversized_source_is_rejected_before_anything_is_written( + client, compile_env, monkeypatch +): + """The source cap bounds the disk write, so it must fire before the write: + a 413 that arrives after the temp dir was populated bounds nothing.""" + monkeypatch.setattr(compile_handler, "COMPILE_MAX_SOURCE_BYTES", 1024) + calls = [] + monkeypatch.setattr( + compile_handler.subprocess, "run", + lambda *a, **k: calls.append(a) or FakeCompleted(0), + ) + resp = _post(client, {"source": "x" * 2048}) + assert resp.status_code == 413 + body = resp.get_json() + assert body["ok"] is False + assert "1024" in body["log"], "the refusal must name the limit" + assert calls == [], "the compiler must never see an oversized source" + assert not compile_env["work"].exists(), "nothing may reach the work dir" + + +def test_an_oversized_body_is_refused_from_its_declared_length_before_parsing( + client, compile_env, monkeypatch +): + """A body over the cap is 413 straight from Content-Length. The payload + here is not even JSON: a 400 would prove the parser read it, a 413 proves + it was refused unread.""" + monkeypatch.setattr(compile_handler, "COMPILE_MAX_SOURCE_BYTES", 1024) + raw = b"x" * (4 * 1024 + 5000) # over 4*cap + 4096 envelope slack + resp = client.post( + "/compile", + data=raw, + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {COMPILE_TOKEN}", + }, + ) + assert resp.status_code == 413 + assert isinstance(resp.get_json()["log"], str) + + +def test_a_source_within_the_cap_still_compiles(client, compile_env, monkeypatch): + monkeypatch.setattr(compile_handler, "COMPILE_MAX_SOURCE_BYTES", 1024) + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), ex5_bytes=b"EX5"), + ) + resp = _post(client, {"source": "void OnTick(){}"}) + assert resp.status_code == 200 + assert resp.get_json()["ok"] is True + + +def test_an_oversized_artifact_is_refused_before_it_is_encoded( + client, compile_env, monkeypatch +): + """The output cap bounds memory and response size, so it is checked on + disk: the refusal must carry no ex5_base64 at all, not a truncated one. + And it is not a 422 - the caller's source compiled fine; the server is + declining to return the result.""" + monkeypatch.setattr(compile_handler, "COMPILE_MAX_EX5_BYTES", 1024) + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), ex5_bytes=b"B" * 2048), + ) + resp = _post(client, {"source": "void OnTick(){}"}) + assert resp.status_code == 500 + body = resp.get_json() + assert body["ok"] is False + assert "ex5_base64" not in body + assert "1024" in body["log"], "the refusal must name the limit" + + +def test_an_artifact_within_the_cap_is_returned_whole(client, compile_env, monkeypatch): + payload = b"B" * 512 + monkeypatch.setattr(compile_handler, "COMPILE_MAX_EX5_BYTES", 1024) + monkeypatch.setattr( + compile_handler.subprocess, "run", + _fake_metaeditor(_utf16_log(SUCCESS_LOG), ex5_bytes=payload), + ) + resp = _post(client, {"source": "void OnTick(){}"}) + assert resp.status_code == 200 + assert base64.b64decode(resp.get_json()["ex5_base64"]) == payload + + # ── Temp directory hygiene ─────────────────────────────────────────────────── @pytest.mark.parametrize("log_text,ex5,expect_status", [