fix(passthrough): drop retry; complete synthesized-header coverage

CHANGELOG.md

@@ -4,6 +4,67 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
+## [0.2.1] - 2026-08-19
+
+### Fixed
+
+- **Anthropic passthrough — `connectTimeoutMs` now genuinely bounds TCP connection
+  establishment; new `headerTimeoutMs` knob bounds time to first byte** (fixes #27).
+  Previously `upstream.setTimeout(connectTimeoutMs)` was called on the `ClientRequest`
+  object, but Node's implementation defers that call via an internal `'connect'`
+  listener — so it fired only after TCP connect, in the same tick as the
+  `headerTimeoutMs` rearm, meaning `connectTimeoutMs` was never in force for any
+  measurable interval.  Measured against a blackholed IP (`192.0.2.1`, TEST-NET-1)
+  with `connectTimeoutMs=700` and `headerTimeoutMs=5000`, the request previously
+  failed after **75 019 ms** (the macOS kernel TCP timeout) — neither budget fired.
+  The fix arms the timer **directly on the socket** inside the `'socket'` event
+  handler, before `'connect'` fires, so the connect-phase budget is in force for
+  the full TCP handshake window.  The same measured scenario now fails at ~700 ms.
+  On HTTPS, `'connect'` fires after TCP establishment but before the TLS handshake,
+  so TLS negotiation falls under `headerTimeoutMs`, not `connectTimeoutMs` — this is
+  documented in the JSDoc.  The fix also introduces a three-budget design:
+  `connectTimeoutMs` (10 s, TCP establishment only), `headerTimeoutMs` (660 s
+  default, connect→response-headers — defaults to 60 s above Anthropic's own ~600 s
+  server-side ceiling; the extra headroom accounts for the relay's clock starting at
+  TCP connect while the origin's starts only after the full request body is received,
+  so equal budgets would let the relay pre-empt the origin by the upload time plus
+  RTT), and `streamIdleTimeoutMs` (300 s, headers→stream-end, reset by every chunk).
+  **A hung upstream that accepts the TCP connection but never sends headers now takes
+  up to `headerTimeoutMs` (default 660 s) to fail, not 10 s** — tune this knob down
+  if you need faster detection of stalled upstreams.
+
+- **Anthropic passthrough — `x-subswitch-synthesized: 1` header marks every relay-generated response; stripped from proxied responses** (stacks on #30).
+  Previously a relay fault (502 connection failure, 504 timeout, 500 internal proxy error, 413 body too large,
+  503 concurrency gate) was indistinguishable from an upstream outage on the client side.  Every response the
+  relay synthesises itself now carries `x-subswitch-synthesized: 1`; this covers the Anthropic-leg error
+  paths, all codex-leg responses (both streaming SSE and aggregated JSON — the codex leg translates
+  Codex→Anthropic format so every byte it returns is relay-synthesised), and every relay-generated error in
+  `server.ts`.  On the proxied path the header is actively stripped from upstream responses via the hop-by-hop
+  filter so an origin that sets it cannot impersonate the relay.  Bodies and status codes are unchanged — the
+  header is purely additive.  See the new `## x-subswitch-synthesized` section in the README for the
+  operator-facing wire contract.
+
+- **Anthropic passthrough — client abort no longer logs a spurious `anthropic_upstream_error` warn** (stacks on #30).
+  `res.on("close")` was calling `upstream.destroy()` without first setting `settled = true`.  The destroy
+  emits `'error'` on the next tick; with `settled = false` the error handler logged `anthropic_upstream_error`
+  and attempted to write a 502 into an already-closed response.  A client hanging up is a normal event;
+  it now sets `settled = true` before the destroy so the error handler returns early.  Additionally, the
+  error handler now sets `settled = true` before writing the 502 so a theoretically possible second
+  `'error'` emission cannot double-write.
+
+- **Anthropic passthrough — duplicate `anthropic_upstream_error` warn eliminated**
+  (fixes #27).  When a pre-header timeout fired, the timeout handler called
+  `upstream.destroy()` before writing the 504.  Destroying an in-flight
+  `ClientRequest` causes it to emit `error` (ECONNRESET) on the next tick; the
+  error handler's `responded` guard was inert on the pre-header path, so both
+  `anthropic_upstream_timeout` and `anthropic_upstream_error` were logged.  On
+  `main` the actual response sequence was `writeHead(504)` → `res.end()`
+  (synchronous, in the timeout handler) → next tick, error handler sees
+  `headersSent === true` → `res.destroy()`; the client received a complete 504 body
+  and the real defect was the duplicate warn log only.  A new `settled` flag is now
+  set by whichever handler responds first; the error handler returns early when
+  `settled` is true, ensuring exactly one warn event per timeout.
+
 ## [0.2.0] - 2026-08-09
 
 ### Upgrading from 0.1.0

README.md

@@ -277,8 +277,9 @@ All keys and their defaults:
 | `port` | `4141` | Port the proxy listens on |
 | `logLevel` | `"info"` | Log verbosity: `debug`, `info`, `warn`, or `error` |
 | `anthropic.baseUrl` | `"https://api.anthropic.com"` | Anthropic passthrough base URL |
-| `anthropic.connectTimeoutMs` | `10000` (10 s) | **Anthropic leg only** — TCP connection timeout (see note below) |
-| `anthropic.streamIdleTimeoutMs` | `300000` (5 min) | Anthropic stream idle timeout |
+| `anthropic.connectTimeoutMs` | `10000` (10 s) | **Anthropic leg only** — TCP connection establishment timeout (see note below) |
+| `anthropic.headerTimeoutMs` | `660000` (11 min) | **Anthropic leg only** — time from TCP connect to first response byte; defaults to 60 s above Anthropic's own ~600 s server-side ceiling so the relay never fires before the origin does (the relay's clock starts earlier than the origin's — see note below) |
+| `anthropic.streamIdleTimeoutMs` | `300000` (5 min) | Anthropic stream idle timeout (headers→stream-end, reset by every chunk) |
 | `anthropic.maxUpstreamSockets` | `32` | **Anthropic leg only** — max sockets in the keep-alive pool (see note below) |
 | `anthropic.allowInsecureBaseUrl` | `false` | **Security opt-in** — when false (the default), `subswitch serve` refuses to start if `anthropic.baseUrl` points at a host other than `api.anthropic.com`. Set to `true` only when routing through a trusted proxy in front of Anthropic's API. Loopback addresses are always exempt. |
 | `providers.codex.baseUrl` | `"https://chatgpt.com/backend-api/codex"` | Codex backend base URL — override to route subswitch through the wire recorder |
@@ -297,11 +298,11 @@ All keys and their defaults:
 | `limits.pingIntervalMs` | `15000` (15 s) | Interval between SSE ping frames sent to clients during long Codex streams |
 | `limits.maxConcurrentRequests` | `32` | In-flight request ceiling; requests above this limit receive a 503 |
 
-> **Why `connectTimeoutMs` and `maxUpstreamSockets` are Anthropic-leg-only**: the
+> **Why `connectTimeoutMs`, `headerTimeoutMs`, and `maxUpstreamSockets` are Anthropic-leg-only**: the
 > Anthropic passthrough uses a node:http agent with an explicit keep-alive pool, so
-> both knobs have meaningful effect there. The Codex leg uses Node's global `fetch`
-> (undici's global dispatcher), which `maxUpstreamSockets` does not control — shipping
-> them as per-provider keys would be config that bounds nothing on the Codex side.
+> all three knobs have meaningful effect there. The Codex leg uses Node's global `fetch`
+> (undici's global dispatcher), which these knobs do not control — shipping them as
+> per-provider keys would be config that bounds nothing on the Codex side.
 
 ### `subswitch models --json`
 
@@ -394,6 +395,36 @@ is not a TTY (useful in terminals that misreport TTY state); `CI` — also suppr
 color and disables interactive `init` prompts (treated as a non-interactive
 environment).
 
+## `x-subswitch-synthesized`
+
+Every HTTP response that subswitch generates itself — rather than proxying
+verbatim from an upstream — carries the response header:
+
+```
+x-subswitch-synthesized: 1
+```
+
+This header is present on:
+
+- **Anthropic-leg relay errors**: 502 (upstream connection failure), 504
+  (upstream timeout), 413 (request body too large), 503 (concurrency gate),
+  500 (internal proxy error).
+- **Codex-leg responses**: every byte returned on the codex leg is synthesized
+  by the relay (it translates OpenAI Responses format → Anthropic Messages
+  format), so the header is present on both streaming and non-streaming codex
+  responses, and on all codex-leg error responses.
+- **Relay management endpoints**: `/__subswitch/health`, `/__subswitch/404`,
+  and all other relay-internal routes.
+
+The header is **absent** on responses proxied verbatim from the Anthropic
+origin — including upstream errors (429 rate-limit, 529 overloaded, 500
+upstream internal error, etc.).  The header is also **stripped** from any
+upstream response that carries it, so the marker is authoritative: its presence
+means the relay synthesised the response; its absence means the upstream did.
+
+Operators can use this header in load-balancer health rules, log filters, or
+alerting to distinguish relay faults from upstream outages.
+
 ## Testing
 
 ```sh

src/anthropic-passthrough.ts

@@ -12,6 +12,9 @@ import type { Logger } from "./logger.js";
  *
  * Per RFC 7230 §6.1. `host` is also excluded: Node sets it on the outbound
  * connection. `connection` is managed by the keep-alive agent.
+ *
+ * `x-subswitch-synthesized` is also stripped from proxied responses so that
+ * the marker is authoritative: only the relay itself can assert it.
  */
 const HOP_BY_HOP = new Set([
   "host",
@@ -24,6 +27,7 @@ const HOP_BY_HOP = new Set([
   "trailer",
   "transfer-encoding",
   "upgrade",
+  "x-subswitch-synthesized",
 ]);
 
 /**
@@ -53,7 +57,29 @@ const filterRawHeaders = (rawHeaders: readonly string[]): string[] => {
 
 export interface PassthroughOptions {
   readonly baseUrl: string;
+  /**
+   * Bounds TCP connection establishment only (milliseconds).
+   * The timer is armed directly on the socket (not via `ClientRequest.setTimeout`,
+   * which defers internally and cannot bound the connect phase).  Once TCP connects,
+   * the timer is re-armed to `headerTimeoutMs`.
+   *
+   * On HTTPS connections, `'connect'` fires after TCP establishment but before the
+   * TLS handshake, so TLS negotiation falls under `headerTimeoutMs`, not this budget.
+   *
+   * For pooled/keep-alive sockets (no connect phase), `headerTimeoutMs` is armed
+   * immediately and this budget has no effect.
+   */
   readonly connectTimeoutMs: number;
+  /**
+   * Bounds the connect→response-headers phase (time to first byte), in milliseconds.
+   * Armed on socket connect (or immediately for pooled sockets).
+   * Re-armed to `streamIdleTimeoutMs` once headers arrive.
+   */
+  readonly headerTimeoutMs: number;
+  /**
+   * Bounds the headers→stream-end phase, in milliseconds.
+   * Reset by every received chunk.
+   */
   readonly streamIdleTimeoutMs: number;
   readonly logger: Logger;
   /** Maximum sockets in the keep-alive pool. Config key: limits.maxUpstreamSockets. */
@@ -75,16 +101,17 @@ export const createAnthropicForwarder = (options: PassthroughOptions): Anthropic
 
   // Create a keep-alive agent for persistent connections to the upstream.
   // This matches Claude Code's own direct connection behaviour (parity).
-  //
-  // Residual risk: a stale pooled socket can ECONNRESET a POST; the existing
-  // upstream.on("error") handler returns a clean 502 and Claude Code retries.
-  // This is the same behaviour as any keep-alive HTTP client.
   const agentOpts: http.AgentOptions = { keepAlive: true, maxSockets: options.maxUpstreamSockets, scheduling: "lifo" };
   const agent = options.agent ?? (target.protocol === "https:" ? new https.Agent(agentOpts) : new http.Agent(agentOpts));
 
   return (req, res, body) => {
     const path = `${basePath}${req.url ?? "/"}`;
-    let responded = false;
+    // `settled` is set by whichever handler wins the race — the response
+    // callback (headers received) or the timeout handler (504 written).
+    // The error handler uses it as its early-return guard so that a
+    // destroy() issued by the timeout handler does not produce a duplicate
+    // `anthropic_upstream_error` warn after the 504 has already been sent.
+    let settled = false;
 
     const upstream = client.request(
       {
@@ -100,20 +127,62 @@ export const createAnthropicForwarder = (options: PassthroughOptions): Anthropic
         // (cross-name position of interleaved duplicates is not guaranteed).
       },
       (upstreamRes) => {
-        responded = true;
+        settled = true;
         upstream.setTimeout(options.streamIdleTimeoutMs);
         // Response direction: writeHead accepts a flat [name, value, ...] array
         // directly (Node's _storeHeader Array branch), preserving the upstream's
         // original header casing, order, and duplicates byte-for-byte.
+        // filterRawHeaders strips x-subswitch-synthesized so an origin that sets
+        // it cannot impersonate the relay's synthesized-response marker.
         res.writeHead(upstreamRes.statusCode ?? 502, filterRawHeaders(upstreamRes.rawHeaders));
         res.socket?.setNoDelay(true);
         upstreamRes.pipe(res);
         upstreamRes.on("error", () => res.destroy());
       },
     );
 
-    upstream.setTimeout(options.connectTimeoutMs);
-    upstream.on("socket", (socket) => socket.setNoDelay(true));
+    // Timer arming — three-budget design:
+    //
+    // 1. connectTimeoutMs — bounds TCP establishment, armed DIRECTLY on the
+    //    socket (not via upstream.setTimeout).  ClientRequest.setTimeout()
+    //    defers internally via its own 'connect' listener, so it would fire
+    //    only after connect — in the same tick as the headerTimeoutMs rearm,
+    //    leaving connectTimeoutMs never actually in force.
+    //
+    //    Node v22's internal socket-timeout handler (onTimeout) skips
+    //    req.emit('timeout') when socket.connecting is true, so we must
+    //    propagate the timeout manually via upstream.emit('timeout').
+    //    On HTTPS, 'connect' fires after TCP but BEFORE the TLS handshake, so
+    //    TLS negotiation falls under headerTimeoutMs, not connectTimeoutMs.
+    //
+    // 2. headerTimeoutMs — armed on 'connect' (or immediately for a pooled
+    //    socket where no 'connect' event will ever fire).  Bounds the window
+    //    from TCP-connected to first response byte.  upstream.setTimeout() in
+    //    the connected state propagates normally via Node's internal handler.
+    //
+    // 3. streamIdleTimeoutMs — armed in the response callback once headers
+    //    arrive; reset by every received chunk.
+    upstream.on("socket", (socket) => {
+      socket.setNoDelay(true);
+      if (socket.connecting) {
+        socket.setTimeout(options.connectTimeoutMs);
+        const onConnectTimeout = () => {
+          socket.removeListener("connect", onConnect);
+          socket.setTimeout(0); // disarm before manual propagation
+          upstream.emit("timeout"); // triggers our handler → 504
+        };
+        const onConnect = () => {
+          socket.removeListener("timeout", onConnectTimeout);
+          socket.setTimeout(0); // cancel connect budget
+          upstream.setTimeout(options.headerTimeoutMs);
+        };
+        socket.once("timeout", onConnectTimeout);
+        socket.once("connect", onConnect);
+      } else {
+        // Pooled/keep-alive socket — no connect phase; arm header budget immediately.
+        upstream.setTimeout(options.headerTimeoutMs);
+      }
+    });
 
     // Request direction: build a Map from the filtered rawHeaders so that
     // duplicates (same lowercase key, different values) are preserved as array
@@ -140,29 +209,52 @@ export const createAnthropicForwarder = (options: PassthroughOptions): Anthropic
     }
 
     upstream.on("timeout", () => {
-      upstream.destroy();
+      // Log and write the 504 BEFORE destroy() to narrow the race window with
+      // the 'error' handler.  destroy() on an in-flight ClientRequest emits
+      // 'error' (ECONNRESET) on the next tick; setting `settled = true` here
+      // prevents that error from producing a duplicate warn log.
       options.logger.log("warn", "anthropic_upstream_timeout", { path: req.url ?? "/" });
       if (!res.headersSent) {
-        res.writeHead(504, { "content-type": "application/json" });
+        settled = true;
+        res.writeHead(504, { "content-type": "application/json", "x-subswitch-synthesized": "1" });
         res.end(toAnthropicErrorBody("api_error", "upstream timed out"));
       } else {
         res.destroy();
       }
+      upstream.destroy();
     });
 
     upstream.on("error", () => {
-      if (responded) return;
+      if (settled) return;
+      // Log and surface as 502 only when the client response is still open.
+      // Set settled before writing so a second 'error' emission (practically
+      // unreachable after socket destroy, but possible in theory) cannot
+      // double-write.
       options.logger.log("warn", "anthropic_upstream_error", { path: req.url ?? "/" });
       if (!res.headersSent) {
-        res.writeHead(502, { "content-type": "application/json" });
+        settled = true;
+        res.writeHead(502, { "content-type": "application/json", "x-subswitch-synthesized": "1" });
         res.end(toAnthropicErrorBody("api_error", "upstream connection failed"));
       } else {
         res.destroy();
       }
     });
 
+    // Change 3: set settled before destroy() so the error event emitted by
+    // destroy() on the next tick does not trigger a spurious
+    // anthropic_upstream_error warn or attempt to write a 502 into an already-
+    // closed response.  A client abort is an entirely normal event — it must
+    // produce no warn log and no synthetic HTTP response.
+    //
+    // Note: the timeout handler does not check `settled` before calling
+    // upstream.destroy() because it sets `settled = true` itself before the
+    // destroy, so by the time the 'error' event fires on the next tick the
+    // guard has already been set and the error handler returns early.
     res.on("close", () => {
-      if (!res.writableFinished) upstream.destroy();
+      if (!res.writableFinished) {
+        settled = true;
+        upstream.destroy();
+      }
     });
 
     if (body !== undefined) {

src/codex-handler.ts

@@ -349,7 +349,7 @@ export const createCodexHandler = <P extends ProviderId>(deps: CodexHandlerDeps<
       resetIdle();
 
       if (wantStream) {
-        res.writeHead(200, { "content-type": "text/event-stream; charset=utf-8", "cache-control": "no-cache" });
+        res.writeHead(200, { "content-type": "text/event-stream; charset=utf-8", "cache-control": "no-cache", "x-subswitch-synthesized": "1" });
         res.socket?.setNoDelay(true);
         // res is written manually rather than placed inside pipeline(): on an
         // upstream error, pipeline destroys every stream it owns, which would