feat(telemetry): halve JSONL storage, cap SSH sample rate, restructure OTLP export (#1003)

Cuts local telemetry disk usage roughly in half, caps the noisiest
emitter, and aligns the OTLP export with OTLP standards.

- JSONL files open with a once-per-file header carrying the 8
  session-constant context fields; rows keep only per-event fields plus
  `deployment_url`. Headers are rewritten on rotation so every rolling
  file stays independently readable. Saves ~45% of file bytes.
- `http.requests` emits `count.*` keys only when nonzero (~8% of bytes).
- `ssh.network.sampled` changes are gated behind a 15s cooldown.
- OTLP output is restructured into per-session resource blocks: session
  context becomes the OTel resource, data points group under one metric
  per (name, unit), and cumulative counters reset per block so a new
  session does not inherit prior totals.
- A telemetry file that cannot be read or parsed no longer aborts the
  export: the stream skips it from its first bad line (keeping events
  already parsed), each skip is logged with file:line detail, the
  completion notification names the skip count, and the diagnostic span
  gains a `file.skipped_count` measurement.

Author: EhabY

src/core/container.ts

@@ -53,18 +53,17 @@ export class ServiceContainer implements vscode.Disposable {
 			this.logger,
 		);
 
-		const sessionId = newSessionId();
+		const session = buildSession(
+			extractExtensionVersion(context.extension.packageJSON),
+			newSessionId(),
+		);
 		const localJsonlSink = LocalJsonlSink.start(
 			{
 				baseDir: this.pathResolver.getTelemetryPath(),
-				sessionId,
+				session,
 			},
 			this.logger,
 		);
-		const session = buildSession(
-			extractExtensionVersion(context.extension.packageJSON),
-			sessionId,
-		);
 		this.telemetryService = new TelemetryService(
 			session,
 			[localJsonlSink],

src/instrumentation/CONVENTIONS.md

@@ -115,6 +115,8 @@ Coder's own pipeline, so a bare `cache_source` can't collide with a future OTel
 - **Measurements** are raw numbers. Don't pre-bucket into string labels: both
   export as record attributes, and a query can bucket the raw number at read
   time. `result` and `durationMs` are framework-managed and cannot be set.
+  `durationMs` never reaches OTLP; the export derives span start/end times
+  from it.
 - **Units.** There is no unit field at emit time, so put the unit in the
   measurement key as a `_ms` / `_seconds` / `_mbits` suffix, the same way for
   every event.

src/instrumentation/EVENTS.md

@@ -34,46 +34,73 @@ Signal kinds, which each category groups its events by:
 
 Framework-managed envelope fields (wire keys, JSONL):
 
-| Field             | Meaning                                                                                     |
-| ----------------- | ------------------------------------------------------------------------------------------- |
-| `event_id`        | Unique per event (OTel span id, 16 hex)                                                     |
-| `event_name`      | The event names below                                                                       |
-| `timestamp`       | ISO 8601 emission time                                                                      |
-| `event_sequence`  | Monotonic per-session counter                                                               |
-| `schema_version`  | Integer, currently `1`; bumped only on breaking wire changes, additive fields never bump it |
-| `trace_id`        | Spans and their child events only (OTel trace id, 32 hex)                                   |
-| `parent_event_id` | Phases and span logs only; the parent span's `event_id`                                     |
-| `error`           | `{ message, type?, code? }`, only when an error was captured                                |
-
-Session context, stamped on every event under `context`:
-
-| Field                                  | Source                                                                    |
-| -------------------------------------- | ------------------------------------------------------------------------- |
-| `extension_version`                    | package.json version                                                      |
-| `machine_id`                           | `vscode.env.machineId`                                                    |
-| `session_id`                           | Generated per session                                                     |
-| `os_type` / `os_version` / `host_arch` | `process.platform` (windows normalized) / `os.release()` / `process.arch` |
-| `platform_name` / `platform_version`   | `vscode.env.appName` / `vscode.version`                                   |
-| `deployment_url`                       | Set once known via `setDeploymentUrl`                                     |
+| Field             | Meaning                                                                |
+| ----------------- | ---------------------------------------------------------------------- |
+| `event_id`        | Unique per event (OTel span id, 16 hex)                                |
+| `event_name`      | The event names below                                                  |
+| `timestamp`       | ISO 8601 emission time                                                 |
+| `event_sequence`  | Monotonic per-session counter                                          |
+| `deployment_url`  | Deployment active at emit time; empty until set via `setDeploymentUrl` |
+| `trace_id`        | Spans and their child events only (OTel trace id, 32 hex)              |
+| `parent_event_id` | Phases and span logs only; the parent span's `event_id`                |
+| `error`           | `{ message, type?, code? }`, only when an error was captured           |
+
+Session-constant context is written once per file instead of on every row:
+the first line of every telemetry file (including rotated `.N` segments) is
+a header carrying the wire schema version, the sink start time, and the
+session context; each row below it implicitly inherits the version and
+context.
+
+```json
+{
+	"kind": "header",
+	"schema_version": 1,
+	"timestamp": "...",
+	"context": {
+		"extension_version": "...",
+		"machine_id": "...",
+		"session_id": "...",
+		"os_type": "...",
+		"os_version": "...",
+		"host_arch": "...",
+		"platform_name": "...",
+		"platform_version": "..."
+	}
+}
+```
+
+| Field                                  | Source                                                                                      |
+| -------------------------------------- | ------------------------------------------------------------------------------------------- |
+| `schema_version`                       | Integer, currently `1`; bumped only on breaking wire changes, additive fields never bump it |
+| `timestamp`                            | ISO 8601 sink start time; at or before every row's timestamp                                |
+| `extension_version`                    | package.json version                                                                        |
+| `machine_id`                           | `vscode.env.machineId`                                                                      |
+| `session_id`                           | Generated per session                                                                       |
+| `os_type` / `os_version` / `host_arch` | `process.platform` (windows normalized) / `os.release()` / `process.arch`                   |
+| `platform_name` / `platform_version`   | `vscode.env.appName` / `vscode.version`                                                     |
 
 On OTLP export the context becomes resource attributes (`service.name:
 coder-vscode-extension`, `service.version`, `service.instance.id`, `host.id`,
 `host.arch`, `os.type`, `os.version`, `vscode.platform.name`,
-`vscode.platform.version`, `coder.deployment.url`) plus per-record provenance
-(`coder.event.extension_version`, `coder.event.session_id`,
-`coder.event.deployment_url`).
+`vscode.platform.version`, `coder.deployment.url`) on the resource block
+holding the producing session's records.
 
 ## Consuming exports
 
 Events buffer on disk as JSONL; nothing leaves the machine on its own. The
 **Coder: Export Telemetry** command flushes the buffer and writes a chosen
 date range in one of two formats:
 
-- **JSON**: one file holding an array of the wire-format rows described
-  above, for direct inspection or ad-hoc processing.
+- **JSON**: one file holding an array of self-contained events, each
+  carrying its full context and schema version, for direct inspection or
+  ad-hoc processing.
 - **OTLP**: a zip of standard OTLP/JSON envelopes (spans in `traces.json`,
   logs in `logs.json`, metric events as data points in `metrics.json`) plus
-  a `manifest.json` describing the export. Feed these to any OTel-compatible
+  a `manifest.json` describing the export. Each envelope holds one resource
+  block per producing session and UTC date, carrying that session's context
+  as resource attributes; within a block, metric data points are grouped
+  under one `metrics[]` entry per metric name and unit, and cumulative
+  counters restart at the block boundary. Feed these to any OTel-compatible
   tool that ingests OTLP/JSON, such as an OpenTelemetry Collector pipeline or
   your observability backend's import tooling.
 
@@ -271,6 +298,7 @@ Emitted by `DiagnosticTelemetry` around each diagnostic command.
 | `interval.count` (measurement)             | speed test only                                                  |
 | `throughput_mbits` (measurement)           | speed test only                                                  |
 | `event.count` (measurement)                | telemetry export only                                            |
+| `file.skipped_count` (measurement)         | telemetry export only; unreadable files skipped, omitted at zero |
 
 ## Deployment
 
@@ -358,9 +386,10 @@ Emitted by `SshTelemetry`.
 
 #### `ssh.network.sampled`
 
-Tunnel network sample. Emitted on a p2p flip, a preferred-DERP change, a
-meaningful latency change (at least 25 ms or 20 %), or a roughly 60 s
-heartbeat.
+Tunnel network sample. Emitted on a roughly 60 s heartbeat, or on a p2p
+flip, a preferred-DERP change, or a meaningful latency change (at least
+25 ms and at least 20 %). Change-triggered emissions are limited to one per
+15 s; a change that persists past that cooldown is emitted when it expires.
 
 | Attribute                      | Values                                                             |
 | ------------------------------ | ------------------------------------------------------------------ |
@@ -380,13 +409,13 @@ Emitted by `HttpRequestsTelemetry`, which lives with the HTTP logging in
 `src/logging`. A per-minute rollup of REST traffic, one event per method and
 route bucket.
 
-| Attribute                                                              | Values                                                              |
-| ---------------------------------------------------------------------- | ------------------------------------------------------------------- |
-| `method`                                                               | HTTP method                                                         |
-| `route`                                                                | normalized route (ids replaced by placeholders)                     |
-| `window_seconds` (measurement)                                         | actual window length                                                |
-| `count.1xx` through `count.5xx`, `count.network_error` (measurements)  | export as cumulative counters with unit `{request}`                 |
-| `duration.p50_ms`, `duration.p95_ms`, `duration.p99_ms` (measurements) | export as gauges (`http.requests.duration.p50` etc.) with unit `ms` |
+| Attribute                                                              | Values                                                                                                               |
+| ---------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- |
+| `method`                                                               | HTTP method                                                                                                          |
+| `route`                                                                | normalized route (ids replaced by placeholders)                                                                      |
+| `window_seconds` (measurement)                                         | actual window length                                                                                                 |
+| `count.1xx` through `count.5xx`, `count.network_error` (measurements)  | omitted when 0; export as cumulative counters with unit `{request}`                                                  |
+| `duration.p50_ms`, `duration.p95_ms`, `duration.p99_ms` (measurements) | omitted when no request carried timing metadata; export as gauges (`http.requests.duration.p50` etc.) with unit `ms` |
 
 ## WebSocket connections
 

src/instrumentation/diagnostics.ts

@@ -2,6 +2,7 @@ import { recordAborted, recordError } from "./outcomes";
 
 import type { SpeedtestResult } from "@repo/shared";
 
+import type { ExportResult } from "../telemetry/export/pipeline";
 import type { ExportFormat } from "../telemetry/export/writers/types";
 import type { TelemetryReporter } from "../telemetry/reporter";
 import type { Span } from "../telemetry/span";
@@ -29,7 +30,7 @@ export interface DiagnosticTrace {
 	error(category?: DiagnosticErrorCategory): void;
 	setRequestedDuration(seconds: number): void;
 	succeedSpeedtest(result: SpeedtestResult): void;
-	succeedExport(format: ExportFormat, eventCount: number): void;
+	succeedExport(format: ExportFormat, result: ExportResult): void;
 }
 
 /** Emits `command.diagnostic.completed` around each diagnostic command. */
@@ -71,8 +72,11 @@ class SpanDiagnosticTrace implements DiagnosticTrace {
 		);
 	}
 
-	public succeedExport(format: ExportFormat, eventCount: number): void {
+	public succeedExport(format: ExportFormat, result: ExportResult): void {
 		this.span.setProperty("format", format);
-		this.span.setMeasurement("event.count", eventCount);
+		this.span.setMeasurement("event.count", result.eventCount);
+		if (result.skippedFileCount > 0) {
+			this.span.setMeasurement("file.skipped_count", result.skippedFileCount);
+		}
 	}
 }

src/instrumentation/ssh.ts

@@ -2,6 +2,7 @@ import type { NetworkInfo } from "../remote/sshProcess";
 import type { TelemetryReporter } from "../telemetry/reporter";
 
 const NETWORK_SAMPLE_INTERVAL_MS = 60_000;
+const NETWORK_CHANGE_COOLDOWN_MS = 15_000;
 const NETWORK_LATENCY_CHANGE_RATIO = 0.2;
 const NETWORK_LATENCY_MIN_ABSOLUTE_CHANGE_MS = 25;
 
@@ -141,15 +142,22 @@ export class SshTelemetry {
 	}
 }
 
-/** Emit on p2p flip, DERP change, large latency swing, or heartbeat interval. */
+/** Emit on the heartbeat interval, or on a p2p flip, DERP change, or large
+ * latency swing once the change cooldown has elapsed. Suppression leaves the
+ * last emitted sample in place, so a change that persists through the
+ * cooldown is emitted when it expires rather than lost. */
 function shouldEmitSample(
 	previous: NetworkSample,
 	current: NetworkInfo,
 	now: number,
 ): boolean {
-	if (now - previous.emittedAtMs >= NETWORK_SAMPLE_INTERVAL_MS) {
+	const sinceLastEmit = now - previous.emittedAtMs;
+	if (sinceLastEmit >= NETWORK_SAMPLE_INTERVAL_MS) {
 		return true;
 	}
+	if (sinceLastEmit < NETWORK_CHANGE_COOLDOWN_MS) {
+		return false;
+	}
 	if (current.p2p !== previous.p2p) {
 		return true;
 	}
@@ -167,8 +175,9 @@ function hasMeaningfulLatencyChange(
 		return current !== 0;
 	}
 	const absoluteChange = Math.abs(current - previous);
+	// The absolute floor mutes fast-link jitter; the ratio mutes slow-link noise.
 	return (
-		absoluteChange >= NETWORK_LATENCY_MIN_ABSOLUTE_CHANGE_MS ||
+		absoluteChange >= NETWORK_LATENCY_MIN_ABSOLUTE_CHANGE_MS &&
 		absoluteChange / Math.abs(previous) >= NETWORK_LATENCY_CHANGE_RATIO
 	);
 }

src/logging/httpRequestsTelemetry.ts

@@ -142,23 +142,31 @@ export class HttpRequestsTelemetry implements Disposable {
 		);
 		for (const [method, byRoute] of this.#buckets) {
 			for (const [route, bucket] of byRoute) {
-				const sortedDurations = bucket.durationsMs.toSorted((a, b) => a - b);
-				this.#telemetry.log(
-					EVENT_NAME,
-					{ method, route },
-					{
-						window_seconds: elapsedSeconds,
-						"count.1xx": bucket.count1xx,
-						"count.2xx": bucket.count2xx,
-						"count.3xx": bucket.count3xx,
-						"count.4xx": bucket.count4xx,
-						"count.5xx": bucket.count5xx,
-						"count.network_error": bucket.countNetworkError,
-						"duration.p50_ms": percentile(sortedDurations, 0.5),
-						"duration.p95_ms": percentile(sortedDurations, 0.95),
-						"duration.p99_ms": percentile(sortedDurations, 0.99),
-					},
-				);
+				const counts: Record<string, number> = {
+					"count.1xx": bucket.count1xx,
+					"count.2xx": bucket.count2xx,
+					"count.3xx": bucket.count3xx,
+					"count.4xx": bucket.count4xx,
+					"count.5xx": bucket.count5xx,
+					"count.network_error": bucket.countNetworkError,
+				};
+				const measurements: Record<string, number> = {
+					window_seconds: elapsedSeconds,
+				};
+				// Zero counters are omitted; absence reads as "none in this window".
+				for (const [key, count] of Object.entries(counts)) {
+					if (count > 0) {
+						measurements[key] = count;
+					}
+				}
+				// Percentiles are omitted when no request carried timing metadata.
+				if (bucket.durationsMs.length > 0) {
+					const sorted = bucket.durationsMs.toSorted((a, b) => a - b);
+					measurements["duration.p50_ms"] = percentile(sorted, 0.5);
+					measurements["duration.p95_ms"] = percentile(sorted, 0.95);
+					measurements["duration.p99_ms"] = percentile(sorted, 0.99);
+				}
+				this.#telemetry.log(EVENT_NAME, { method, route }, measurements);
 			}
 		}
 		this.#buckets.clear();
@@ -189,6 +197,7 @@ function elapsedMs(
 }
 
 function percentile(sortedValues: readonly number[], p: number): number {
+	// Indexing an empty array would return undefined as a number.
 	if (sortedValues.length === 0) {
 		return 0;
 	}