Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
<!-- fork:start (Cujuju/t3code). Upstream's guide follows unchanged below fork:end. -->

> **About this fork.** This is a fork of [pingdotgg/t3code](https://github.com/pingdotgg/t3code). Cujuju
> uses it to develop T3 Code's CLIProxyAPI usage-hub integration (hub-supplied account limits, hub account
> names, waking limited threads) and to send those changes upstream, such as pingdotgg/t3code#27901.
>
> - Changes that aren't specific to this fork belong upstream. Follow the guide below and open them at
> [pingdotgg/t3code](https://github.com/pingdotgg/t3code).
> - PRs here target `main`. The maintainer picks the merge method.
> - Upstream syncs come in as PRs that merge upstream `main`. They must be merged with a merge commit,
> never squashed or rebased, so later syncs stay clean.

<!-- fork:end -->

# Contributing

## Developer setup
Expand Down
7 changes: 3 additions & 4 deletions apps/server/src/observability/Observability.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,14 @@ import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as References from "effect/References";
import * as Tracer from "effect/Tracer";
import * as OtlpExporter from "effect/observability/OtlpExporter";
import * as OtlpMetrics from "effect/observability/OtlpMetrics";
import * as OtlpTracer from "effect/observability/OtlpTracer";
import * as Statement from "effect/sql/Statement";

import * as ServerConfig from "../config.ts";
import * as ResourceAttribution from "../resourceTelemetry/ResourceAttribution.ts";
import * as ServerLogger from "../serverLogger.ts";
import * as BrowserTraceCollector from "./BrowserTraceCollector.ts";
import * as ServerTraceExporter from "./ServerTraceExporter.ts";

export const layer = Layer.unwrap(
Effect.gen(function* () {
Expand Down Expand Up @@ -55,7 +54,7 @@ export const layer = Layer.unwrap(
const delegate =
config.otlpTracesUrl === undefined
? undefined
: yield* OtlpTracer.make({
: yield* ServerTraceExporter.make({
url: config.otlpTracesUrl,
exportInterval: `${traces.exportIntervalMs} millis`,
headers: traces.headers,
Expand All @@ -76,7 +75,7 @@ export const layer = Layer.unwrap(
BrowserTraceCollector.layer(sink),
);
}),
).pipe(Layer.provide(OtlpExporter.layerFlusher), Layer.provideMerge(layerSerialization));
).pipe(Layer.provideMerge(layerSerialization));

const metricsLayer =
config.otlpMetricsUrl === undefined
Expand Down
127 changes: 127 additions & 0 deletions apps/server/src/observability/ServerTraceExporter.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
import { assert, it } from "@effect/vitest";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as TestClock from "effect/testing/TestClock";
import { HttpClient, HttpClientResponse } from "effect/http";
import { OtlpSerialization } from "effect/observability";
import type * as OtlpTracer from "effect/observability/OtlpTracer";

import * as ServerTraceExporter from "./ServerTraceExporter.ts";

const options = {
url: "http://localhost:4318/v1/traces",
exportInterval: "1 second",
resource: { serviceName: "t3-test" },
} as const;

const sender = (client: HttpClient.HttpClient) =>
ServerTraceExporter.make(options).pipe(
Effect.provide(
Layer.mergeAll(OtlpSerialization.layerJson, Layer.succeed(HttpClient.HttpClient, client)),
),
);

const emit = (name: string) => Effect.void.pipe(Effect.withSpan(name, { root: true }));

it.effect("limits pending trace spans and serializes exports while a collector is slow", () =>
Effect.gen(function* () {
const firstResponse = yield* Deferred.make<void>();
const batches: Array<ReadonlyArray<string>> = [];
const tracer = yield* sender(
HttpClient.make((request) =>
Effect.gen(function* () {
if (request.body._tag !== "Uint8Array") return yield* Effect.die("Expected JSON");
const payload = JSON.parse(
new TextDecoder().decode(request.body.body),
) as OtlpTracer.TraceData;
batches.push(
payload.resourceSpans.flatMap((resource) =>
resource.scopeSpans.flatMap((scope) => scope.spans.map((span) => span.name)),
),
);
if (batches.length === 1) yield* Deferred.await(firstResponse);
return HttpClientResponse.fromWeb(request, new Response(null, { status: 204 }));
}),
),
);

yield* Effect.addFinalizer(() => Deferred.succeed(firstResponse, undefined));
yield* emit("first").pipe(Effect.withTracer(tracer));
yield* TestClock.adjust("1 second");
assert.deepStrictEqual(batches, [["first"]]);
yield* Effect.forEach(
Array.from({ length: 2_500 }, (_, index) => index),
(index) => emit(index < 1_000 ? "pending" : "overflow"),
{ discard: true },
).pipe(Effect.withTracer(tracer));
yield* TestClock.adjust("1 second");
assert.deepStrictEqual(batches, [["first"]]);

yield* Deferred.succeed(firstResponse, undefined);
yield* TestClock.adjust("1 second");
assert.lengthOf(batches, 2);
assert.lengthOf(batches[1]!, 1_000);
assert.isTrue(batches[1]!.every((name) => name === "pending"));

yield* emit("after").pipe(Effect.withTracer(tracer));
yield* TestClock.adjust("1 second");
assert.deepStrictEqual(batches[2], ["after"]);
}),
);

it.effect("aborts an unresponsive trace export and sends later spans", () =>
Effect.gen(function* () {
const firstAborted = yield* Deferred.make<void>();
const firstResponse = yield* Deferred.make<void>();
let requests = 0;
const tracer = yield* sender(
HttpClient.make((request) =>
Effect.suspend(() => {
requests++;
const response = HttpClientResponse.fromWeb(request, new Response(null, { status: 204 }));
return requests === 1
? Deferred.await(firstResponse).pipe(
Effect.onInterrupt(() => Deferred.succeed(firstAborted, undefined)),
Effect.as(response),
)
: Effect.succeed(response);
}),
),
);

yield* Effect.addFinalizer(() => Deferred.succeed(firstResponse, undefined));
yield* emit("stalled").pipe(Effect.withTracer(tracer));
yield* TestClock.adjust("1 second");
assert.strictEqual(requests, 1);
yield* TestClock.adjust("10 seconds");
assert.isTrue(yield* Deferred.isDone(firstAborted));

// The Effect exporter skips retrying a failed collector for one minute.
yield* TestClock.adjust("1 minute");
yield* emit("recovered").pipe(Effect.withTracer(tracer));
yield* TestClock.adjust("1 second");
assert.strictEqual(requests, 2);
}),
);

it.effect("flushes pending spans when the exporter scope closes", () =>
Effect.gen(function* () {
let requests = 0;
yield* Effect.scoped(
Effect.gen(function* () {
const tracer = yield* sender(
HttpClient.make((request) =>
Effect.sync(() => {
requests++;
return HttpClientResponse.fromWeb(request, new Response(null, { status: 204 }));
}),
),
);
yield* emit("pending").pipe(Effect.withTracer(tracer));
assert.strictEqual(requests, 0);
}),
);
assert.strictEqual(requests, 1);
}),
);
75 changes: 75 additions & 0 deletions apps/server/src/observability/ServerTraceExporter.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import * as Context from "effect/Context";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Queue from "effect/Queue";
import * as Tracer from "effect/Tracer";
import * as OtlpExporter from "effect/observability/OtlpExporter";
import * as OtlpTracer from "effect/observability/OtlpTracer";

const MAX_PENDING_SPANS = 1_000;
const EXPORT_TIMEOUT = "10 seconds";

export const make = Effect.fn("ServerTraceExporter.make")(
function* (options: {
readonly url: string;
readonly exportInterval: Duration.Input;
readonly headers?: Parameters<typeof OtlpTracer.make>[0]["headers"];
readonly resource?: Parameters<typeof OtlpTracer.make>[0]["resource"];
}) {
const flusher = yield* OtlpExporter.Flusher;
// Own both flush triggers so a slow collector cannot accumulate concurrent export bodies.
const tracer = yield* OtlpTracer.make({
...options,
exportInterval: Duration.infinity,
maxBatchSize: Number.POSITIVE_INFINITY,
});
const wake = yield* Queue.sliding<void>(1);
let pending = 0;
let dropped = 0;

const flush = Effect.suspend(() => {
const droppedSpans = dropped;
dropped = 0;
pending = 0;
return flusher.flush.pipe(
Effect.timeoutOption(EXPORT_TIMEOUT),
Effect.andThen(
droppedSpans === 0
? Effect.void
: Effect.logWarning("Dropped OTLP trace spans while the export buffer was full", {
count: droppedSpans,
}),
),
);
});

yield* Queue.take(wake).pipe(
Effect.timeoutOption(options.exportInterval),
Effect.andThen(flush),
Effect.forever,
Effect.withTracerEnabled(false),
Effect.updateContext(Context.omit(Tracer.ParentSpan)<never>),
Effect.forkScoped,
);

return Tracer.make({
span(spanOptions) {
const span = tracer.span(spanOptions);
if (!span.sampled) return span;
const end = span.end.bind(span);
span.end = (endTime, exit) => {
if (pending >= MAX_PENDING_SPANS) {
dropped++;
return;
}
end(endTime, exit);
pending++;
if (pending === MAX_PENDING_SPANS) Queue.offerUnsafe(wake, undefined);
};
return span;
},
});
},
Effect.provide(Layer.fresh(OtlpExporter.layerFlusher)),
);
Loading