Instrument application with opentelemetry (#3)

* Add otel telemetry to next app

* Simplify otel implementation

* remove plan

Author: ostenbom

app/api/team/route.ts

@@ -12,6 +12,12 @@ export async function GET() {
 		});
 	}
 
+	if (await isFaultActive("api-team-db-read-skipped")) {
+		return Response.json(null, {
+			headers: { "x-fault-injected": "api-team-db-read-skipped" },
+		});
+	}
+
 	const team = await getTeamForUser();
 	return Response.json(team);
 }

instrumentation.ts

@@ -0,0 +1,19 @@
+import { registerOTel } from "@vercel/otel";
+
+function backendOtelEnabled() {
+	return Boolean(process.env.OTEL_EXPORTER_OTLP_ENDPOINT);
+}
+
+export function register() {
+	if (!backendOtelEnabled()) {
+		return;
+	}
+
+	registerOTel({
+		serviceName: process.env.OTEL_SERVICE_NAME || "playwright-tutorial-next",
+		attributes: {
+			"service.namespace": "playwright-tutorial",
+			"endform.telemetry.source": "next-app",
+		},
+	});
+}

lib/auth/middleware.ts

@@ -6,15 +6,15 @@ import type { TeamDataWithMembers, User } from "@/lib/db/schema";
 export type ActionState = {
 	error?: string;
 	success?: string;
-	[key: string]: any; // This allows for additional properties
+	[key: string]: string | number | readonly string[] | undefined;
 };
 
-type ValidatedActionFunction<S extends z.ZodType<any, any>, T> = (
+type ValidatedActionFunction<S extends z.ZodType, T> = (
 	data: z.infer<S>,
 	formData: FormData,
 ) => Promise<T>;
 
-export function validatedAction<S extends z.ZodType<any, any>, T>(
+export function validatedAction<S extends z.ZodType, T>(
 	schema: S,
 	action: ValidatedActionFunction<S, T>,
 ) {
@@ -28,13 +28,13 @@ export function validatedAction<S extends z.ZodType<any, any>, T>(
 	};
 }
 
-type ValidatedActionWithUserFunction<S extends z.ZodType<any, any>, T> = (
+type ValidatedActionWithUserFunction<S extends z.ZodType, T> = (
 	data: z.infer<S>,
 	formData: FormData,
 	user: User,
 ) => Promise<T>;
 
-export function validatedActionWithUser<S extends z.ZodType<any, any>, T>(
+export function validatedActionWithUser<S extends z.ZodType, T>(
 	schema: S,
 	action: ValidatedActionWithUserFunction<S, T>,
 ) {

lib/db/drizzle.ts

@@ -6,6 +6,7 @@ import {
 	type AsyncRemoteCallback,
 	drizzle as drizzleProxy,
 } from "drizzle-orm/sqlite-proxy";
+import { withSpan } from "../telemetry";
 import { resolveRuntimeDatabaseConfig } from "./config";
 import * as schema from "./schema";
 
@@ -14,6 +15,11 @@ dotenv.config();
 type QueryMethod = Parameters<AsyncRemoteCallback>[2];
 type BatchQuery = Parameters<AsyncBatchRemoteCallback>[0][number];
 type ProxyQueryResult = Awaited<ReturnType<AsyncRemoteCallback>>;
+type LibsqlClient = ReturnType<typeof createClient>;
+type InstrumentableLibsqlClient = {
+	execute: (...args: unknown[]) => Promise<unknown>;
+	batch: (...args: unknown[]) => Promise<unknown>;
+};
 
 const databaseConfig = resolveRuntimeDatabaseConfig();
 
@@ -25,6 +31,16 @@ async function executeProxyQuery(
 	sql: string,
 	params: unknown[],
 	method: QueryMethod,
+): Promise<ProxyQueryResult> {
+	return withDbSpan(sql, method, () =>
+		executeProxyQueryUntraced(sql, params, method),
+	);
+}
+
+async function executeProxyQueryUntraced(
+	sql: string,
+	params: unknown[],
+	method: QueryMethod,
 ): Promise<ProxyQueryResult> {
 	const proxyUrl =
 		databaseConfig.mode === "proxy" ? databaseConfig.proxyUrl : undefined;
@@ -51,6 +67,15 @@ async function executeProxyQuery(
 
 async function executeProxyBatch(
 	queries: BatchQuery[],
+): Promise<ProxyQueryResult[]> {
+	return withDbBatchSpan(
+		queries.map((query) => query.sql),
+		() => executeProxyBatchUntraced(queries),
+	);
+}
+
+async function executeProxyBatchUntraced(
+	queries: BatchQuery[],
 ): Promise<ProxyQueryResult[]> {
 	const proxyUrl =
 		databaseConfig.mode === "proxy" ? databaseConfig.proxyUrl : undefined;
@@ -84,6 +109,149 @@ function ensureTrailingSlash(url: string) {
 	return url.endsWith("/") ? url : `${url}/`;
 }
 
+function instrumentLibsqlClient(client: LibsqlClient): LibsqlClient {
+	const instrumented = client as unknown as InstrumentableLibsqlClient;
+	const execute = instrumented.execute.bind(client);
+	const batch = instrumented.batch.bind(client);
+
+	instrumented.execute = (...args: unknown[]) => {
+		const sql = sqlFromStatement(args[0]);
+		return withDbSpan(sql, "execute", () => execute(...args));
+	};
+
+	instrumented.batch = (...args: unknown[]) => {
+		const statements = Array.isArray(args[0]) ? args[0] : [];
+
+		return withDbBatchSpan(
+			statements.map((statement) => sqlFromStatement(statement)),
+			() => batch(...args),
+		);
+	};
+
+	return client;
+}
+
+function withDbSpan<T>(
+	sql: string | undefined,
+	method: string,
+	fn: () => Promise<T>,
+) {
+	const operation = sqlOperation(sql) ?? method;
+	const collection = sqlCollection(sql);
+
+	return withSpan(
+		"db.query",
+		dbQueryAttributes(operation, collection, method),
+		async (span) => {
+			if (await shouldInjectDbLatencySpike(operation, collection)) {
+				span?.setAttribute("app.result", "latency_spike");
+				await sleep(5000);
+			}
+
+			return fn();
+		},
+	);
+}
+
+function withDbBatchSpan<T>(
+	sqlStatements: (string | undefined)[],
+	fn: () => Promise<T>,
+) {
+	return withSpan("db.batch", dbBatchAttributes(sqlStatements), async () =>
+		fn(),
+	);
+}
+
+function dbQueryAttributes(
+	operation: string,
+	collection: string | undefined,
+	method: string,
+) {
+	return {
+		"db.system": "sqlite",
+		"db.operation": operation,
+		"db.query.method": method,
+		...(collection ? { "db.collection": collection } : {}),
+	};
+}
+
+function dbBatchAttributes(sqlStatements: (string | undefined)[]) {
+	const operations = [
+		...new Set(sqlStatements.map(sqlOperation).filter(isString)),
+	];
+	const collections = [
+		...new Set(sqlStatements.map(sqlCollection).filter(isString)),
+	];
+
+	return {
+		"db.system": "sqlite",
+		"db.operation": operations.length === 1 ? operations[0] : "batch",
+		"db.statement_count": sqlStatements.length,
+		...(collections.length === 1 ? { "db.collection": collections[0] } : {}),
+	};
+}
+
+async function shouldInjectDbLatencySpike(
+	operation: string,
+	collection: string | undefined,
+) {
+	return (
+		operation === "select" &&
+		collection === "team_members" &&
+		(await isRequestFaultActive("api-team-db-latency-spike"))
+	);
+}
+
+async function isRequestFaultActive(faultName: string) {
+	try {
+		const { headers } = await import("next/headers");
+		return (await headers()).get("x-faults")?.trim() === faultName;
+	} catch {
+		return false;
+	}
+}
+
+function sleep(ms: number) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function sqlFromStatement(statement: unknown): string | undefined {
+	if (typeof statement === "string") {
+		return statement;
+	}
+	if (Array.isArray(statement) && typeof statement[0] === "string") {
+		return statement[0];
+	}
+	if (isRecord(statement) && typeof statement.sql === "string") {
+		return statement.sql;
+	}
+
+	return undefined;
+}
+
+function sqlOperation(sql: string | undefined) {
+	return sql?.trim().split(/\s+/, 1)[0]?.toLowerCase();
+}
+
+function sqlCollection(sql: string | undefined) {
+	if (!sql) return undefined;
+
+	const normalized = sql.replace(/["`]/g, " ");
+	const match = normalized.match(
+		/\b(?:from|into|update|join)\s+([a-zA-Z_][a-zA-Z0-9_]*)/i,
+	);
+
+	return match?.[1]?.toLowerCase();
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null;
+}
+
+function isString(value: string | undefined): value is string {
+	return typeof value === "string";
+}
+
 function createDatabase() {
 	if (databaseConfig.mode === "proxy") {
 		return drizzleProxy(executeProxyQuery, executeProxyBatch, { schema });
@@ -100,7 +268,7 @@ function createDatabase() {
 				},
 	);
 
-	return drizzleLibsql(client, { schema });
+	return drizzleLibsql(instrumentLibsqlClient(client), { schema });
 }
 
 export const db = createDatabase();

lib/db/queries.ts

@@ -12,16 +12,12 @@ import {
 
 export async function getUser() {
 	const sessionCookie = (await cookies()).get("session");
-	if (!sessionCookie || !sessionCookie.value) {
+	if (!sessionCookie?.value) {
 		return null;
 	}
 
 	const sessionData = await verifyToken(sessionCookie.value);
-	if (
-		!sessionData ||
-		!sessionData.user ||
-		typeof sessionData.user.id !== "number"
-	) {
+	if (!sessionData?.user || typeof sessionData.user.id !== "number") {
 		return null;
 	}
 

lib/telemetry.ts

@@ -0,0 +1,39 @@
+import {
+	type Attributes,
+	type Span,
+	SpanStatusCode,
+	trace,
+} from "@opentelemetry/api";
+
+export function isTelemetryEnabled() {
+	return Boolean(process.env.OTEL_EXPORTER_OTLP_ENDPOINT);
+}
+
+export async function withSpan<T>(
+	name: string,
+	attributes: Attributes,
+	fn: (span?: Span) => Promise<T>,
+) {
+	if (!isTelemetryEnabled()) {
+		return fn();
+	}
+
+	const tracer = trace.getTracer("playwright-tutorial-next");
+
+	return tracer.startActiveSpan(name, { attributes }, async (span) => {
+		try {
+			const result = await fn(span);
+			span.setStatus({ code: SpanStatusCode.OK });
+			return result;
+		} catch (error) {
+			span.recordException(error as Error);
+			span.setStatus({
+				code: SpanStatusCode.ERROR,
+				message: error instanceof Error ? error.message : String(error),
+			});
+			throw error;
+		} finally {
+			span.end();
+		}
+	});
+}

package.json

@@ -16,10 +16,12 @@
 	},
 	"dependencies": {
 		"@libsql/client": "^0.17.4",
+		"@opentelemetry/api": "^1.9.1",
 		"@tailwindcss/postcss": "4.3.1",
 		"@types/node": "^24.13.2",
 		"@types/react": "19.2.17",
 		"@types/react-dom": "19.2.3",
+		"@vercel/otel": "^2.1.3",
 		"bcryptjs": "^3.0.3",
 		"class-variance-authority": "^0.7.1",
 		"clsx": "^2.1.1",