diff --git a/packages/sdk/src/realtime/client.ts b/packages/sdk/src/realtime/client.ts index b505e17..ec66de3 100644 --- a/packages/sdk/src/realtime/client.ts +++ b/packages/sdk/src/realtime/client.ts @@ -81,6 +81,18 @@ const realTimeClientConnectOptionsSchema = z.object({ resolution: z.enum(["720p", "1080p"]).optional(), /** Local track publish codec. Desktop Safari is always pinned to vp8 and ignores this value. */ preferredVideoCodec: z.enum(["h264", "vp8", "vp9"]).optional(), + /** + * Seed the publisher's initial bandwidth estimate, in kbps + * (`x-google-start-bitrate` applied to the camera track's SDP). By default + * browsers start the estimate at ~300 kbps and ramp over several seconds, + * so sessions begin at reduced input resolution on links that could carry + * more. Seeding raises the starting point: the estimator's startup probes + * scale from this value (3x/6x), so full-resolution bandwidth is validated + * within the first probe round on capable links. Values above ~1100 kbps + * measurably degrade session startup on very weak (<1 Mbps) uplinks — + * prefer canary-guarded rollouts. 0/undefined = browser default. + */ + startBitrateKbps: z.number().int().min(0).max(10_000).optional(), /** * @deprecated Glass-to-glass measurement now runs automatically in browsers * when LiveKit frame metadata is available. This legacy flag is accepted for @@ -154,6 +166,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { initialState, resolution, preferredVideoCodec, + startBitrateKbps, } = parsedOptions.data; const mirror = parsedOptions.data.mirror ?? false; @@ -214,6 +227,7 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => { initialPassthrough: initialState?.passthrough, logger, videoCodec: preparedConnection.videoCodec, + startBitrateKbps, createMediaChannel: preparedConnection.createMediaChannel, }); diff --git a/packages/sdk/src/realtime/media-channel.ts b/packages/sdk/src/realtime/media-channel.ts index 1e543a8..ec3b36b 100644 --- a/packages/sdk/src/realtime/media-channel.ts +++ b/packages/sdk/src/realtime/media-channel.ts @@ -12,6 +12,7 @@ import { createConsoleLogger, type Logger } from "../utils/logger"; import { REALTIME_CONFIG } from "./config-realtime"; import { loadLiveKitClient } from "./livekit"; import type { RealtimeObservability } from "./observability/realtime-observability"; +import { installStartBitrateMunge } from "./start-bitrate"; export type VideoCodec = "h264" | "vp8" | "vp9" | "av1"; @@ -48,6 +49,8 @@ export interface MediaChannelConfig { localStream: MediaStream | null; logger?: Logger; videoCodec?: VideoCodec; + /** Seed the publisher's initial bandwidth estimate (kbps). See `startBitrateKbps` on connect options. */ + startBitrateKbps?: number; createFrameMetadataWorker?: () => Worker; } @@ -75,6 +78,7 @@ export class LiveKitMediaChannel implements MediaChannel { private frameMetadataEnabled = false; private events: Emitter = mitt(); private readonly logger: Logger; + private uninstallStartBitrateMunge: (() => void) | null = null; constructor(private readonly config: MediaChannelConfig) { this.logger = config.logger ?? createConsoleLogger("warn"); @@ -95,6 +99,12 @@ export class LiveKitMediaChannel implements MediaChannel { async connect(opts: MediaConnectOptions): Promise { const { Room: LiveKitRoom, RoomEvent, Track } = await loadLiveKitClient(); this.cameraTrackSource = Track.Source.Camera; + if (this.config.startBitrateKbps && !this.uninstallStartBitrateMunge) { + // Installed for the channel's whole lifetime (not just this connect): + // LiveKit full reconnects create fresh peer connections that must be + // seeded too. Uninstalled in disconnect(). + this.uninstallStartBitrateMunge = installStartBitrateMunge(this.config.startBitrateKbps, this.logger); + } if (!this.room) { let worker: Worker | undefined; if (this.config.createFrameMetadataWorker) { @@ -178,6 +188,8 @@ export class LiveKitMediaChannel implements MediaChannel { } disconnect(): void { + this.uninstallStartBitrateMunge?.(); + this.uninstallStartBitrateMunge = null; const room = this.room; this.room = null; this.cameraTrackSource = null; diff --git a/packages/sdk/src/realtime/start-bitrate.ts b/packages/sdk/src/realtime/start-bitrate.ts new file mode 100644 index 0000000..1984a00 --- /dev/null +++ b/packages/sdk/src/realtime/start-bitrate.ts @@ -0,0 +1,131 @@ +import type { Logger } from "../utils/logger"; + +/** + * Video codecs whose fmtp carries `x-google-start-bitrate`. Payload types are + * matched from `a=rtpmap` so retransmission/FEC payloads (rtx, red, ulpfec — + * also clocked at /90000) are never munged. + */ +const VIDEO_CODEC_RTPMAP = /^a=rtpmap:(\d+) (VP8|VP9|H264|H265|AV1)\/90000/i; + +/** + * Append `x-google-start-bitrate=` to every video codec's fmtp in the + * SDP (adding an fmtp line when the codec has none — VP8 typically doesn't). + * + * libwebrtc reads the parameter off the applied session descriptions and uses + * it as the initial send-side bandwidth estimate, replacing the stock + * ~300 kbps cold start. Idempotent: descriptions that already carry the + * parameter (e.g. on a renegotiation of an already-munged session) are left + * untouched. + */ +export function mungeStartBitrate(sdp: string, startKbps: number): string { + if (!(startKbps > 0)) return sdp; + const lines = sdp.split("\r\n"); + + const videoPts = new Set(); + const ptsWithFmtp = new Set(); + for (const line of lines) { + const rtpmap = line.match(VIDEO_CODEC_RTPMAP); + if (rtpmap?.[1]) videoPts.add(rtpmap[1]); + const fmtp = line.match(/^a=fmtp:(\d+) /); + if (fmtp?.[1]) ptsWithFmtp.add(fmtp[1]); + } + if (videoPts.size === 0) return sdp; + + const param = `x-google-start-bitrate=${startKbps}`; + const out: string[] = []; + for (const line of lines) { + const fmtp = line.match(/^a=fmtp:(\d+) (.+)$/); + if (fmtp?.[1] && fmtp[2] && videoPts.has(fmtp[1]) && !line.includes("x-google-start-bitrate")) { + out.push(`a=fmtp:${fmtp[1]} ${fmtp[2]};${param}`); + continue; + } + out.push(line); + const rtpmap = line.match(VIDEO_CODEC_RTPMAP); + if (rtpmap?.[1] && !ptsWithFmtp.has(rtpmap[1])) { + // Codec has no fmtp line anywhere in the SDP — give it one so the + // parameter still reaches libwebrtc (fmtp placement after rtpmap is + // valid and conventional). + out.push(`a=fmtp:${rtpmap[1]} ${param}`); + ptsWithFmtp.add(rtpmap[1]); + } + } + return out.join("\r\n"); +} + +/** + * Patch the global `RTCPeerConnection` so every description applied while the + * patch is installed — local offers and remote answers alike, across LiveKit + * reconnects — carries the start-bitrate parameter. Returns an uninstaller. + * + * Patching the global is deliberate: livekit-client owns its peer connections + * and exposes no SDP hook, and this is the exact mechanism validated on the + * probe rig. The uninstaller restores the original constructor only if nobody + * else has re-patched the global since (never clobbers a foreign patch). + */ +export function installStartBitrateMunge(startKbps: number, logger?: Logger): () => void { + if (!(startKbps > 0)) return () => {}; + const g = globalThis as { RTCPeerConnection?: typeof RTCPeerConnection }; + const OriginalPC = g.RTCPeerConnection; + if (typeof OriginalPC !== "function") { + logger?.warn("startBitrateKbps ignored: no global RTCPeerConnection in this environment"); + return () => {}; + } + + let logged = false; + const logOnce = (leg: string) => { + if (logged) return; + logged = true; + logger?.info(`publisher start-bitrate seeded to ${startKbps} kbps (x-google-start-bitrate, ${leg})`); + }; + + const munged = (description: T, leg: string): T => { + if (typeof description.sdp !== "string") return description; + const sdp = mungeStartBitrate(description.sdp, startKbps); + if (sdp === description.sdp) return description; + logOnce(leg); + return { ...description, sdp }; + }; + + class StartBitratePC extends OriginalPC { + // Seed ONLY the first offer/answer pair. libwebrtc re-initializes the + // send-side estimate to x-google-start-bitrate on EVERY applied + // description that carries it — munging renegotiations (e.g. LiveKit's + // track-publish round) resets an already-converged estimator back down + // to the seed (measured live: est 5301 kbps → 1106 kbps at the second + // negotiation, exactly the seed value). + #mungedLegs = 0; + + #maybeMunge(description: T, leg: string): T { + if (this.#mungedLegs >= 2) return description; + const result = munged(description, leg); + if (result !== description) this.#mungedLegs++; + return result; + } + + override setLocalDescription(description?: RTCLocalSessionDescriptionInit): Promise { + // Argless form (implicit offer/answer) has no SDP to munge — the + // browser builds the description internally; the remote-answer leg + // still seeds those sessions. + if (description === undefined) return super.setLocalDescription(); + return super.setLocalDescription(this.#maybeMunge(description, "local offer")); + } + override setRemoteDescription(description: RTCSessionDescriptionInit): Promise { + return super.setRemoteDescription(this.#maybeMunge(description, "remote answer")); + } + } + + try { + g.RTCPeerConnection = StartBitratePC as typeof RTCPeerConnection; + } catch (error) { + logger?.warn("startBitrateKbps ignored: RTCPeerConnection global is not patchable", { + error: error instanceof Error ? error.message : String(error), + }); + return () => {}; + } + + return () => { + if (g.RTCPeerConnection === StartBitratePC) { + g.RTCPeerConnection = OriginalPC; + } + }; +} diff --git a/packages/sdk/src/realtime/stream-session.ts b/packages/sdk/src/realtime/stream-session.ts index dedbfb3..582162d 100644 --- a/packages/sdk/src/realtime/stream-session.ts +++ b/packages/sdk/src/realtime/stream-session.ts @@ -62,6 +62,7 @@ interface StreamSessionConfig { initialPassthrough?: boolean; logger?: Logger; videoCodec?: VideoCodec; + startBitrateKbps?: number; createMediaChannel: MediaChannelFactory; } @@ -338,6 +339,7 @@ export class StreamSession { localStream: this.config.localStream, logger: this.logger, videoCodec: this.config.videoCodec, + startBitrateKbps: this.config.startBitrateKbps, }); this.wireSignalingEvents(); this.wireMediaEvents(); diff --git a/packages/sdk/tests/start-bitrate.unit.test.ts b/packages/sdk/tests/start-bitrate.unit.test.ts new file mode 100644 index 0000000..9d7f831 --- /dev/null +++ b/packages/sdk/tests/start-bitrate.unit.test.ts @@ -0,0 +1,168 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { installStartBitrateMunge, mungeStartBitrate } from "../src/realtime/start-bitrate.js"; + +const SDP = [ + "v=0", + "o=- 1 1 IN IP4 127.0.0.1", + "m=audio 9 UDP/TLS/RTP/SAVPF 111", + "a=rtpmap:111 opus/48000/2", + "a=fmtp:111 minptime=10;useinbandfec=1", + "m=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99 100", + "a=rtpmap:96 VP9/90000", + "a=fmtp:96 profile-id=0", + "a=rtpmap:97 rtx/90000", + "a=fmtp:97 apt=96", + "a=rtpmap:98 VP8/90000", + "a=rtpmap:99 H264/90000", + "a=fmtp:99 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", + "a=rtpmap:100 red/90000", + "", +].join("\r\n"); + +describe("mungeStartBitrate", () => { + it("appends the parameter to every video codec fmtp — VP9 and H264, both payload types", () => { + const out = mungeStartBitrate(SDP, 1100); + expect(out).toContain("a=fmtp:96 profile-id=0;x-google-start-bitrate=1100"); + expect(out).toContain( + "a=fmtp:99 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f;x-google-start-bitrate=1100", + ); + }); + + it("adds an fmtp line for video codecs that have none (VP8)", () => { + const out = mungeStartBitrate(SDP, 1100); + const lines = out.split("\r\n"); + const vp8Index = lines.indexOf("a=rtpmap:98 VP8/90000"); + expect(lines[vp8Index + 1]).toBe("a=fmtp:98 x-google-start-bitrate=1100"); + }); + + it("never touches audio, rtx, or red payloads", () => { + const out = mungeStartBitrate(SDP, 1100); + expect(out).toContain("a=fmtp:111 minptime=10;useinbandfec=1"); + expect(out).toContain("a=fmtp:97 apt=96"); + expect(out).not.toContain("a=fmtp:97 apt=96;x-google"); + expect(out).not.toContain("a=fmtp:100"); + }); + + it("is idempotent: an already-munged description is returned unchanged", () => { + const once = mungeStartBitrate(SDP, 1100); + expect(mungeStartBitrate(once, 1100)).toBe(once); + }); + + it("is the identity for non-positive values and video-free SDPs", () => { + expect(mungeStartBitrate(SDP, 0)).toBe(SDP); + const audioOnly = "m=audio 9 UDP/TLS/RTP/SAVPF 111\r\na=rtpmap:111 opus/48000/2"; + expect(mungeStartBitrate(audioOnly, 1100)).toBe(audioOnly); + }); +}); + +type AnyDescription = { type?: string; sdp?: string } | undefined; + +class FakePC { + static instances: FakePC[] = []; + lastLocal: AnyDescription | "ARGLESS"; + lastRemote: AnyDescription; + constructor() { + FakePC.instances.push(this); + } + setLocalDescription(...args: [AnyDescription?]): Promise { + this.lastLocal = args.length === 0 ? "ARGLESS" : args[0]; + return Promise.resolve(); + } + setRemoteDescription(description: AnyDescription): Promise { + this.lastRemote = description; + return Promise.resolve(); + } +} + +const g = globalThis as { RTCPeerConnection?: unknown }; + +describe("installStartBitrateMunge", () => { + afterEach(() => { + delete g.RTCPeerConnection; + FakePC.instances = []; + }); + + it("munges local offers and remote answers on connections created while installed", async () => { + g.RTCPeerConnection = FakePC; + const uninstall = installStartBitrateMunge(1100); + const pc = new (g.RTCPeerConnection as new () => FakePC)(); + + await pc.setLocalDescription({ type: "offer", sdp: SDP }); + expect((pc.lastLocal as { sdp: string }).sdp).toContain("x-google-start-bitrate=1100"); + + await pc.setRemoteDescription({ type: "answer", sdp: SDP }); + expect((pc.lastRemote as { sdp: string }).sdp).toContain("x-google-start-bitrate=1100"); + uninstall(); + }); + + it("seeds only the first offer/answer pair — renegotiations pass through unmunged", async () => { + g.RTCPeerConnection = FakePC; + const uninstall = installStartBitrateMunge(1100); + const pc = new (g.RTCPeerConnection as new () => FakePC)(); + + await pc.setLocalDescription({ type: "offer", sdp: SDP }); + await pc.setRemoteDescription({ type: "answer", sdp: SDP }); + expect((pc.lastLocal as { sdp: string }).sdp).toContain("x-google-start-bitrate=1100"); + expect((pc.lastRemote as { sdp: string }).sdp).toContain("x-google-start-bitrate=1100"); + + // Renegotiation (track publish): re-munging would reset a converged + // estimator back to the seed — these must pass through untouched. + await pc.setLocalDescription({ type: "offer", sdp: SDP }); + await pc.setRemoteDescription({ type: "answer", sdp: SDP }); + expect((pc.lastLocal as { sdp: string }).sdp).not.toContain("x-google-start-bitrate"); + expect((pc.lastRemote as { sdp: string }).sdp).not.toContain("x-google-start-bitrate"); + + // A NEW peer connection (reconnect) gets its own first-pair seeding. + const pc2 = new (g.RTCPeerConnection as new () => FakePC)(); + await pc2.setLocalDescription({ type: "offer", sdp: SDP }); + expect((pc2.lastLocal as { sdp: string }).sdp).toContain("x-google-start-bitrate=1100"); + uninstall(); + }); + + it("passes the argless setLocalDescription form through untouched", async () => { + g.RTCPeerConnection = FakePC; + const uninstall = installStartBitrateMunge(1100); + const pc = new (g.RTCPeerConnection as new () => FakePC)(); + await pc.setLocalDescription(); + expect(pc.lastLocal).toBe("ARGLESS"); + uninstall(); + }); + + it("logs once on the first munged description", async () => { + g.RTCPeerConnection = FakePC; + const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }; + const uninstall = installStartBitrateMunge(1100, logger); + const pc = new (g.RTCPeerConnection as new () => FakePC)(); + await pc.setLocalDescription({ type: "offer", sdp: SDP }); + await pc.setRemoteDescription({ type: "answer", sdp: SDP }); + expect(logger.info).toHaveBeenCalledTimes(1); + expect(logger.info.mock.calls[0]?.[0]).toContain("1100 kbps"); + uninstall(); + }); + + it("uninstall restores the original constructor, but never clobbers a foreign patch", () => { + g.RTCPeerConnection = FakePC; + const uninstall = installStartBitrateMunge(1100); + expect(g.RTCPeerConnection).not.toBe(FakePC); + + const foreign = class {}; + g.RTCPeerConnection = foreign; + uninstall(); + expect(g.RTCPeerConnection).toBe(foreign); + + g.RTCPeerConnection = FakePC; + const uninstall2 = installStartBitrateMunge(1100); + uninstall2(); + expect(g.RTCPeerConnection).toBe(FakePC); + }); + + it("is a no-op without a global RTCPeerConnection or for non-positive values", () => { + const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }; + expect(installStartBitrateMunge(1100, logger)).toBeTypeOf("function"); + expect(logger.warn).toHaveBeenCalledOnce(); + + g.RTCPeerConnection = FakePC; + installStartBitrateMunge(0)(); + expect(g.RTCPeerConnection).toBe(FakePC); + }); +});