Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/closura-arc-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,8 @@ jobs:
set -euo pipefail
vp test run \
apps/server/src/speech/CloudSpeech.test.ts \
apps/server/src/speech/GroqSpeechStream.test.ts \
apps/server/src/speech/stream.test.ts \
packages/contracts/src/threadInbox.test.ts \
packages/contracts/src/threadInbox.invariants.test.ts \
packages/contracts/src/threadInbox.upgrade.test.ts \
Expand Down
4 changes: 4 additions & 0 deletions apps/server/src/serverSettings.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1857,6 +1857,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
xai: SECRET_REDACTED,
openrouter: "",
assemblyai: "",
groq: "",
});
assert.equal(legacy.sourceControlHosts.bitbucket?.accessToken, SECRET_REDACTED);
assert.equal(legacy.usageLimitSources[hub]?.managementKey, SECRET_REDACTED);
Expand Down Expand Up @@ -1888,6 +1889,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
xai: "literal-fixture",
openrouter: "",
assemblyai: "",
groq: "",
});
assert.deepEqual(echoed.easDeviceHosts[0]?.accessToken, easReference);
assert.deepEqual(echoed.simlockDeviceHosts[0]?.agentToken, simlockReference);
Expand Down Expand Up @@ -1978,6 +1980,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
xai: work,
openrouter: "",
assemblyai: "",
groq: "",
});
assert.equal(removed.easDeviceHosts[0]?.accessToken, "");
assert.deepEqual(Object.keys(removed.secretProviders), ["work"]);
Expand All @@ -1992,6 +1995,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
"cloudSpeechCredentials.xai",
"cloudSpeechCredentials.openrouter",
"cloudSpeechCredentials.assemblyai",
"cloudSpeechCredentials.groq",
"easDeviceHosts.[].accessToken",
"simlockDeviceHosts.[].agentToken",
"usageLimitSources.*.managementKey",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ it.effect("reads a collected reference again once it is invalidated", () =>
xai: "",
openrouter: "",
assemblyai: "",
groq: "",
},
usageLimitSources,
textGenerators: {},
Expand Down
51 changes: 46 additions & 5 deletions apps/server/src/speech/CloudSpeech.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved
xai: true,
openrouter: false,
assemblyai: false,
groq: false,
});
yield* Effect.flip(transcribe(new Uint8Array(3)));
expect(upstream).not.toHaveBeenCalled();
Expand Down Expand Up @@ -306,11 +307,12 @@ it.effect(
);

it.effect(
"a host-injected AssemblyAI key works without settings and never outranks a saved key",
"host-injected AssemblyAI and Groq keys work without settings and never outrank a saved key",
() =>
Effect.gen(function* () {
const upstream = vi.fn<typeof fetch>(async () => Response.json({ text: "hello" }));
process.env.ASSEMBLYAI_API_KEY = "host-fixture";
process.env.GROQ_API_KEY = "groq-host-fixture";
const service = yield* CloudSpeech.CloudSpeech.pipe(
Effect.provide(
CloudSpeech.layer.pipe(
Expand All @@ -319,14 +321,20 @@ it.effect(
Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})),
Layer.provide(
ConfigProvider.layer(
ConfigProvider.fromEnv({ env: { ASSEMBLYAI_API_KEY: " host-fixture\n" } }),
ConfigProvider.fromEnv({
env: {
ASSEMBLYAI_API_KEY: " host-fixture\n",
GROQ_API_KEY: "groq-host-fixture",
},
}),
),
),
),
),
);
// Agent subprocesses must not inherit the key.
expect(process.env.ASSEMBLYAI_API_KEY).toBeUndefined();
expect(process.env.GROQ_API_KEY).toBeUndefined();
const authorization = () =>
new Headers(upstream.mock.calls.at(-1)![1]?.headers).get("authorization");
const transcribe = () =>
Expand All @@ -340,17 +348,50 @@ it.effect(
xai: false,
openrouter: false,
assemblyai: true,
hostManaged: ["assemblyai"],
groq: true,
hostManaged: ["assemblyai", "groq"],
});
expect(yield* transcribe()).toBe("hello");
expect(authorization()).toBe("host-fixture");
yield* service.setKey("assemblyai", Redacted.make("saved-fixture"));
expect((yield* service.configured).hostManaged).toBeUndefined();
expect((yield* service.configured).hostManaged).toEqual(["groq"]);
yield* transcribe();
expect(authorization()).toBe("saved-fixture");
yield* service.setKey("assemblyai", null);
expect((yield* service.configured).hostManaged).toEqual(["assemblyai"]);
expect((yield* service.configured).hostManaged).toEqual(["assemblyai", "groq"]);
yield* transcribe();
expect(authorization()).toBe("host-fixture");
expect(
yield* service
.transcribe("groq", new Uint8Array(new Float32Array(1600).buffer), {
...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
speechLanguage: "en",
speechCustomWords: [
{ term: "T3 Code", aliases: [] },
{ term: "Closura", aliases: [] },
],
})
.pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)),
).toBe("hello");
const [url, init] = upstream.mock.calls.at(-1)!;
expect(String(url)).toBe("https://api.groq.com/openai/v1/audio/transcriptions");
expect(init?.redirect).toBe("error");
expect(authorization()).toBe("Bearer groq-host-fixture");
const form = init?.body as FormData;
expect(form.get("model")).toBe("whisper-large-v3-turbo");
expect(form.get("language")).toBe("en");
expect(form.get("prompt")).toBe("T3 Code, Closura");
upstream.mockResolvedValueOnce(new Response(null, { status: 429 }));
expect(
yield* Effect.flip(
service
.transcribe("groq", new Uint8Array(new Float32Array(1600).buffer), {
...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
})
.pipe(Effect.provideService(FetchHttpClient.Fetch, upstream)),
),
).toMatchObject({ provider: "groq", reason: "rate-limit" });
yield* service.setKey("groq", Redacted.make("groq-saved-fixture"));
expect((yield* service.configured).hostManaged).toEqual(["assemblyai"]);
}),
);
82 changes: 57 additions & 25 deletions apps/server/src/speech/CloudSpeech.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
type SecretReference,
type ServerSettingsError,
type CloudSpeechKeys,
CloudSpeechCredentials,
type CloudSpeechProvider,
EnvironmentSpeechTranscriptionResult,
type SpeechTranscriptionOptions,
Expand All @@ -34,14 +35,21 @@ const endpoints = {
openai: { url: "https://api.openai.com/v1/audio/transcriptions", model: "gpt-transcribe" },
xai: { url: "https://api.x.ai/v1/stt", model: "grok-voice-transcribe-2.0" },
openrouter: { url: "https://openrouter.ai/api/v1/audio/transcriptions" },
groq: {
url: "https://api.groq.com/openai/v1/audio/transcriptions",
model: "whisper-large-v3-turbo",
},
};
const GROQ_PROMPT_CHARACTERS = 800;
const OPENROUTER_CATALOG_URL =
"https://openrouter.ai/api/v1/models?output_modalities=transcription&zdr=true";
/**
* Keys a host injects at launch (for example `infisical run -- t3`), so nobody types or sees them.
* A key saved in settings still wins.
*/
const HOST_KEY_VARIABLES = { assemblyai: "ASSEMBLYAI_API_KEY" } as const;
const HOST_KEY_VARIABLES = { assemblyai: "ASSEMBLYAI_API_KEY", groq: "GROQ_API_KEY" } as const;
type HostKeyProvider = keyof typeof HOST_KEY_VARIABLES;
const HOST_KEY_PROVIDERS = Object.keys(HOST_KEY_VARIABLES) as HostKeyProvider[];
const isApiKey = Schema.is(CloudSpeechApiKey);
const isModelId = Schema.is(CloudSpeechModelId);
const sameCredential = Schema.toEquivalence(SettingsSecretValue);
Expand Down Expand Up @@ -113,6 +121,21 @@ function assemblyAiBody(audio: ArrayBuffer, language: string, options: SpeechTra
return HttpClientRequest.bodyFormData(form);
}

function groqBody(audio: ArrayBuffer, language: string, options: SpeechTranscriptionOptions) {
const form = new FormData();
form.set("model", endpoints.groq.model);
form.set("response_format", "json");
form.set("temperature", "0");
if (language !== "auto") form.set("language", language);
const prompt = options.speechCustomWords
.map(({ term }) => term)
.join(", ")
.slice(0, GROQ_PROMPT_CHARACTERS);
if (prompt) form.set("prompt", prompt);
form.set("file", new Blob([audio], { type: "audio/wav" }), "dictation.wav");
return HttpClientRequest.bodyFormData(form);
}

function openRouterBody(model: string, audio: ArrayBuffer, language: string) {
return HttpClientRequest.bodyJsonUnsafe({
model,
Expand Down Expand Up @@ -147,35 +170,42 @@ export const layer = Layer.effect(
const settings = yield* ServerSettings.ServerSettingsService;
const resolver = yield* SecretResolver.SecretResolver;
const http = yield* HttpClient.HttpClient;
const hostAssemblyAi = Option.filter(
Option.map(yield* Config.Redacted(HOST_KEY_VARIABLES.assemblyai).pipe(Config.option), (key) =>
Redacted.value(key).trim(),
),
isApiKey,
);
// Agent subprocesses inherit this environment; only this service should hold the key.
// @effect-diagnostics-next-line processEnvInEffect:off
delete process.env[HOST_KEY_VARIABLES.assemblyai];
const hostKeys: Partial<Record<HostKeyProvider, string>> = {};
for (const provider of HOST_KEY_PROVIDERS) {
const key = Option.filter(
Option.map(yield* Config.Redacted(HOST_KEY_VARIABLES[provider]).pipe(Config.option), (k) =>
Redacted.value(k).trim(),
),
isApiKey,
);
if (Option.isSome(key)) hostKeys[provider] = key.value;
// Agent subprocesses inherit this environment; only this service should hold the key.
// @effect-diagnostics-next-line processEnvInEffect:off
delete process.env[HOST_KEY_VARIABLES[provider]];
}
const savedCredentials = settings.getSettings.pipe(
Effect.map((value) => value.cloudSpeechCredentials),
);
const hostManagedFor = (saved: Pick<typeof CloudSpeechCredentials.Type, HostKeyProvider>) =>
HOST_KEY_PROVIDERS.filter((provider) => !saved[provider] && hostKeys[provider]);
const credentials = savedCredentials.pipe(
Effect.map((saved) =>
!saved.assemblyai && Option.isSome(hostAssemblyAi)
? { ...saved, assemblyai: hostAssemblyAi.value }
: saved,
),
Effect.map((saved) => ({
...saved,
...Object.fromEntries(hostManagedFor(saved).map((p) => [p, hostKeys[p]!])),
})),
);
const configured = savedCredentials.pipe(
Effect.map(({ openai, xai, openrouter, assemblyai }) => ({
openai: Boolean(openai),
xai: Boolean(xai),
openrouter: Boolean(openrouter),
assemblyai: Boolean(assemblyai) || Option.isSome(hostAssemblyAi),
...(!assemblyai && Option.isSome(hostAssemblyAi)
? { hostManaged: ["assemblyai" as const] }
: {}),
})),
Effect.map((saved) => {
const hostManaged = hostManagedFor(saved);
return {
openai: Boolean(saved.openai),
xai: Boolean(saved.xai),
openrouter: Boolean(saved.openrouter),
assemblyai: Boolean(saved.assemblyai) || hostManaged.includes("assemblyai"),
groq: Boolean(saved.groq) || hostManaged.includes("groq"),
...(hostManaged.length ? { hostManaged } : {}),
};
}),
);
const execute = (request: HttpClientRequest.HttpClientRequest) =>
http
Expand Down Expand Up @@ -262,7 +292,9 @@ export const layer = Layer.effect(
? openRouterBody(model!, audio, language)
: provider === "assemblyai"
? assemblyAiBody(audio, language, options)
: audioBody(provider, audio, language, options),
: provider === "groq"
? groqBody(audio, language, options)
: audioBody(provider, audio, language, options),
);
const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" });
// A catalog fetch or vault lookup may have outlived the credential it started with.
Expand Down
Loading
Loading