Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions apps/server/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
"diff": "8.0.3",
"effect": "catalog:",
"ffi-rs": "1.3.2",
"jsonc-parser": "3.3.1",
"node-pty": "^1.2.0-beta.15",
"playwright-core": "1.64.1",
"proper-lockfile": "4.1.2",
Expand Down
11 changes: 10 additions & 1 deletion apps/server/src/provider/CodexProvider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,10 @@ export const codexRoutesSpendNativeQuota = (
): boolean =>
account.requiresOpenaiAuth && account.account?.type !== "apiKey" && routesToNativeOpenai(config);

/** Shown when a custom model provider serves the instance; its gateway reports the accounts. */
export const CODEX_CUSTOM_PROVIDER_USAGE_MESSAGE =
"Not applicable: this instance routes through a custom model provider, not OpenAI quota.";

const CODEX_APP_SERVER_PROBE_FORCE_KILL_AFTER = "2 seconds" as const;
const CODEX_CATALOG_REFRESH_TIMEOUT_MS = 3_000;
const CODEX_CONFIG_PROBE_TIMEOUT_MS = 3_000;
Expand Down Expand Up @@ -1133,7 +1137,12 @@ export const checkCodexProviderStatus = Effect.fn("checkCodexProviderStatus")(fu
// Sign-in guidance still applies when Codex itself says it needs an OpenAI login.
const usageLimits =
!snapshot.nativeUsage && !accountStatus.message
? makeUnavailableUsageLimits({ checkedAt, reason: "unsupported" })
? makeUnavailableUsageLimits({
checkedAt,
reason: "unsupported",
// A gateway route reports its pooled accounts itself; say so rather than vanish.
...(snapshot.endpoint ? { message: CODEX_CUSTOM_PROVIDER_USAGE_MESSAGE } : {}),
})
: snapshot.rateLimits === undefined || "failure" in snapshot.rateLimits
? makeUnavailableUsageLimits({
checkedAt,
Expand Down
20 changes: 20 additions & 0 deletions apps/server/src/provider/ProviderRegistry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ import { createModelCapabilities } from "@t3tools/shared/model";
import { applyServerSettingsPatch } from "@t3tools/shared/serverSettings";

import {
CODEX_CUSTOM_PROVIDER_USAGE_MESSAGE,
checkCodexProviderStatus,
mapCodexModelCapabilities,
type CodexAppServerProviderSnapshot,
Expand Down Expand Up @@ -734,6 +735,25 @@ it.layer(
}),
);

it.effect("says why usage is not applicable for a gateway-routed instance", () =>
Effect.gen(function* () {
const status = yield* checkCodexProviderStatus(defaultCodexSettings, () =>
Effect.succeed(
makeCodexProbeSnapshot({
account: { account: null, requiresOpenaiAuth: false },
endpoint: "http://127.0.0.1:3425",
nativeUsage: false,
}),
),
);
assert.strictEqual(status.usageLimits?.unavailable?.reason, "unsupported");
assert.strictEqual(
status.usageLimits?.unavailable?.message,
CODEX_CUSTOM_PROVIDER_USAGE_MESSAGE,
);
}),
);

it.effect("offers cyber access only on catalog models that advertise it", () =>
Effect.gen(function* () {
const catalogModel = (slug: string, cyber?: Array<"standard" | "daybreakBlue">) => ({
Expand Down
3 changes: 3 additions & 0 deletions apps/server/src/usage/UsageLimitSources.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
import * as NodeServices from "@effect/platform-node/NodeServices";
import { expect, it } from "@effect/vitest";
import {
ProviderDriverKind,
Expand Down Expand Up @@ -105,6 +106,7 @@ it.effect("keeps malformed source URLs encodable for config subscribers", () =>
HttpClient.make(() => Effect.die("Unexpected network request")),
),
NodeCrypto.layer,
NodeServices.layer,
]),
),
);
Expand Down Expand Up @@ -171,6 +173,7 @@ it.effect("reads a management key from its secret provider", () => {
),
),
NodeCrypto.layer,
NodeServices.layer,
]),
);
});
Expand Down
84 changes: 78 additions & 6 deletions apps/server/src/usage/UsageLimitSources.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ import * as DateTime from "effect/DateTime";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Equal from "effect/Equal";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Path from "effect/Path";
import * as PubSub from "effect/PubSub";
import * as Ref from "effect/Ref";
import * as Semaphore from "effect/Semaphore";
Expand All @@ -29,6 +31,7 @@ import { resolveSettingsSecrets } from "../secrets/settingsSecretValue.ts";
import * as Settings from "../serverSettings.ts";
import { windowLimitsModel } from "../provider/claudeUsageLimits.ts";
import { makeCliproxyApi } from "./cliproxyApi.ts";
import { discoverGatewayOrigins, gatewayOrigin, makeMagpieApi } from "./magpieQuotas.ts";

export class UsageLimitSources extends Context.Service<
UsageLimitSources,
Expand Down Expand Up @@ -61,6 +64,9 @@ function missingManagementKey(config: UsageLimitSourceConfig): string {
/** @public */
export const make = Effect.gen(function* () {
const api = yield* makeCliproxyApi;
const magpie = yield* makeMagpieApi;
const fileSystem = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const settingsService = yield* Settings.ServerSettingsService;
const secrets = yield* SecretResolver.SecretResolver;
const backgroundPolicy = yield* BackgroundPolicy.BackgroundPolicy;
Expand Down Expand Up @@ -95,19 +101,78 @@ export const make = Effect.gen(function* () {
checkedAt,
} as const;
const managementKey = yield* readManagementKey(config);
if (managementKey.length === 0) {
// A Magpie gateway on loopback answers without a key.
if (config.kind === "cliproxy" && managementKey.length === 0) {
return { ...base, accounts: [], error: missingManagementKey(config) };
}
const accounts = yield* api
.readAccounts({ url: config.url, managementKey })
.pipe(Effect.result);
const accounts = yield* (
config.kind === "magpie"
? magpie.readAccounts({ url: config.url, managementKey })
: api.readAccounts({ url: config.url, managementKey })
).pipe(Effect.result);
if (accounts._tag === "Failure") {
yield* Effect.logDebug("usage limit source read failed", { id, cause: accounts.failure });
return { ...base, accounts: [], error: accounts.failure.detail };
}
return { ...base, accounts: accounts.success };
});

// Gateways the enabled instances already route through, read without
// configuration. An origin that does not answer the quotas route is not a
// Magpie gateway and is dropped quietly; nobody configured it.
const discoverMagpieSources = Effect.fn("UsageLimitSources.discoverMagpieSources")(
function* (
settings: ServerSettings | null,
configured: ReadonlyArray<UsageLimitSourceSnapshot>,
) {
if (!settings) return [];
const explicit = new Set(
Object.values(settings.usageLimitSources)
.filter((config) => config.enabled && config.kind === "magpie")
.map((config) => gatewayOrigin(config.url)),
);
const origins = (yield* discoverGatewayOrigins(settings, process.env).pipe(
Effect.provideService(FileSystem.FileSystem, fileSystem),
Effect.provideService(Path.Path, path),
)).filter((origin) => !explicit.has(origin));
const reads = yield* Effect.forEach(
origins,
(origin) =>
magpie.readAccounts({ url: origin, managementKey: "" }).pipe(
Effect.map((accounts) => ({ origin, accounts })),
Effect.option,
),
{ concurrency: 4 },
);
const checkedAt = DateTime.formatIso(yield* DateTime.now);
// A forward and the gateway it reaches report the same accounts: show one.
const seen = new Set(configured.map((source) => accountSet(source.accounts)));
const discovered: UsageLimitSourceSnapshot[] = [];
for (const read of reads) {
if (read._tag === "None") continue;
const key = accountSet(read.value.accounts);
if (seen.has(key)) continue;
seen.add(key);
discovered.push({
id: `magpie:${new URL(read.value.origin).host}` as UsageLimitSourceId,
kind: "magpie",
label: "Magpie",
origin: read.value.origin,
endpoint: read.value.origin,
checkedAt,
accounts: read.value.accounts,
});
}
return discovered.length > 1
? discovered.map((source) => ({
...source,
label: `Magpie · ${source.id.slice("magpie:".length)}`,
}))
: discovered;
},
Effect.orElseSucceed((): ReadonlyArray<UsageLimitSourceSnapshot> => []),
);

const publish = Effect.fn("UsageLimitSources.publish")(function* (
next: ReadonlyArray<UsageLimitSourceSnapshot>,
) {
Expand All @@ -131,7 +196,7 @@ export const make = Effect.gen(function* () {
([id, config]) => readSource(id as UsageLimitSourceId, config),
{ concurrency: 4 },
);
yield* publish(snapshots);
yield* publish([...snapshots, ...(yield* discoverMagpieSources(settings, snapshots))]);
}).pipe(refreshLock.withPermits(1), Effect.ignoreCause({ log: true }));

// Shares the refresh lock so a stale in-flight read cannot overwrite a redemption.
Expand All @@ -142,7 +207,7 @@ export const make = Effect.gen(function* () {
Effect.mapError(() => new UsageLimitSourceError({ detail: "Could not read hub settings." })),
);
const config = settings.usageLimitSources[input.sourceId];
if (!config?.enabled) {
if (!config?.enabled || config.kind !== "cliproxy") {
return yield* new UsageLimitSourceError({
detail: "The usage limit source is missing or disabled.",
});
Expand Down Expand Up @@ -209,6 +274,13 @@ export const make = Effect.gen(function* () {
} satisfies UsageLimitSources["Service"];
});

function accountSet(accounts: ReadonlyArray<{ readonly id: string }>): string {
return accounts
.map((account) => account.id)
.sort()
.join("\n");
}

export const layer = Layer.effect(UsageLimitSources, make).pipe(
Layer.provide(SecretResolverLayer.layer),
);
Expand Down
Loading
Loading