Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion memory-bank/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,11 @@ ordinary Bedrock models through Converse, and OpenAI families through Mantle.
Mantle has one independently cached discovery source,
`GET https://bedrock-mantle.{region}.api.aws/v1/models`, signed for the
`bedrock-mantle` service. A Mantle discovery failure never clears or replaces
the Converse cache.
the Converse cache. `listMantleModels()` returns `[]` only for answers a retry
won't change (a 404, or an access-denied body), and those are cached for the
TTL. Every other failure (expired credentials, signature rejections, 5xx,
unexpected bodies) returns `null`, lists no Mantle models, and is not cached,
so the next listing retries it, for example after `aws sso login`.

AWS publishes no FIPS endpoint for Mantle. When the standard AWS FIPS setting
is enabled, the shared transport policy suppresses Mantle discovery and the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,54 @@ describe("Bedrock provider Mantle aggregation", () => {
);
});

it("re-lists Mantle on the next listing after credential resolution fails", async () => {
const actual = await vi.importActual<typeof import("../bedrock-mantle-models")>(
"../bedrock-mantle-models",
);
listMantleModels.mockImplementation(actual.listMantleModels);
vi.stubGlobal(
"fetch",
vi.fn(
async () =>
new Response(JSON.stringify({ data: [{ id: "openai.gpt-5.6-terra" }] }), {
status: 200,
}),
),
);
// Converse and Mantle both resolve credentials during one listing, so
// the failure lasts until the user signs in again, not for one call.
let signedIn = false;
createAwsCredentialProvider.mockReturnValue(async () => {
if (!signedIn) throw new Error(SSO_TOKEN_EXPIRED);
return { accessKeyId: "key", secretAccessKey: "secret" };
});
const registry = new ProviderRegistry(logger());
registerBedrockProvider(registry, logger());
const credentials = credentialsFor("us-east-2");

try {
await registry.getModelsForProvider("bedrock", credentials);
signedIn = true;
const models = await registry.getModelsForProvider("bedrock", credentials);

expect(models.some((model) => model.id === "openai.gpt-5.6-terra")).toBe(true);
} finally {
vi.unstubAllGlobals();
}
});

it("caches an empty Mantle listing", async () => {
listMantleModels.mockResolvedValueOnce([]);
const registry = new ProviderRegistry(logger());
registerBedrockProvider(registry, logger());
const credentials = credentialsFor("us-east-2");

await registry.getModelsForProvider("bedrock", credentials);
await registry.getModelsForProvider("bedrock", credentials);

expect(listMantleModels).toHaveBeenCalledTimes(1);
});

it("starts independently stale discovery sources concurrently", async () => {
let resolveMantle: ((value: Array<{ id: string }>) => void) | undefined;
let resolveConverse: ((value: { modelSummaries: [] }) => void) | undefined;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,18 @@ import { fileURLToPath } from "node:url";

import { describe, expect, it, vi } from "vitest";

import { createAwsCredentialProvider } from "../../aws-credentials";
import type { AwsCredentials, Logger } from "../../types";
import { listMantleModels } from "../bedrock-mantle-models";

vi.mock("../../aws-credentials", async (importOriginal) => {
const actual = await importOriginal<typeof import("../../aws-credentials")>();
return {
...actual,
createAwsCredentialProvider: vi.fn(actual.createAwsCredentialProvider),
};
});

const credentials: AwsCredentials = {
type: "aws-credentials",
region: "us-east-2",
Expand Down Expand Up @@ -73,7 +82,7 @@ describe("listMantleModels", () => {
return new Response("The request signature we calculated does not match", { status: 403 });
});

await expect(listMantleModels(credentials, log, {}, fetchFunction)).resolves.toEqual([]);
await expect(listMantleModels(credentials, log, {}, fetchFunction)).resolves.toBeNull();
expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("signature"));
});

Expand All @@ -100,26 +109,57 @@ describe("listMantleModels", () => {
expect(permissionLog.warn).not.toHaveBeenCalled();

const authLog = logger();
await listMantleModels(
credentials,
authLog,
{},
vi.fn(
async () =>
new Response("The security token included in the request is expired", { status: 401 }),
await expect(
listMantleModels(
credentials,
authLog,
{},
vi.fn(
async () =>
new Response("The security token included in the request is expired", {
status: 401,
}),
),
),
);
).resolves.toBeNull();
expect(authLog.warn).toHaveBeenCalledWith(expect.stringContaining("security token"));
});

it("returns an empty list for malformed success bodies", async () => {
it("returns null for malformed success bodies", async () => {
const log = logger();
const fetchFunction = vi.fn(async () => new Response("{", { status: 200 }));

await expect(listMantleModels(credentials, log, {}, fetchFunction)).resolves.toEqual([]);
await expect(listMantleModels(credentials, log, {}, fetchFunction)).resolves.toBeNull();
expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("malformed JSON"));
});

it.each([
["an unrecognized 401", new Response("Unauthorized", { status: 401 })],
["a service failure", new Response("Service Unavailable", { status: 503 })],
["an unexpected body", new Response(JSON.stringify({ models: [] }), { status: 200 })],
])("returns null for %s so the listing is retried", async (_case, response) => {
await expect(
listMantleModels(
credentials,
logger(),
{},
vi.fn(async () => response),
),
).resolves.toBeNull();
});

it("returns null without a request when credential resolution fails", async () => {
vi.mocked(createAwsCredentialProvider).mockReturnValueOnce(async () => {
throw new Error("Token is expired. To refresh this SSO session run 'aws sso login'.");
});
const log = logger();
const fetchFunction = vi.fn(async () => new Response("{}", { status: 200 }));

await expect(listMantleModels(credentials, log, {}, fetchFunction)).resolves.toBeNull();
expect(fetchFunction).not.toHaveBeenCalled();
expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("Token is expired"));
});

describe("through a gateway", () => {
it("lists from the gateway base, signed for bedrock-mantle in the credentials' region", async () => {
const fetchFunction = vi.fn(
Expand Down Expand Up @@ -176,21 +216,23 @@ describe("listMantleModels", () => {

it("keeps signature rejections as a generic failure warning", async () => {
const log = logger();
await listMantleModels(
credentials,
log,
{ baseUrl: GATEWAY_MANTLE_BASE },
vi.fn(
async () =>
new Response("The request signature we calculated does not match", { status: 403 }),
await expect(
listMantleModels(
credentials,
log,
{ baseUrl: GATEWAY_MANTLE_BASE },
vi.fn(
async () =>
new Response("The request signature we calculated does not match", { status: 403 }),
),
),
);
).resolves.toBeNull();

expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("failed (403)"));
expect(log.warn).not.toHaveBeenCalledWith(expect.stringContaining("needs"));
});

it("rejects on abort instead of returning an empty list", async () => {
it("rejects on abort instead of returning null", async () => {
const controller = new AbortController();
const fetchFunction = vi.fn(
(_input: RequestInfo | URL, init?: RequestInit) =>
Expand Down
33 changes: 21 additions & 12 deletions packages/ai-provider-bridge/src/providers/bedrock-mantle-models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,23 +66,27 @@ export interface ListMantleModelsOptions {
* listing as a warning that names the missing action.
*/
baseUrl?: string;
/** Aborts the listing; an aborted listing rejects instead of returning `[]`. */
/** Aborts the listing; an aborted listing rejects instead of returning `null`. */
signal?: AbortSignal;
}

/**
* List Mantle model IDs without allowing Mantle discovery to break Bedrock's
* existing Converse catalog. The live service exposes its complete catalog at
* `/v1/models`; inference routing is assigned later from model-family rules.
* Every failure yields `[]` except an abort through `options.signal`, which
* rejects so a cancelled discovery is not held behind a hung listing.
*
* Returns `[]` only when a retry would give the same answer: Mantle is absent
* in the region (404) or the listing is denied for lack of permission. Every
* other failure returns `null`, so a caller can retry rather than cache it.
* An abort through `options.signal` rejects instead, so a cancelled discovery
* is not held behind a hung listing.
*/
export async function listMantleModels(
credentials: AwsCredentials,
logger: Logger,
options: ListMantleModelsOptions = {},
fetchFunction: typeof globalThis.fetch = globalThis.fetch,
): Promise<MantleModelListing[]> {
): Promise<MantleModelListing[] | null> {
const isGateway = options.baseUrl !== undefined;
const baseUrl = (
options.baseUrl ?? `https://bedrock-mantle.${credentials.region}.api.aws/v1`
Expand Down Expand Up @@ -120,7 +124,9 @@ export async function listMantleModels(
if (!response.ok) {
if (response.status === 404) {
logger.debug(`[Bedrock Mantle] Model listing is unavailable at ${url} (404)`);
} else if (
return [];
}
if (
isGateway &&
response.status === 403 &&
!isSignatureRejection(responseBody) &&
Expand All @@ -132,37 +138,40 @@ export async function listMantleModels(
`[Bedrock Mantle] Model listing at ${url} was denied; the role needs ` +
`bedrock-mantle:ListModels to list OpenAI models: ${responseBody}`,
);
} else if (
return [];
}
if (
(response.status === 401 || response.status === 403) &&
!isSignatureRejection(responseBody)
) {
logger.debug(
`[Bedrock Mantle] Model listing denied; bedrock-mantle:ListModels may be missing: ${responseBody}`,
);
} else {
logger.warn(`[Bedrock Mantle] Model listing failed (${response.status}): ${responseBody}`);
// Only a recognized permission denial is a stable answer.
return isAccessDenied(responseBody) ? [] : null;
}
return [];
logger.warn(`[Bedrock Mantle] Model listing failed (${response.status}): ${responseBody}`);
return null;
}

let parsed: unknown;
try {
parsed = JSON.parse(responseBody);
} catch {
logger.warn(`[Bedrock Mantle] Model listing returned malformed JSON: ${responseBody}`);
return [];
return null;
}

const models = parseModels(parsed);
if (!models) {
logger.warn(`[Bedrock Mantle] Model listing returned an unexpected body: ${responseBody}`);
return [];
return null;
}
return [...new Set(models.map((model) => model.id))].map((id) => ({ id }));
} catch (error) {
if (signal?.aborted) throw error;
const message = error instanceof Error ? error.message : String(error);
logger.warn(`[Bedrock Mantle] Model listing failed: ${message}`);
return [];
return null;
}
}
2 changes: 2 additions & 0 deletions packages/ai-provider-bridge/src/providers/bedrock-provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,8 @@ function createBedrockModelFetcher(
}

const mantleListings = await listMantleModels(credentials, logger);
// A failed listing stays uncached so the next listing retries it.
if (mantleListings === null) return [];
const mantleModels: ModelInfo[] = mantleListings.flatMap(({ id }) => {
const capabilityFacts = getBedrockMantleModelCapabilities(id);
if (!capabilityFacts) {
Expand Down
6 changes: 3 additions & 3 deletions packages/ai-provider-bridge/src/providers/connect-provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -584,8 +584,8 @@ async function discoverBedrockGatewayModels(
return [];
},
);
// listMantleModels logs its own failures and returns [] for them; only an
// abort rejects.
// listMantleModels logs its own failures and returns null for them; only
// an abort rejects. A failure lists no Mantle models.
const mantleModels = mantleEnabled
? listMantleModels(
{
Expand All @@ -606,7 +606,7 @@ async function discoverBedrockGatewayModels(
providerId,
integration,
connectUrl,
listings.map(({ id }) => id),
(listings ?? []).map(({ id }) => id),
logger,
),
)
Expand Down
Loading