From 8c589ad2b20d78bac94730dc994f4d85d765bd87 Mon Sep 17 00:00:00 2001 From: Winston Chang Date: Sun, 8 Nov 2026 00:30:36 -0600 Subject: [PATCH] Don't cache failed Bedrock Mantle listings listMantleModels turned every failure into [], and the direct Bedrock fetcher cached that [] for the one-hour TTL. After an expired AWS session and a re-auth outside the in-app flow (for example `aws sso login`), Bedrock listed Claude models but no OpenAI (Mantle) models for up to an hour. listMantleModels now returns [] only when a retry would give the same answer: a 404 (no Mantle in the region) or a recognized access-denied body. Every other failure (credential resolution, signature or token rejection, unrecognized 401/403, 5xx, malformed or unexpected bodies) returns null. getMantleModels caches arrays and never caches null, so the next listing retries. The Connect gateway consumer treats null as [], as before. --- memory-bank/architecture.md | 6 +- .../__tests__/bedrock-fetcher.test.ts | 48 +++++++++++ .../__tests__/bedrock-mantle-models.test.ts | 82 ++++++++++++++----- .../src/providers/bedrock-mantle-models.ts | 33 +++++--- .../src/providers/bedrock-provider.ts | 2 + .../src/providers/connect-provider.ts | 6 +- 6 files changed, 141 insertions(+), 36 deletions(-) diff --git a/memory-bank/architecture.md b/memory-bank/architecture.md index 205a8a3..72c5eee 100644 --- a/memory-bank/architecture.md +++ b/memory-bank/architecture.md @@ -328,7 +328,11 @@ ordinary Bedrock models through Converse, and OpenAI families through Mantle. Mantle has one independently cached discovery source, `GET https://bedrock-mantle.{region}.api.aws/v1/models`, signed for the `bedrock-mantle` service. A Mantle discovery failure never clears or replaces -the Converse cache. +the Converse cache. `listMantleModels()` returns `[]` only for answers a retry +won't change (a 404, or an access-denied body), and those are cached for the +TTL. Every other failure (expired credentials, signature rejections, 5xx, +unexpected bodies) returns `null`, lists no Mantle models, and is not cached, +so the next listing retries it, for example after `aws sso login`. AWS publishes no FIPS endpoint for Mantle. When the standard AWS FIPS setting is enabled, the shared transport policy suppresses Mantle discovery and the diff --git a/packages/ai-provider-bridge/src/providers/__tests__/bedrock-fetcher.test.ts b/packages/ai-provider-bridge/src/providers/__tests__/bedrock-fetcher.test.ts index 8606266..6742b76 100644 --- a/packages/ai-provider-bridge/src/providers/__tests__/bedrock-fetcher.test.ts +++ b/packages/ai-provider-bridge/src/providers/__tests__/bedrock-fetcher.test.ts @@ -260,6 +260,54 @@ describe("Bedrock provider Mantle aggregation", () => { ); }); + it("re-lists Mantle on the next listing after credential resolution fails", async () => { + const actual = await vi.importActual( + "../bedrock-mantle-models", + ); + listMantleModels.mockImplementation(actual.listMantleModels); + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response(JSON.stringify({ data: [{ id: "openai.gpt-5.6-terra" }] }), { + status: 200, + }), + ), + ); + // Converse and Mantle both resolve credentials during one listing, so + // the failure lasts until the user signs in again, not for one call. + let signedIn = false; + createAwsCredentialProvider.mockReturnValue(async () => { + if (!signedIn) throw new Error(SSO_TOKEN_EXPIRED); + return { accessKeyId: "key", secretAccessKey: "secret" }; + }); + const registry = new ProviderRegistry(logger()); + registerBedrockProvider(registry, logger()); + const credentials = credentialsFor("us-east-2"); + + try { + await registry.getModelsForProvider("bedrock", credentials); + signedIn = true; + const models = await registry.getModelsForProvider("bedrock", credentials); + + expect(models.some((model) => model.id === "openai.gpt-5.6-terra")).toBe(true); + } finally { + vi.unstubAllGlobals(); + } + }); + + it("caches an empty Mantle listing", async () => { + listMantleModels.mockResolvedValueOnce([]); + const registry = new ProviderRegistry(logger()); + registerBedrockProvider(registry, logger()); + const credentials = credentialsFor("us-east-2"); + + await registry.getModelsForProvider("bedrock", credentials); + await registry.getModelsForProvider("bedrock", credentials); + + expect(listMantleModels).toHaveBeenCalledTimes(1); + }); + it("starts independently stale discovery sources concurrently", async () => { let resolveMantle: ((value: Array<{ id: string }>) => void) | undefined; let resolveConverse: ((value: { modelSummaries: [] }) => void) | undefined; diff --git a/packages/ai-provider-bridge/src/providers/__tests__/bedrock-mantle-models.test.ts b/packages/ai-provider-bridge/src/providers/__tests__/bedrock-mantle-models.test.ts index 646443f..8d419bc 100644 --- a/packages/ai-provider-bridge/src/providers/__tests__/bedrock-mantle-models.test.ts +++ b/packages/ai-provider-bridge/src/providers/__tests__/bedrock-mantle-models.test.ts @@ -7,9 +7,18 @@ import { fileURLToPath } from "node:url"; import { describe, expect, it, vi } from "vitest"; +import { createAwsCredentialProvider } from "../../aws-credentials"; import type { AwsCredentials, Logger } from "../../types"; import { listMantleModels } from "../bedrock-mantle-models"; +vi.mock("../../aws-credentials", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createAwsCredentialProvider: vi.fn(actual.createAwsCredentialProvider), + }; +}); + const credentials: AwsCredentials = { type: "aws-credentials", region: "us-east-2", @@ -73,7 +82,7 @@ describe("listMantleModels", () => { return new Response("The request signature we calculated does not match", { status: 403 }); }); - await expect(listMantleModels(credentials, log, {}, fetchFunction)).resolves.toEqual([]); + await expect(listMantleModels(credentials, log, {}, fetchFunction)).resolves.toBeNull(); expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("signature")); }); @@ -100,26 +109,57 @@ describe("listMantleModels", () => { expect(permissionLog.warn).not.toHaveBeenCalled(); const authLog = logger(); - await listMantleModels( - credentials, - authLog, - {}, - vi.fn( - async () => - new Response("The security token included in the request is expired", { status: 401 }), + await expect( + listMantleModels( + credentials, + authLog, + {}, + vi.fn( + async () => + new Response("The security token included in the request is expired", { + status: 401, + }), + ), ), - ); + ).resolves.toBeNull(); expect(authLog.warn).toHaveBeenCalledWith(expect.stringContaining("security token")); }); - it("returns an empty list for malformed success bodies", async () => { + it("returns null for malformed success bodies", async () => { const log = logger(); const fetchFunction = vi.fn(async () => new Response("{", { status: 200 })); - await expect(listMantleModels(credentials, log, {}, fetchFunction)).resolves.toEqual([]); + await expect(listMantleModels(credentials, log, {}, fetchFunction)).resolves.toBeNull(); expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("malformed JSON")); }); + it.each([ + ["an unrecognized 401", new Response("Unauthorized", { status: 401 })], + ["a service failure", new Response("Service Unavailable", { status: 503 })], + ["an unexpected body", new Response(JSON.stringify({ models: [] }), { status: 200 })], + ])("returns null for %s so the listing is retried", async (_case, response) => { + await expect( + listMantleModels( + credentials, + logger(), + {}, + vi.fn(async () => response), + ), + ).resolves.toBeNull(); + }); + + it("returns null without a request when credential resolution fails", async () => { + vi.mocked(createAwsCredentialProvider).mockReturnValueOnce(async () => { + throw new Error("Token is expired. To refresh this SSO session run 'aws sso login'."); + }); + const log = logger(); + const fetchFunction = vi.fn(async () => new Response("{}", { status: 200 })); + + await expect(listMantleModels(credentials, log, {}, fetchFunction)).resolves.toBeNull(); + expect(fetchFunction).not.toHaveBeenCalled(); + expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("Token is expired")); + }); + describe("through a gateway", () => { it("lists from the gateway base, signed for bedrock-mantle in the credentials' region", async () => { const fetchFunction = vi.fn( @@ -176,21 +216,23 @@ describe("listMantleModels", () => { it("keeps signature rejections as a generic failure warning", async () => { const log = logger(); - await listMantleModels( - credentials, - log, - { baseUrl: GATEWAY_MANTLE_BASE }, - vi.fn( - async () => - new Response("The request signature we calculated does not match", { status: 403 }), + await expect( + listMantleModels( + credentials, + log, + { baseUrl: GATEWAY_MANTLE_BASE }, + vi.fn( + async () => + new Response("The request signature we calculated does not match", { status: 403 }), + ), ), - ); + ).resolves.toBeNull(); expect(log.warn).toHaveBeenCalledWith(expect.stringContaining("failed (403)")); expect(log.warn).not.toHaveBeenCalledWith(expect.stringContaining("needs")); }); - it("rejects on abort instead of returning an empty list", async () => { + it("rejects on abort instead of returning null", async () => { const controller = new AbortController(); const fetchFunction = vi.fn( (_input: RequestInfo | URL, init?: RequestInit) => diff --git a/packages/ai-provider-bridge/src/providers/bedrock-mantle-models.ts b/packages/ai-provider-bridge/src/providers/bedrock-mantle-models.ts index 86de40a..392a0f9 100644 --- a/packages/ai-provider-bridge/src/providers/bedrock-mantle-models.ts +++ b/packages/ai-provider-bridge/src/providers/bedrock-mantle-models.ts @@ -66,7 +66,7 @@ export interface ListMantleModelsOptions { * listing as a warning that names the missing action. */ baseUrl?: string; - /** Aborts the listing; an aborted listing rejects instead of returning `[]`. */ + /** Aborts the listing; an aborted listing rejects instead of returning `null`. */ signal?: AbortSignal; } @@ -74,15 +74,19 @@ export interface ListMantleModelsOptions { * List Mantle model IDs without allowing Mantle discovery to break Bedrock's * existing Converse catalog. The live service exposes its complete catalog at * `/v1/models`; inference routing is assigned later from model-family rules. - * Every failure yields `[]` except an abort through `options.signal`, which - * rejects so a cancelled discovery is not held behind a hung listing. + * + * Returns `[]` only when a retry would give the same answer: Mantle is absent + * in the region (404) or the listing is denied for lack of permission. Every + * other failure returns `null`, so a caller can retry rather than cache it. + * An abort through `options.signal` rejects instead, so a cancelled discovery + * is not held behind a hung listing. */ export async function listMantleModels( credentials: AwsCredentials, logger: Logger, options: ListMantleModelsOptions = {}, fetchFunction: typeof globalThis.fetch = globalThis.fetch, -): Promise { +): Promise { const isGateway = options.baseUrl !== undefined; const baseUrl = ( options.baseUrl ?? `https://bedrock-mantle.${credentials.region}.api.aws/v1` @@ -120,7 +124,9 @@ export async function listMantleModels( if (!response.ok) { if (response.status === 404) { logger.debug(`[Bedrock Mantle] Model listing is unavailable at ${url} (404)`); - } else if ( + return []; + } + if ( isGateway && response.status === 403 && !isSignatureRejection(responseBody) && @@ -132,17 +138,20 @@ export async function listMantleModels( `[Bedrock Mantle] Model listing at ${url} was denied; the role needs ` + `bedrock-mantle:ListModels to list OpenAI models: ${responseBody}`, ); - } else if ( + return []; + } + if ( (response.status === 401 || response.status === 403) && !isSignatureRejection(responseBody) ) { logger.debug( `[Bedrock Mantle] Model listing denied; bedrock-mantle:ListModels may be missing: ${responseBody}`, ); - } else { - logger.warn(`[Bedrock Mantle] Model listing failed (${response.status}): ${responseBody}`); + // Only a recognized permission denial is a stable answer. + return isAccessDenied(responseBody) ? [] : null; } - return []; + logger.warn(`[Bedrock Mantle] Model listing failed (${response.status}): ${responseBody}`); + return null; } let parsed: unknown; @@ -150,19 +159,19 @@ export async function listMantleModels( parsed = JSON.parse(responseBody); } catch { logger.warn(`[Bedrock Mantle] Model listing returned malformed JSON: ${responseBody}`); - return []; + return null; } const models = parseModels(parsed); if (!models) { logger.warn(`[Bedrock Mantle] Model listing returned an unexpected body: ${responseBody}`); - return []; + return null; } return [...new Set(models.map((model) => model.id))].map((id) => ({ id })); } catch (error) { if (signal?.aborted) throw error; const message = error instanceof Error ? error.message : String(error); logger.warn(`[Bedrock Mantle] Model listing failed: ${message}`); - return []; + return null; } } diff --git a/packages/ai-provider-bridge/src/providers/bedrock-provider.ts b/packages/ai-provider-bridge/src/providers/bedrock-provider.ts index e497d31..1969cde 100644 --- a/packages/ai-provider-bridge/src/providers/bedrock-provider.ts +++ b/packages/ai-provider-bridge/src/providers/bedrock-provider.ts @@ -305,6 +305,8 @@ function createBedrockModelFetcher( } const mantleListings = await listMantleModels(credentials, logger); + // A failed listing stays uncached so the next listing retries it. + if (mantleListings === null) return []; const mantleModels: ModelInfo[] = mantleListings.flatMap(({ id }) => { const capabilityFacts = getBedrockMantleModelCapabilities(id); if (!capabilityFacts) { diff --git a/packages/ai-provider-bridge/src/providers/connect-provider.ts b/packages/ai-provider-bridge/src/providers/connect-provider.ts index 7475177..dd5d149 100644 --- a/packages/ai-provider-bridge/src/providers/connect-provider.ts +++ b/packages/ai-provider-bridge/src/providers/connect-provider.ts @@ -584,8 +584,8 @@ async function discoverBedrockGatewayModels( return []; }, ); - // listMantleModels logs its own failures and returns [] for them; only an - // abort rejects. + // listMantleModels logs its own failures and returns null for them; only + // an abort rejects. A failure lists no Mantle models. const mantleModels = mantleEnabled ? listMantleModels( { @@ -606,7 +606,7 @@ async function discoverBedrockGatewayModels( providerId, integration, connectUrl, - listings.map(({ id }) => id), + (listings ?? []).map(({ id }) => id), logger, ), )