Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions packages/workshop-backend/__tests__/web-fetch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,32 @@ describe("webFetch document conversion", () => {
// only cares that the call didn't go through toMarkdown.
expect(typeof result.body).toBe("string");
});

it("times out a response whose body stalls after the headers arrive", async () => {
vi.useFakeTimers();
try {
// Headers arrive at once; the body yields one chunk and then never completes, as the
// runtime would present a stalled origin. Only an abort ends the read.
globalThis.fetch = vi.fn(async (_url: string, init: RequestInit) =>
new Response(new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new TextEncoder().encode("partial"));
init.signal?.addEventListener("abort", () =>
controller.error(Object.assign(new Error("aborted"), { name: "AbortError" })));
},
}), { headers: { "content-type": "text/plain" } }),
) as unknown as typeof globalThis.fetch;

// Assert before advancing: the rejection lands inside advanceTimersByTimeAsync().
const settled = expect(
webFetch(makeEnv(), { url: "https://example.com/slow", raw: true }),
).rejects.toThrow(/timed out/);
await vi.advanceTimersByTimeAsync(60_000);
await settled;
} finally {
vi.useRealTimers();
}
});
});

describe("formatWebFetchResult", () => {
Expand Down
48 changes: 27 additions & 21 deletions packages/workshop-backend/src/web-fetch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,12 @@ export async function webFetch(
const timeoutId = setTimeout(() => abortController.abort(), FETCH_TIMEOUT_MS);

let response: Response;
let finalUrl: URL;
let contentType: string;
let bytes: Uint8Array;
let truncated: boolean;
// The timeout covers the body too: a response whose headers arrive promptly but whose body
// then stalls would otherwise hold the tool call open indefinitely.
try {
response = await fetch(parsed.toString(), {
method: "GET",
Expand All @@ -282,6 +288,27 @@ export async function webFetch(
},
signal: abortController.signal,
});

// `response.url` is set by the runtime to the final URL after any redirects. Fall back
// to the original URL if it happens to be empty.
finalUrl = response.url ? new URL(response.url) : parsed;
contentType = response.headers.get("content-type") ?? "";

// Respect the Content-Signal header (https://contentsignals.org/). If the site
// explicitly sets `ai-input=no`, we must not feed its content to the AI agent.
if (contentSignalDenies(response, "ai-input")) {
try {
await response.body?.cancel();
} catch {
// Ignore.
}
throw new Error(
`The site at ${finalUrl} sets Content-Signal: ai-input=no, indicating that ` +
`it does not permit its content to be used as AI input.`,
);
}

({ bytes, truncated } = await readBodyCapped(response, maxBytes));
} catch (err) {
if (
err instanceof Error &&
Expand All @@ -294,27 +321,6 @@ export async function webFetch(
clearTimeout(timeoutId);
}

// `response.url` is set by the runtime to the final URL after any redirects. Fall back
// to the original URL if it happens to be empty.
const finalUrl = response.url ? new URL(response.url) : parsed;
const contentType = response.headers.get("content-type") ?? "";

// Respect the Content-Signal header (https://contentsignals.org/). If the site
// explicitly sets `ai-input=no`, we must not feed its content to the AI agent.
if (contentSignalDenies(response, "ai-input")) {
try {
await response.body?.cancel();
} catch {
// Ignore.
}
throw new Error(
`The site at ${finalUrl} sets Content-Signal: ai-input=no, indicating that ` +
`it does not permit its content to be used as AI input.`,
);
}

const { bytes, truncated } = await readBodyCapped(response, maxBytes);

let body: string;
if (input.raw) {
body = decodeUtf8(bytes);
Expand Down
Loading