diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..74f9a2c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,11 @@ +# AGENTS.md + +## Cross-client parity + +This JavaScript client and [dClimate/dclimate-client-py](https://github.com/dClimate/dclimate-client-py) are sibling libraries. Keep their user-visible capabilities and behavior aligned unless a language or runtime difference makes a change inapplicable. + +- For every public API or behavior change—especially STAC/IPFS resolution, dataset loading and selection, metadata, errors, and catalog listing—inspect the corresponding implementation, tests, documentation, and relevant open work in the Python client before finishing. +- Unless the user explicitly limits the task to one repository, treat an applicable sibling-library update as part of the same task. Add equivalent tests and documentation in both projects, using idiomatic APIs for each language rather than mechanically copying implementation details. +- If a change is not applicable to the sibling, or the sibling cannot be updated in the current task, state the reason and leave a concrete follow-up in the handoff or pull-request description. Do not silently allow accidental divergence. +- When reviewing either client, treat undocumented behavioral differences as possible defects and verify whether parity should be restored. + diff --git a/README.md b/README.md index 7289ed4..cadd6f9 100644 --- a/README.md +++ b/README.md @@ -296,6 +296,22 @@ catalog.forEach(({ collection, datasets }) => { }); ``` +### Resolving a CID directly + +Use the public STAC resolver when you need the selected CID and variant without loading the dataset. The API is natively asynchronous and uses the platform's pooled `fetch` implementation. + +```typescript +import { resolveCidFromStacServer } from "@dclimate/dclimate-client-js"; + +const resolved = await resolveCidFromStacServer( + "ecmwf_aifs", + "temperature_forecast", + "single" +); + +console.log(resolved.cid, resolved.variant); +``` + ## Configuration ### Client options @@ -357,6 +373,12 @@ Metric attributes include the gateway URL, store type, and status. The dataset C - `listAvailableDatasets()` - Get the full dataset catalog - `siren` - Namespaced Siren REST API client (getter; throws `SirenNotConfiguredError` unless `siren` is configured) +### STAC utilities + +- `resolveCidFromStacServer(collection, dataset, variant?, serverUrl?)` - Resolve a CID and selected variant without loading the dataset +- `resolveDatasetCidFromStacServer(collection, dataset, variant?, serverUrl?)` - Resolve only the CID string +- `listAvailableDatasetsFromStacServer(serverUrl?)` - List collections, datasets, and variants directly from the paginated STAC API + ### SirenClient (via `client.siren` or standalone) - `getMetricData(query)` - Fetch metric data for a region over a date range diff --git a/package-lock.json b/package-lock.json index cae5a03..f623d59 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dclimate/dclimate-client-js", - "version": "0.6.0", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dclimate/dclimate-client-js", - "version": "0.6.0", + "version": "0.7.0", "license": "MIT", "dependencies": { "@dclimate/jaxray": "^0.7.0", diff --git a/package.json b/package.json index 77b4778..126b92b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dclimate/dclimate-client-js", - "version": "0.6.0", + "version": "0.7.0", "description": "JavaScript client for dClimate datasets using jaxray and IPFS stores", "type": "module", "main": "./dist/node/index.js", diff --git a/src/index.ts b/src/index.ts index 728e0bf..5698898 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,6 +14,9 @@ export { getConcatenableItemsFromStac, listAvailableDatasetsFromStac, listAvailableDatasetsFromStacServer, + resolveCidFromStacServer, + resolveDatasetCidFromStacServer, + DEFAULT_STAC_SERVER_URL, getRootCatalogCid, resolveIpfsUri, type StacCatalog, @@ -25,6 +28,9 @@ export { type StacOrganization, type SpatialExtent, type TemporalExtent, + type StacServerSearchResponse, + type StacServerItem, + type ResolvedCidFromServer, StacCatalogError, StacLoadError, StacResolutionError, diff --git a/src/stac/stac-server.ts b/src/stac/stac-server.ts index b1dbf20..9a9b7e2 100644 --- a/src/stac/stac-server.ts +++ b/src/stac/stac-server.ts @@ -47,6 +47,197 @@ export interface ResolvedCidFromServer { const MAX_STAC_SEARCH_PAGES = 50; +type StacSearchBody = Record | undefined; +type StacSearchHeaders = Record; + +interface StacSearchPage { + features?: T[]; + links?: unknown; +} + +interface StacSearchRequest { + url: string; + method: "GET" | "POST"; + body: StacSearchBody; + headers: StacSearchHeaders; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function stringHeaders(value: unknown): StacSearchHeaders { + if (!isRecord(value)) return {}; + const entries = Object.entries(value); + if (!entries.every(([, headerValue]) => typeof headerValue === "string")) { + return {}; + } + return Object.fromEntries(entries) as StacSearchHeaders; +} + +function stableJson(value: unknown): string { + if (value === undefined) return "undefined"; + if (Array.isArray(value)) { + return `[${value.map((entry) => stableJson(entry)).join(",")}]`; + } + if (isRecord(value)) { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +function normalizedOrigin(url: string): string { + const parsed = new URL(url); + const protocol = parsed.protocol.toLowerCase(); + const hostname = parsed.hostname.replace(/\.+$/, "").toLowerCase(); + const port = + parsed.port || (protocol === "http:" ? "80" : protocol === "https:" ? "443" : ""); + return `${protocol}//${hostname}:${port}`; +} + +function nextSearchRequest( + serverUrl: string, + currentUrl: string, + originalBody: Record, + page: StacSearchPage +): StacSearchRequest | undefined { + const links = Array.isArray(page.links) ? page.links : []; + const nextLink = links.find( + (link) => isRecord(link) && link.rel === "next" && Boolean(link.href) + ); + if (!isRecord(nextLink)) return undefined; + if (typeof nextLink.href !== "string") { + throw new Error("STAC pagination link href must be a string"); + } + + const parsedNextUrl = new URL(nextLink.href, currentUrl); + const nextUrl = parsedNextUrl.toString(); + if ( + normalizedOrigin(nextUrl) !== normalizedOrigin(serverUrl) || + parsedNextUrl.username !== "" || + parsedNextUrl.password !== "" + ) { + throw new Error( + `STAC pagination link must use the configured server origin ${normalizedOrigin(serverUrl)}: ${nextUrl}` + ); + } + + const method = String(nextLink.method ?? "GET").toUpperCase(); + if (method !== "GET" && method !== "POST") { + throw new Error( + `STAC pagination link uses an unsupported method: '${method}'` + ); + } + + // Continuation headers may contain credentials. Forward them only after + // validating an encrypted same-origin link; plaintext endpoints still + // paginate, but without server-supplied headers. + const headers = + parsedNextUrl.protocol.toLowerCase() === "https:" + ? stringHeaders(nextLink.headers) + : {}; + + let body: StacSearchBody; + if (isRecord(nextLink.body)) { + body = nextLink.merge + ? { ...originalBody, ...nextLink.body } + : nextLink.body; + } else if (nextLink.merge) { + body = { ...originalBody }; + } else { + body = undefined; + } + + return { url: nextUrl, method, body, headers }; +} + +function requestUrl(request: StacSearchRequest): string { + if (request.method !== "GET" || !request.body) return request.url; + const url = new URL(request.url); + for (const [key, value] of Object.entries(request.body)) { + url.searchParams.delete(key); + if (Array.isArray(value)) { + for (const item of value) url.searchParams.append(key, String(item)); + } else if (value !== undefined && value !== null) { + url.searchParams.set(key, String(value)); + } + } + return url.toString(); +} + +async function fetchSearchPage( + request: StacSearchRequest +): Promise> { + const response = await fetch(requestUrl(request), { + method: request.method, + redirect: "manual", + headers: + request.method === "POST" + ? { "Content-Type": "application/json", ...request.headers } + : request.headers, + ...(request.method === "POST" + ? { body: JSON.stringify(request.body ?? null) } + : {}), + }); + if (!response.ok) { + const text = await response.text(); + throw new Error(`STAC server error ${response.status}: ${text}`); + } + const page = (await response.json()) as StacSearchPage; + if (!isRecord(page)) { + throw new Error("STAC server returned an invalid search response"); + } + if (page.features !== undefined && !Array.isArray(page.features)) { + throw new Error("STAC server returned invalid search features"); + } + return page; +} + +async function* searchPages( + serverUrl: string, + originalBody: Record +): AsyncGenerator> { + let request: StacSearchRequest = { + url: `${serverUrl.replace(/\/+$/, "")}/search`, + method: "POST", + body: originalBody, + headers: {}, + }; + const seen = new Set(); + + for (let pageNumber = 0; pageNumber < MAX_STAC_SEARCH_PAGES; pageNumber++) { + const pageKey = [ + request.method, + request.url, + stableJson(request.body), + stableJson(request.headers), + ].join("\n"); + if (seen.has(pageKey)) return; + seen.add(pageKey); + + const page = await fetchSearchPage(request); + yield page; + + const features = Array.isArray(page.features) ? page.features : []; + if (features.length === 0) return; + const nextRequest = nextSearchRequest( + serverUrl, + request.url, + originalBody, + page + ); + if (!nextRequest) return; + request = nextRequest; + } + + throw new Error( + `STAC server pagination exceeded ${MAX_STAC_SEARCH_PAGES} pages; results truncated` + ); +} + function datasetIdFromItemId( itemId: string, collection: string @@ -98,76 +289,9 @@ export async function resolveCidFromStacServer( collections: [collection], }; - const searchUrl = `${serverUrl}/search`; - // The URL that produced the current response; relative `next` hrefs resolve - // against this, not the server root. - let currentUrl = searchUrl; - let response = await fetch(searchUrl, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(body), - }); - const features: StacServerItem[] = []; - for (let page = 0; page < MAX_STAC_SEARCH_PAGES; page++) { - if (!response.ok) { - const text = await response.text(); - throw new Error(`STAC server error ${response.status}: ${text}`); - } - - const data: StacServerSearchResponse = await response.json(); - features.push(...(data.features || [])); - - const nextLink = data.links?.find( - (link) => - typeof link === "object" && - link !== null && - link.rel === "next" && - typeof link.href === "string" - ); - if (!nextLink) { - break; - } - if (page === MAX_STAC_SEARCH_PAGES - 1) { - // A next link remains but we've hit the page cap. Surface the truncation - // instead of returning a silently partial result set, so callers can - // distinguish "not found" from "not yet fetched" (and fall back to the - // full IPFS catalog walk). - throw new Error( - `STAC server pagination for '${collection}' exceeded ${MAX_STAC_SEARCH_PAGES} pages; results truncated` - ); - } - - // STAC API pagination: next links may be plain GET hrefs, or POST links - // carrying a token body (the dClimate server's shape) that must be - // re-POSTed. Per the STAC API link contract, a link may also carry - // `headers` (e.g. a header-based cursor), and `merge` governs both body and - // headers: merge them onto the original request when true, otherwise the - // link's values replace them. Resolve relative hrefs against the URL that - // produced this response (the /search endpoint), not the server root, so a - // bare `?token=…` targets /search. - const nextUrl = new URL(nextLink.href, currentUrl).toString(); - const baseHeaders: Record = { - "Content-Type": "application/json", - }; - const nextHeaders = nextLink.merge - ? { ...baseHeaders, ...(nextLink.headers ?? {}) } - : nextLink.headers ?? baseHeaders; - if ((nextLink.method ?? "GET").toUpperCase() === "POST") { - const nextBody = nextLink.merge - ? { ...body, ...(nextLink.body ?? {}) } - : nextLink.body ?? body; - response = await fetch(nextUrl, { - method: "POST", - // Content-Type is a floor for the JSON body even if a replacing - // (non-merge) link omits it. - headers: { "Content-Type": "application/json", ...nextHeaders }, - body: JSON.stringify(nextBody), - }); - } else { - response = await fetch(nextUrl, { headers: nextHeaders }); - } - currentUrl = nextUrl; + for await (const page of searchPages(serverUrl, body)) { + features.push(...(page.features ?? [])); } // Filter to the exact dataset. A prefix match would conflate datasets such @@ -262,10 +386,6 @@ interface StacServerSearchFeature { properties: Record; } -interface StacServerSearchPage { - features: StacServerSearchFeature[]; -} - function stripIpfsScheme(cid: string | undefined): string | undefined { if (!cid) return undefined; return cid.startsWith("ipfs://") ? cid.replace(/^ipfs:\/\//, "") : cid; @@ -290,20 +410,26 @@ function stripIpfsScheme(cid: string | undefined): string | undefined { * - Category (historical/forecast) isn't populated here — the IPFS walker * pulls it from `dclimate:collections:` on the org link, which * has no STAC API equivalent. - * - The fixed `limit: 1000` covers today's catalog (~45 items) by a wide - * margin. If the catalog grows past that, switch to following the - * STAC `next` link instead of a single request. + * - Search pagination is bounded and repeated requests are detected to avoid + * looping on malformed `next` links. */ export async function listAvailableDatasetsFromStacServer( serverUrl: string = DEFAULT_STAC_SERVER_URL ): Promise { - const [collectionsResp, searchResp] = await Promise.all([ - fetch(`${serverUrl}/collections`), - fetch(`${serverUrl}/search`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ limit: 1000 }), + const searchFeaturesPromise = (async () => { + const features: StacServerSearchFeature[] = []; + for await (const page of searchPages(serverUrl, { + limit: 100, + })) { + features.push(...(page.features ?? [])); + } + return features; + })(); + const [collectionsResp, searchFeatures] = await Promise.all([ + fetch(`${serverUrl.replace(/\/+$/, "")}/collections`, { + redirect: "manual", }), + searchFeaturesPromise, ]); if (!collectionsResp.ok) { @@ -312,15 +438,7 @@ export async function listAvailableDatasetsFromStacServer( `STAC server /collections error ${collectionsResp.status}: ${text}` ); } - if (!searchResp.ok) { - const text = await searchResp.text(); - throw new Error( - `STAC server /search error ${searchResp.status}: ${text}` - ); - } - const collectionsBody = (await collectionsResp.json()) as StacServerCollectionsResponse; - const searchBody = (await searchResp.json()) as StacServerSearchPage; interface CollectionAccumulator { title?: string; @@ -343,7 +461,7 @@ export async function listAvailableDatasetsFromStacServer( }); } - for (const feature of searchBody.features ?? []) { + for (const feature of searchFeatures) { const collectionId = feature.collection ?? (feature.id.includes("-") ? feature.id.split("-")[0] : undefined); diff --git a/tests/review-fixes/stac-server-pagination.test.ts b/tests/review-fixes/stac-server-pagination.test.ts index cd974c2..ebb27ad 100644 --- a/tests/review-fixes/stac-server-pagination.test.ts +++ b/tests/review-fixes/stac-server-pagination.test.ts @@ -112,20 +112,26 @@ describe("resolveCidFromStacServer pagination", () => { // Every page advertises another next link, so the walk can never terminate // naturally. Rather than silently truncating (and reporting the target as // missing), it must throw so callers can fall back to the full catalog. - const fetchMock = vi.fn( - async () => - ({ + let requestNumber = 0; + const fetchMock = vi.fn(async () => { + requestNumber += 1; + return { ok: true, status: 200, statusText: "OK", json: async () => ({ type: "FeatureCollection", features: [feature(0)], - links: [{ rel: "next", href: `${serverUrl}/search?page=next` }], + links: [ + { + rel: "next", + href: `${serverUrl}/search?page=${requestNumber + 1}`, + }, + ], }), text: async () => "", - }) as Response, - ); + } as Response; + }); vi.stubGlobal("fetch", fetchMock); await expect( diff --git a/tests/review-fixes/stac-server-parity.test.ts b/tests/review-fixes/stac-server-parity.test.ts new file mode 100644 index 0000000..3ab5906 --- /dev/null +++ b/tests/review-fixes/stac-server-parity.test.ts @@ -0,0 +1,176 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + listAvailableDatasetsFromStacServer, + resolveCidFromStacServer, +} from "../../src/index.js"; + +const collection = "example_collection"; +const dataset = "temperature_mean"; + +function feature(name: string, cid: string) { + return { + type: "Feature" as const, + id: `${collection}-${name}-default`, + collection, + properties: { + "dclimate:dataset_id": name, + "dclimate:variant": "default", + "dclimate:latest_dataset_cid": `ipfs://${cid}`, + }, + assets: { data: { href: `ipfs://${cid}` } }, + }; +} + +function response(body: unknown, status = 200): Response { + return { + ok: status >= 200 && status < 300, + status, + json: async () => body, + text: async () => "redirected", + } as Response; +} + +describe("STAC server parity hardening", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it.each([ + "https://attacker.example/collect", + "http://stac.example/search?page=2", + ])("rejects an untrusted pagination link %s", async (nextHref) => { + const fetchMock = vi.fn(async () => + response({ + features: [feature("other_dataset", "bafy-other")], + links: [{ rel: "next", href: nextHref }], + }) + ); + vi.stubGlobal("fetch", fetchMock); + + await expect( + resolveCidFromStacServer( + collection, + dataset, + undefined, + "https://stac.example" + ) + ).rejects.toThrow(/configured server origin/); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("rejects unsupported pagination methods", async () => { + const fetchMock = vi.fn(async () => + response({ + features: [feature("other_dataset", "bafy-other")], + links: [ + { + rel: "next", + href: "/search?page=2", + method: "DELETE", + }, + ], + }) + ); + vi.stubGlobal("fetch", fetchMock); + + await expect( + resolveCidFromStacServer( + collection, + dataset, + undefined, + "https://stac.example" + ) + ).rejects.toThrow(/unsupported method: 'DELETE'/); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("does not forward linked headers over plaintext HTTP", async () => { + const seenHeaders: Array> = []; + const fetchMock = vi.fn( + async (_input: string | URL | Request, init?: RequestInit) => { + seenHeaders.push((init?.headers ?? {}) as Record); + return response( + seenHeaders.length === 1 + ? { + features: [feature("other_dataset", "bafy-other")], + links: [ + { + rel: "next", + href: "/search?page=2", + headers: { Authorization: "Bearer continuation" }, + }, + ], + } + : { features: [feature(dataset, "bafy-target")] } + ); + } + ); + vi.stubGlobal("fetch", fetchMock); + + await expect( + resolveCidFromStacServer( + collection, + dataset, + undefined, + "http://stac.example" + ) + ).resolves.toMatchObject({ cid: "bafy-target" }); + expect(seenHeaders[1]?.Authorization).toBeUndefined(); + }); + + it("disables automatic redirect following", async () => { + const fetchMock = vi.fn(async () => response({}, 302)); + vi.stubGlobal("fetch", fetchMock); + + await expect( + resolveCidFromStacServer( + collection, + dataset, + undefined, + "https://stac.example" + ) + ).rejects.toThrow(/302/); + expect(fetchMock).toHaveBeenCalledWith( + "https://stac.example/search", + expect.objectContaining({ redirect: "manual" }) + ); + }); + + it("uses paginated search results when listing datasets", async () => { + const fetchMock = vi.fn( + async (input: string | URL | Request) => { + const url = input instanceof Request ? input.url : input.toString(); + if (url.endsWith("/collections")) { + return response({ collections: [{ id: collection }] }); + } + if (url.endsWith("?page=2")) { + return response({ features: [feature(dataset, "bafy-target")] }); + } + return response({ + features: [feature("other_dataset", "bafy-other")], + links: [{ rel: "next", href: "/search?page=2" }], + }); + } + ); + vi.stubGlobal("fetch", fetchMock); + + const catalog = await listAvailableDatasetsFromStacServer( + "https://stac.example" + ); + + expect(catalog[0]?.datasets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + dataset, + variants: [ + expect.objectContaining({ + variant: "default", + cid: "bafy-target", + }), + ], + }), + ]) + ); + expect(fetchMock).toHaveBeenCalledTimes(3); + }); +});