From da9055c012755dc9259fff1be3cb24225a0c1463 Mon Sep 17 00:00:00 2001 From: eloramirez1356 Date: Wed, 5 Aug 2026 16:06:32 -0500 Subject: [PATCH 1/5] feat: discover dataset version history through STAC --- README.md | 28 +++++ src/client.ts | 152 ++++++++++++++++++-------- src/errors.ts | 8 ++ src/index.ts | 12 +++ src/stac/index.ts | 1 + src/stac/stac-catalog.ts | 37 ++++++- src/stac/stac-server.ts | 17 ++- src/types.ts | 17 +++ src/versions/index.ts | 14 +++ src/versions/types.ts | 44 ++++++++ src/versions/version-client.ts | 61 +++++++++++ tests/stac-version-discovery.test.ts | 155 +++++++++++++++++++++++++++ tests/version-client.test.ts | 85 +++++++++++++++ 13 files changed, 580 insertions(+), 51 deletions(-) create mode 100644 src/versions/index.ts create mode 100644 src/versions/types.ts create mode 100644 src/versions/version-client.ts create mode 100644 tests/stac-version-discovery.test.ts create mode 100644 tests/version-client.test.ts diff --git a/README.md b/README.md index 7289ed4..f82d4bf 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,34 @@ const slice = await point.timeRange({ console.log(await slice.toRecords("precipitation")); ``` +### Dataset version history + +For datasets that advertise version history in STAC, the client follows the +item's `dclimate:versions_api` URL. STAC therefore selects Hydrogen, Tritium, +or a future version service without a client-side dataset routing table. + +```typescript +const versions = await client.listDatasetVersions({ + collection: "noaa_aigfs", + dataset: "wind_u_forecast", + variant: "operational", + filters: { + anchored: true, + isCitable: true, + versionLabel: "2026-08", + }, +}); + +for (const release of versions.versions) { + console.log(release.versionLabel, release.cid); +} +``` + +The low-level `listVersionsFromUrl`, `getExactVersionFromUrl`, and +`getCitationFromUrl` helpers are also exported for applications that already +have the complete URLs. Items backed by hard-coded CIDs may not advertise a +version-history service. + ### Siren REST API usage Use Siren methods by configuring `siren` in the client options. diff --git a/src/client.ts b/src/client.ts index 45c1ad8..8549bba 100644 --- a/src/client.ts +++ b/src/client.ts @@ -4,12 +4,17 @@ import { ClientOptions, DatasetMetadata, DatasetRequest, + DatasetVersionsRequest, GeoSelectionOptions, LoadDatasetOptions, } from "./types.js"; import { DEFAULT_IPFS_GATEWAY } from "./constants.js"; import { openDatasetFromCid, IpfsElements } from "./ipfs/open-dataset.js"; -import { DatasetNotFoundError, SirenNotConfiguredError } from "./errors.js"; +import { + DatasetNotFoundError, + SirenNotConfiguredError, + VersionHistoryUnavailableError, +} from "./errors.js"; import { normalizeSegment } from "./utils.js"; import { concatenateVariants, type VariantToLoad } from "./actions/concatenate-variants.js"; @@ -20,6 +25,7 @@ import { listAvailableDatasetsFromStac, type StacCatalog, type ConcatenableStacItem, + type ResolvedDatasetFromStac, } from "./stac/index.js"; import { DatasetCatalog } from "./stac/stac-catalog.js"; import { @@ -28,6 +34,8 @@ import { DEFAULT_STAC_SERVER_URL, } from "./stac/stac-server.js"; import { SirenClient } from "./siren/siren-client.js"; +import { listVersionsFromUrl } from "./versions/version-client.js"; +import type { DatasetVersionListing } from "./versions/types.js"; function normalizeZarrGroup(group?: string): string | undefined { const normalized = group?.replace(/^\/+/, "").replace(/\/+$/, ""); @@ -89,6 +97,74 @@ export class DClimateClient { return this.listAvailableDatasets(); } + private async resolveDatasetDetails( + request: DatasetRequest, + gatewayUrl: string = this.gatewayUrl + ): Promise { + if (!request.collection || !request.dataset) { + throw new DatasetNotFoundError( + "Collection and dataset names must be provided." + ); + } + + const collection = + request.organization && + !request.collection.startsWith(`${request.organization}_`) + ? `${request.organization}_${request.collection}` + : request.collection; + + if (this.stacServerUrl) { + try { + const resolved = await resolveCidFromStacServer( + collection, + request.dataset, + request.variant, + this.stacServerUrl + ); + return { + ...resolved, + organizationId: + request.organization ?? + (resolved.collectionId.includes("_") + ? resolved.collectionId.split("_")[0] + : undefined), + }; + } catch { + // Fall through to the IPFS-hosted STAC catalog. + } + } + + const catalog = await this.getStacCatalog(gatewayUrl); + return resolveDatasetFromStac( + catalog, + collection, + request.dataset, + request.variant, + request.organization + ); + } + + async listDatasetVersions({ + collection, + dataset, + variant, + organization, + filters, + }: DatasetVersionsRequest): Promise { + const resolved = await this.resolveDatasetDetails({ + collection, + dataset, + variant, + organization, + }); + if (!resolved.versionsApi) { + throw new VersionHistoryUnavailableError( + `Version history is not available for ${collection}/${dataset}/${resolved.variant}.` + ); + } + return listVersionsFromUrl(resolved.versionsApi, filters); + } + /** * Access the Siren REST API client (metric data, regions, metrics). * Namespaced so Siren stays separate from the core dataset API: @@ -151,7 +227,7 @@ export class DClimateClient { const normalizedDatasetKey = normalizeSegment(request.dataset); const autoConcatenate = options.autoConcatenate; - let resolvedOrganization = request.organization; + const resolvedOrganization = request.organization; let resolvedCollection = request.collection; if ( @@ -209,50 +285,20 @@ export class DClimateClient { } // Fall back to single variant loading - let cid: string | null = null; - let metadataDataset = request.dataset; - let metadataCollection = resolvedCollection || request.collection; - let metadataVariant = request.variant ?? ""; - let metadataOrganization = resolvedOrganization; - - // Try STAC server first (faster, avoids loading IPFS catalog) - if (this.stacServerUrl && resolvedCollection) { - try { - const serverResolved = await resolveCidFromStacServer( - resolvedCollection, - request.dataset, - request.variant, - this.stacServerUrl - ); - cid = serverResolved.cid; - metadataCollection = serverResolved.collectionId; - metadataVariant = serverResolved.variant || ""; - metadataDataset = serverResolved.dataset; - } catch { - // Fall back to IPFS catalog - } - } - - // Fallback: Use STAC catalog resolution from IPFS - if (!cid) { - const catalog = await this.getStacCatalog(gatewayUrl); - - const resolved = resolveDatasetFromStac( - catalog, - resolvedCollection || request.collection || "", - request.dataset, - request.variant, - resolvedOrganization - ); - - // Update metadata with resolved values - cid = resolved.cid; - metadataCollection = resolved.collectionId; - metadataVariant = resolved.variant || ""; - resolvedOrganization = resolved.organizationId ?? resolvedOrganization; - metadataOrganization = resolved.organizationId ?? resolvedOrganization; - metadataDataset = request.dataset; - } + const resolved = await this.resolveDatasetDetails( + { + ...request, + collection: resolvedCollection || request.collection, + organization: resolvedOrganization, + }, + gatewayUrl + ); + const cid = resolved.cid; + const metadataDataset = resolved.dataset; + const metadataCollection = resolved.collectionId; + const metadataVariant = resolved.variant || ""; + const metadataOrganization = + resolved.organizationId ?? resolvedOrganization; // Build path from resolved names const pathParts = [metadataCollection, metadataDataset, metadataVariant].filter(Boolean); @@ -274,6 +320,20 @@ export class DClimateClient { cid: cid, source: "stac", fetchedAt: new Date(), + ...(resolved.versionsApi ? { versionsApi: resolved.versionsApi } : {}), + ...(resolved.provenanceApi + ? { provenanceApi: resolved.provenanceApi } + : {}), + ...(resolved.citationApi ? { citationApi: resolved.citationApi } : {}), + ...(resolved.streamId ? { streamId: resolved.streamId } : {}), + ...(resolved.commitId ? { commitId: resolved.commitId } : {}), + ...(resolved.versionLabel ? { versionLabel: resolved.versionLabel } : {}), + ...(resolved.isCitable !== undefined + ? { isCitable: resolved.isCitable } + : {}), + ...(resolved.retentionClass + ? { retentionClass: resolved.retentionClass } + : {}), ...(zarrGroup ? { zarrGroup } : {}), }; diff --git a/src/errors.ts b/src/errors.ts index 8c353be..4d8b1d6 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -14,3 +14,11 @@ export class NoDataFoundError extends DClimateClientError {} export class SirenApiError extends DClimateClientError {} export class SirenNotConfiguredError extends DClimateClientError {} + +export class VersionHistoryUnavailableError extends DClimateClientError {} + +export class VersionApiError extends DClimateClientError { + constructor(message: string, public status?: number) { + super(message); + } +} diff --git a/src/index.ts b/src/index.ts index 728e0bf..bfa064b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,6 +22,7 @@ export { type StacCatalogOptions, type ConcatenableStacItem, type ResolvedDatasetFromStac, + type StacReleaseMetadata, type StacOrganization, type SpatialExtent, type TemporalExtent, @@ -29,6 +30,17 @@ export { StacLoadError, StacResolutionError, } from "./stac/index.js"; +export { + listVersionsFromUrl, + getExactVersionFromUrl, + getCitationFromUrl, + type FetchImplementation, + type CitationInfo, + type DatasetVersion, + type DatasetVersionListing, + type VerificationInfo, + type VersionFilters, +} from "./versions/index.js"; export { SirenClient, type SirenApiKeyAuth, diff --git a/src/stac/index.ts b/src/stac/index.ts index 3835b53..f9cc120 100644 --- a/src/stac/index.ts +++ b/src/stac/index.ts @@ -14,6 +14,7 @@ export { type StacCatalogOptions, type ConcatenableStacItem, type ResolvedDatasetFromStac, + type StacReleaseMetadata, type StacOrganization, type SpatialExtent, type TemporalExtent, diff --git a/src/stac/stac-catalog.ts b/src/stac/stac-catalog.ts index 0bd53df..bbef1e8 100644 --- a/src/stac/stac-catalog.ts +++ b/src/stac/stac-catalog.ts @@ -140,6 +140,40 @@ export function getStringProperty( return typeof value === "string" ? value : undefined; } +function getBooleanProperty( + properties: Record | undefined, + key: string +): boolean | undefined { + const value = properties?.[key]; + return typeof value === "boolean" ? value : undefined; +} + +export interface StacReleaseMetadata { + versionsApi?: string; + provenanceApi?: string; + citationApi?: string; + streamId?: string; + commitId?: string; + versionLabel?: string; + isCitable?: boolean; + retentionClass?: string; +} + +export function getStacReleaseMetadata( + properties: Record | undefined +): StacReleaseMetadata { + return { + versionsApi: getStringProperty(properties, "dclimate:versions_api"), + provenanceApi: getStringProperty(properties, "dclimate:provenance_api"), + citationApi: getStringProperty(properties, "dclimate:citation_api"), + streamId: getStringProperty(properties, "dclimate:stream_id"), + commitId: getStringProperty(properties, "dclimate:commit_id"), + versionLabel: getStringProperty(properties, "dclimate:version_label"), + isCitable: getBooleanProperty(properties, "dclimate:is_citable"), + retentionClass: getStringProperty(properties, "dclimate:retention_class"), + }; +} + function getNumberProperty( properties: Record | undefined, key: string @@ -168,7 +202,7 @@ export interface StacOrganization { catalog: StacCatalog; } -export interface ResolvedDatasetFromStac { +export interface ResolvedDatasetFromStac extends StacReleaseMetadata { cid: string; collectionId: string; organizationId?: string; @@ -691,6 +725,7 @@ export function resolveDatasetFromStac( organizationId, dataset, variant: resolvedVariant || "default", + ...getStacReleaseMetadata(selectedItem.properties), }; } diff --git a/src/stac/stac-server.ts b/src/stac/stac-server.ts index b1dbf20..e70dcfa 100644 --- a/src/stac/stac-server.ts +++ b/src/stac/stac-server.ts @@ -10,8 +10,9 @@ import type { CatalogDataset, DatasetCatalog, DatasetVariantConfig, + StacReleaseMetadata, } from "./stac-catalog.js"; -import { getStringProperty } from "./stac-catalog.js"; +import { getStacReleaseMetadata, getStringProperty } from "./stac-catalog.js"; export const DEFAULT_STAC_SERVER_URL = "https://api.stac.dclimate.net"; @@ -38,7 +39,7 @@ export interface StacServerItem { assets: Record; } -export interface ResolvedCidFromServer { +export interface ResolvedCidFromServer extends StacReleaseMetadata { cid: string; collectionId: string; dataset: string; @@ -215,17 +216,25 @@ export async function resolveCidFromStacServer( // Extract CID from asset const href = selectedItem.assets?.data?.href || ""; - if (!href) { + const advertisedCid = getStringProperty( + selectedItem.properties, + "dclimate:latest_dataset_cid" + ); + const rawCid = href || advertisedCid || ""; + if (!rawCid) { throw new Error(`Item '${selectedItem.id}' has no data asset`); } - const cid = href.startsWith("ipfs://") ? href.replace("ipfs://", "") : href; + const cid = rawCid.startsWith("ipfs://") + ? rawCid.replace("ipfs://", "") + : rawCid; return { cid, collectionId: collection, dataset, variant: resolvedVariant, + ...getStacReleaseMetadata(selectedItem.properties), }; } diff --git a/src/types.ts b/src/types.ts index 1a49a6b..dde6ce6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,6 @@ import type { IPFSELEMENTS_INTERFACE } from "@dclimate/jaxray"; import type { SirenOptions } from "./siren/types.js"; +import type { VersionFilters } from "./versions/types.js"; export type IpfsElements = IPFSELEMENTS_INTERFACE; @@ -98,6 +99,14 @@ export interface DatasetMetadata { */ source: "stac" | "stac_concatenated" | "direct_cid"; fetchedAt: Date; + versionsApi?: string; + provenanceApi?: string; + citationApi?: string; + streamId?: string; + commitId?: string; + versionLabel?: string; + isCitable?: boolean; + retentionClass?: string; } export interface DatasetRequest { @@ -108,6 +117,14 @@ export interface DatasetRequest { cid?: string; } +export interface DatasetVersionsRequest { + collection: string; + dataset: string; + variant?: string; + organization?: string; + filters?: VersionFilters; +} + export interface DataArrayObject { data: unknown; dims: string[]; diff --git a/src/versions/index.ts b/src/versions/index.ts new file mode 100644 index 0000000..f00fbb9 --- /dev/null +++ b/src/versions/index.ts @@ -0,0 +1,14 @@ +export { + listVersionsFromUrl, + getExactVersionFromUrl, + getCitationFromUrl, + type FetchImplementation, +} from "./version-client.js"; + +export type { + CitationInfo, + DatasetVersion, + DatasetVersionListing, + VerificationInfo, + VersionFilters, +} from "./types.js"; diff --git a/src/versions/types.ts b/src/versions/types.ts new file mode 100644 index 0000000..75270ba --- /dev/null +++ b/src/versions/types.ts @@ -0,0 +1,44 @@ +export interface VerificationInfo { + anchorStatus?: string; + [key: string]: unknown; +} + +export interface DatasetVersion { + dataset: string; + cid: string; + oldCid?: string; + timestamp?: number; + streamId?: string; + commitId?: string; + controllerDid?: string; + publishedAt?: string; + versionLabel?: string; + releaseClass?: string; + isCitable?: boolean; + retentionClass?: string; + verification?: VerificationInfo; +} + +export interface DatasetVersionListing { + dataset: string; + streamId?: string; + versions: DatasetVersion[]; +} + +export interface CitationInfo { + dataset: string; + streamId?: string; + commitId?: string; + cid: string; + publishedAt?: string; + versionLabel?: string; + isCitable?: boolean; + retentionClass?: string; + citation: string; +} + +export interface VersionFilters { + anchored?: boolean; + isCitable?: boolean; + versionLabel?: string; +} diff --git a/src/versions/version-client.ts b/src/versions/version-client.ts new file mode 100644 index 0000000..8626369 --- /dev/null +++ b/src/versions/version-client.ts @@ -0,0 +1,61 @@ +import { VersionApiError } from "../errors.js"; +import type { + CitationInfo, + DatasetVersion, + DatasetVersionListing, + VersionFilters, +} from "./types.js"; + +export type FetchImplementation = typeof fetch; + +async function requestJson( + url: string, + fetchImpl: FetchImplementation +): Promise { + const response = await fetchImpl(url, { + headers: { Accept: "application/json" }, + }); + if (!response.ok) { + const text = await response.text(); + throw new VersionApiError( + `Version API request failed (${response.status}) for ${url}: ${text}`, + response.status + ); + } + return response.json() as Promise; +} + +export async function listVersionsFromUrl( + versionsUrl: string, + filters: VersionFilters = {}, + fetchImpl: FetchImplementation = fetch +): Promise { + const url = new URL(versionsUrl); + if (filters.anchored !== undefined) { + url.searchParams.set("anchored", String(filters.anchored)); + } + if (filters.isCitable !== undefined) { + url.searchParams.set("isCitable", String(filters.isCitable)); + } + if (filters.versionLabel !== undefined) { + url.searchParams.set("versionLabel", filters.versionLabel); + } + return requestJson(url.toString(), fetchImpl); +} + +export async function getExactVersionFromUrl( + versionsUrl: string, + commitId: string, + fetchImpl: FetchImplementation = fetch +): Promise { + const url = new URL(versionsUrl); + url.pathname = `${url.pathname.replace(/\/$/, "")}/${encodeURIComponent(commitId)}`; + return requestJson(url.toString(), fetchImpl); +} + +export async function getCitationFromUrl( + citationUrl: string, + fetchImpl: FetchImplementation = fetch +): Promise { + return requestJson(citationUrl, fetchImpl); +} diff --git a/tests/stac-version-discovery.test.ts b/tests/stac-version-discovery.test.ts new file mode 100644 index 0000000..c6b2ea2 --- /dev/null +++ b/tests/stac-version-discovery.test.ts @@ -0,0 +1,155 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { DClimateClient } from "../src/client.js"; +import { VersionHistoryUnavailableError } from "../src/errors.js"; +import { + resolveDatasetFromStac, + type StacCatalog, +} from "../src/stac/stac-catalog.js"; +import { resolveCidFromStacServer } from "../src/stac/stac-server.js"; + +const properties = { + "dclimate:dataset_id": "wind_u_forecast", + "dclimate:variant": "operational", + "dclimate:versions_api": + "https://hydrogen.dclimate.net/api/datasets/aigfs-wind-u/versions", + "dclimate:provenance_api": + "https://hydrogen.dclimate.net/api/datasets/aigfs-wind-u/versions/commit-1", + "dclimate:citation_api": + "https://hydrogen.dclimate.net/api/datasets/aigfs-wind-u/citation?commitId=commit-1", + "dclimate:stream_id": "stream-1", + "dclimate:commit_id": "commit-1", + "dclimate:version_label": "2026-08", + "dclimate:is_citable": true, + "dclimate:retention_class": "permanent", +}; + +const item = { + type: "Feature" as const, + stac_version: "1.0.0", + id: "noaa_aigfs-wind_u_forecast-operational", + collection: "noaa_aigfs", + properties, + geometry: null, + assets: { data: { href: "ipfs://bafy-current" } }, + links: [], +}; + +function response(body: unknown): Response { + return { + ok: true, + status: 200, + json: async () => body, + text: async () => "", + } as Response; +} + +describe("STAC release discovery", () => { + afterEach(() => vi.unstubAllGlobals()); + + it("extracts release metadata from the hosted STAC server", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => + response({ type: "FeatureCollection", features: [item], links: [] }) + ) + ); + + const resolved = await resolveCidFromStacServer( + "noaa_aigfs", + "wind_u_forecast", + "operational", + "https://stac.test" + ); + + expect(resolved).toMatchObject({ + cid: "bafy-current", + versionsApi: properties["dclimate:versions_api"], + commitId: "commit-1", + isCitable: true, + retentionClass: "permanent", + }); + }); + + it("extracts equivalent metadata from the IPFS STAC representation", () => { + const catalog: StacCatalog = { + type: "Catalog", + stac_version: "1.0.0", + id: "root", + links: [], + collections: [ + { + type: "Collection", + stac_version: "1.0.0", + id: "noaa_aigfs", + organizationId: "noaa", + links: [], + items: [item], + }, + ], + }; + + const resolved = resolveDatasetFromStac( + catalog, + "noaa_aigfs", + "wind_u_forecast", + "operational", + "noaa" + ); + + expect(resolved.versionsApi).toBe(properties["dclimate:versions_api"]); + expect(resolved.provenanceApi).toBe(properties["dclimate:provenance_api"]); + expect(resolved.citationApi).toBe(properties["dclimate:citation_api"]); + }); + + it("lists versions using the full URL advertised by STAC", async () => { + const fetchMock = vi.fn(async (input: string | URL | Request) => { + const url = input.toString(); + if (url === "https://stac.test/search") { + return response({ type: "FeatureCollection", features: [item], links: [] }); + } + if (url.startsWith(properties["dclimate:versions_api"])) { + return response({ dataset: "aigfs-wind-u", versions: [] }); + } + throw new Error(`Unexpected URL: ${url}`); + }); + vi.stubGlobal("fetch", fetchMock); + const client = new DClimateClient({ stacServerUrl: "https://stac.test" }); + + const result = await client.listDatasetVersions({ + collection: "noaa_aigfs", + dataset: "wind_u_forecast", + variant: "operational", + filters: { anchored: true }, + }); + + expect(result.dataset).toBe("aigfs-wind-u"); + expect(fetchMock.mock.calls[1][0].toString()).toBe( + `${properties["dclimate:versions_api"]}?anchored=true` + ); + }); + + it("reports when a STAC item has no version-history capability", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => + response({ + type: "FeatureCollection", + features: [{ ...item, properties: { + "dclimate:dataset_id": "wind_u_forecast", + "dclimate:variant": "operational", + } }], + links: [], + }) + ) + ); + const client = new DClimateClient({ stacServerUrl: "https://stac.test" }); + + await expect( + client.listDatasetVersions({ + collection: "noaa_aigfs", + dataset: "wind_u_forecast", + variant: "operational", + }) + ).rejects.toBeInstanceOf(VersionHistoryUnavailableError); + }); +}); diff --git a/tests/version-client.test.ts b/tests/version-client.test.ts new file mode 100644 index 0000000..e832699 --- /dev/null +++ b/tests/version-client.test.ts @@ -0,0 +1,85 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + getCitationFromUrl, + getExactVersionFromUrl, + listVersionsFromUrl, +} from "../src/versions/version-client.js"; +import { VersionApiError } from "../src/errors.js"; + +function jsonResponse(body: unknown, status = 200): Response { + return { + ok: status >= 200 && status < 300, + status, + json: async () => body, + text: async () => JSON.stringify(body), + } as Response; +} + +describe("STAC-discovered version API URLs", () => { + afterEach(() => vi.restoreAllMocks()); + + it("preserves the Tritium dataset slug and appends filters", async () => { + const fetchMock = vi.fn(async () => + jsonResponse({ dataset: "era5-temperature-2m-finalized", versions: [] }) + ); + + const result = await listVersionsFromUrl( + "https://tritium.dclimate.net/api/datasets/era5-temperature-2m-finalized/versions", + { anchored: true, isCitable: false, versionLabel: "2026-08" }, + fetchMock + ); + + expect(result.dataset).toBe("era5-temperature-2m-finalized"); + const requested = new URL(fetchMock.mock.calls[0][0].toString()); + expect(requested.hostname).toBe("tritium.dclimate.net"); + expect(requested.pathname).toBe( + "/api/datasets/era5-temperature-2m-finalized/versions" + ); + expect(Object.fromEntries(requested.searchParams)).toEqual({ + anchored: "true", + isCitable: "false", + versionLabel: "2026-08", + }); + }); + + it("encodes an exact commit without rebuilding the dataset URL", async () => { + const fetchMock = vi.fn(async () => + jsonResponse({ dataset: "aigfs-wind-u", cid: "bafy-version" }) + ); + + await getExactVersionFromUrl( + "https://hydrogen.dclimate.net/api/datasets/aigfs-wind-u/versions", + "commit/one", + fetchMock + ); + + expect(fetchMock.mock.calls[0][0].toString()).toBe( + "https://hydrogen.dclimate.net/api/datasets/aigfs-wind-u/versions/commit%2Fone" + ); + }); + + it("preserves an existing citation commit query", async () => { + const citationUrl = + "https://hydrogen.dclimate.net/api/datasets/aigfs-wind-u/citation?commitId=commit-1"; + const fetchMock = vi.fn(async () => + jsonResponse({ + dataset: "aigfs-wind-u", + cid: "bafy-version", + citation: "citation text", + }) + ); + + const result = await getCitationFromUrl(citationUrl, fetchMock); + + expect(result.citation).toBe("citation text"); + expect(fetchMock.mock.calls[0][0]).toBe(citationUrl); + }); + + it("reports the response status and URL for service errors", async () => { + const fetchMock = vi.fn(async () => jsonResponse({ detail: "missing" }, 404)); + + await expect( + listVersionsFromUrl("https://hydrogen.test/datasets/missing/versions", {}, fetchMock) + ).rejects.toMatchObject>({ status: 404 }); + }); +}); From c1c3611c8565f22757bcaafebb7bc26bbeb01add Mon Sep 17 00:00:00 2001 From: eloramirez1356 Date: Wed, 5 Aug 2026 16:44:59 -0500 Subject: [PATCH 2/5] fix: align grouped Zarr resolution with STAC --- src/client.ts | 16 ++++++--- src/ipfs/open-dataset.ts | 23 ++++++++++-- src/stac/stac-catalog.ts | 15 ++++++++ src/stac/stac-server.ts | 16 +++++++-- tests/fetch-dataset-cid.test.ts | 53 ++++++++++++++++++++++++++++ tests/open-dataset.test.ts | 24 +++++++++++-- tests/stac-version-discovery.test.ts | 43 +++++++++++++++++++++- 7 files changed, 176 insertions(+), 14 deletions(-) diff --git a/src/client.ts b/src/client.ts index 8549bba..cdb4fa2 100644 --- a/src/client.ts +++ b/src/client.ts @@ -192,7 +192,7 @@ export class DClimateClient { }): Promise<[GeoTemporalDataset, DatasetMetadata] | [Dataset, DatasetMetadata]> { const gatewayUrl = options.gatewayUrl ?? this.gatewayUrl; const ipfsElements = this.resolveIpfsElements(options, gatewayUrl); - const zarrGroup = normalizeZarrGroup(options.zarrGroup); + const explicitZarrGroup = normalizeZarrGroup(options.zarrGroup); if (request.cid) { @@ -200,9 +200,12 @@ export class DClimateClient { const dataset = await openDatasetFromCid(request.cid, { gatewayUrl, ipfsElements, - zarrGroup, + zarrGroup: explicitZarrGroup, shardReadMode: options.shardReadMode, }); + const openedZarrGroup = + explicitZarrGroup ?? + normalizeZarrGroup(dataset.attrs?._ipfs_zarr_group as string | undefined); const metadata: DatasetMetadata = { dataset: "", @@ -213,7 +216,7 @@ export class DClimateClient { path: "", cid: request.cid, fetchedAt: new Date(), - ...(zarrGroup ? { zarrGroup } : {}), + ...(openedZarrGroup ? { zarrGroup: openedZarrGroup } : {}), }; if (options.returnJaxrayDataset) { return [dataset, metadata]; @@ -299,6 +302,7 @@ export class DClimateClient { const metadataVariant = resolved.variant || ""; const metadataOrganization = resolved.organizationId ?? resolvedOrganization; + const zarrGroup = explicitZarrGroup ?? normalizeZarrGroup(resolved.zarrGroup); // Build path from resolved names const pathParts = [metadataCollection, metadataDataset, metadataVariant].filter(Boolean); @@ -376,7 +380,7 @@ export class DClimateClient { } const gatewayUrl = options.gatewayUrl ?? this.gatewayUrl; const ipfsElements = this.resolveIpfsElements(options, gatewayUrl); - const zarrGroup = normalizeZarrGroup(options.zarrGroup); + const explicitZarrGroup = normalizeZarrGroup(options.zarrGroup); // Order by concatPriority so metadata (concatenatedVariants, cid) // reflects the same order the data is concatenated in. @@ -387,6 +391,8 @@ export class DClimateClient { // Load all variants in parallel const variantsToLoad: VariantToLoad[] = await Promise.all( orderedVariants.map(async (variantConfig) => { + const zarrGroup = + explicitZarrGroup ?? normalizeZarrGroup(variantConfig.zarrGroup); // Load the dataset using the CID from STAC const dataset = await openDatasetFromCid(variantConfig.cid, { gatewayUrl, @@ -417,7 +423,7 @@ export class DClimateClient { cid: variantsToLoad[0].dataset.attrs._zarr_cid as string || "concatenated", source: "stac_concatenated", fetchedAt: new Date(), - ...(zarrGroup ? { zarrGroup } : {}), + ...(explicitZarrGroup ? { zarrGroup: explicitZarrGroup } : {}), }; if (options.returnJaxrayDataset) { diff --git a/src/ipfs/open-dataset.ts b/src/ipfs/open-dataset.ts index 25f67da..368fd49 100644 --- a/src/ipfs/open-dataset.ts +++ b/src/ipfs/open-dataset.ts @@ -26,6 +26,13 @@ export function normalizeZarrGroup(group?: string): string | undefined { return normalized || undefined; } +function requiresExplicitZarrGroup(error: unknown): boolean { + return ( + error instanceof Error && + error.message.includes("require an explicit group option") + ); +} + export async function openDatasetFromCid( cid: string, options: OpenDatasetOptions = {} @@ -85,9 +92,19 @@ export async function openDatasetFromCid( } ); - const dataset = zarrGroup - ? await Dataset.open_zarr(store, { group: zarrGroup }) - : await Dataset.open_zarr(store); + let dataset: Dataset; + if (zarrGroup) { + dataset = await Dataset.open_zarr(store, { group: zarrGroup }); + } else { + try { + dataset = await Dataset.open_zarr(store); + } catch (error) { + if (!requiresExplicitZarrGroup(error)) throw error; + dataset = await Dataset.open_zarr(store, { group: "0" }); + dataset.attrs._ipfs_zarr_group = "0"; + } + } + if (zarrGroup) dataset.attrs._ipfs_zarr_group = zarrGroup; status = "ok"; return dataset; } catch (error) { diff --git a/src/stac/stac-catalog.ts b/src/stac/stac-catalog.ts index bbef1e8..7f589f8 100644 --- a/src/stac/stac-catalog.ts +++ b/src/stac/stac-catalog.ts @@ -85,6 +85,7 @@ export interface StacAsset { type?: string; title?: string; roles?: string[]; + [key: string]: unknown; } export interface StacItem { @@ -193,6 +194,7 @@ export interface ConcatenableStacItem { cid: string; concatPriority: number; concatDimension: string; + zarrGroup?: string; } export interface StacOrganization { @@ -208,6 +210,17 @@ export interface ResolvedDatasetFromStac extends StacReleaseMetadata { organizationId?: string; dataset: string; variant: string; + zarrGroup?: string; +} + +export function getStacZarrGroup( + asset: Record | undefined, + properties: Record | undefined +): string | undefined { + return ( + getStringProperty(asset, "dclimate:zarr_group") ?? + getStringProperty(properties, "dclimate:default_zarr_group") + ); } // ============================================================================ @@ -725,6 +738,7 @@ export function resolveDatasetFromStac( organizationId, dataset, variant: resolvedVariant || "default", + zarrGroup: getStacZarrGroup(selectedItem.assets.data, selectedItem.properties), ...getStacReleaseMetadata(selectedItem.properties), }; } @@ -813,6 +827,7 @@ export function getConcatenableItemsFromStac( cid, concatPriority: priority, concatDimension: dimension, + zarrGroup: getStacZarrGroup(dataAsset, item.properties), }); } diff --git a/src/stac/stac-server.ts b/src/stac/stac-server.ts index e70dcfa..1b33ad5 100644 --- a/src/stac/stac-server.ts +++ b/src/stac/stac-server.ts @@ -12,7 +12,11 @@ import type { DatasetVariantConfig, StacReleaseMetadata, } from "./stac-catalog.js"; -import { getStacReleaseMetadata, getStringProperty } from "./stac-catalog.js"; +import { + getStacReleaseMetadata, + getStacZarrGroup, + getStringProperty, +} from "./stac-catalog.js"; export const DEFAULT_STAC_SERVER_URL = "https://api.stac.dclimate.net"; @@ -36,7 +40,10 @@ export interface StacServerItem { id: string; collection?: string; properties: Record; - assets: Record; + assets: Record< + string, + { href: string; type?: string; title?: string; [key: string]: unknown } + >; } export interface ResolvedCidFromServer extends StacReleaseMetadata { @@ -44,6 +51,7 @@ export interface ResolvedCidFromServer extends StacReleaseMetadata { collectionId: string; dataset: string; variant: string; + zarrGroup?: string; } const MAX_STAC_SEARCH_PAGES = 50; @@ -215,7 +223,8 @@ export async function resolveCidFromStacServer( } // Extract CID from asset - const href = selectedItem.assets?.data?.href || ""; + const dataAsset = selectedItem.assets?.data; + const href = dataAsset?.href || ""; const advertisedCid = getStringProperty( selectedItem.properties, "dclimate:latest_dataset_cid" @@ -234,6 +243,7 @@ export async function resolveCidFromStacServer( collectionId: collection, dataset, variant: resolvedVariant, + zarrGroup: getStacZarrGroup(dataAsset, selectedItem.properties), ...getStacReleaseMetadata(selectedItem.properties), }; } diff --git a/tests/fetch-dataset-cid.test.ts b/tests/fetch-dataset-cid.test.ts index e7ef049..52bc32d 100644 --- a/tests/fetch-dataset-cid.test.ts +++ b/tests/fetch-dataset-cid.test.ts @@ -24,6 +24,59 @@ describe("loadDataset CID resolution", () => { }); describe("STAC catalog resolution", () => { + it("uses the STAC data asset group unless the caller overrides it", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ + ok: true, + status: 200, + text: async () => "", + json: async () => ({ + type: "FeatureCollection", + links: [], + features: [ + { + type: "Feature", + id: "test_grouped-pyramid-default", + collection: "test_grouped", + properties: { + "dclimate:dataset_id": "pyramid", + "dclimate:variant": "default", + "dclimate:default_zarr_group": "1", + }, + assets: { + data: { + href: "ipfs://bafygrouped", + "dclimate:zarr_group": "/0/", + }, + }, + }, + ], + }), + })) + ); + const client = new DClimateClient({ stacServerUrl: "https://stac.test" }); + + const [, discoveredMetadata] = await client.loadDataset({ + request: { collection: "test_grouped", dataset: "pyramid" }, + }); + expect(openDatasetFromCidMock).toHaveBeenLastCalledWith( + "bafygrouped", + expect.objectContaining({ zarrGroup: "0" }) + ); + expect(discoveredMetadata.zarrGroup).toBe("0"); + + const [, overriddenMetadata] = await client.loadDataset({ + request: { collection: "test_grouped", dataset: "pyramid" }, + options: { zarrGroup: "/2/" }, + }); + expect(openDatasetFromCidMock).toHaveBeenLastCalledWith( + "bafygrouped", + expect.objectContaining({ zarrGroup: "2" }) + ); + expect(overriddenMetadata.zarrGroup).toBe("2"); + }); + it("resolves CID from STAC for known dataset", async () => { const client = new DClimateClient(); await client.loadDataset({ diff --git a/tests/open-dataset.test.ts b/tests/open-dataset.test.ts index 3b697b7..be5de16 100644 --- a/tests/open-dataset.test.ts +++ b/tests/open-dataset.test.ts @@ -24,7 +24,7 @@ describe("openDatasetFromCid", () => { it("opens an IPFS store and Zarr dataset with default gateway telemetry enabled", async () => { const store = { kind: "store" }; - const dataset = { kind: "dataset" }; + const dataset = { kind: "dataset", attrs: {} }; openIpfsStoreMock.mockResolvedValue({ store }); openZarrMock.mockResolvedValue(dataset); @@ -39,13 +39,33 @@ describe("openDatasetFromCid", () => { it("passes an explicit Zarr group to jaxray", async () => { const store = { kind: "store" }; - const dataset = { kind: "dataset" }; + const dataset = { kind: "dataset", attrs: {} }; openIpfsStoreMock.mockResolvedValue({ store }); openZarrMock.mockResolvedValue(dataset); await expect(openDatasetFromCid("bafygrouped", { zarrGroup: "/0/" })).resolves.toBe(dataset); expect(openZarrMock).toHaveBeenCalledWith(store, { group: "0" }); + expect(dataset.attrs).toEqual({ _ipfs_zarr_group: "0" }); + }); + + it("safely retries group zero when jaxray reports an ambiguous grouped root", async () => { + const store = { kind: "grouped-store" }; + const dataset = { kind: "dataset", attrs: {} as Record }; + openIpfsStoreMock.mockResolvedValue({ store }); + openZarrMock + .mockRejectedValueOnce( + new Error( + "ZarrBackend.open: grouped Zarr stores with multiple top-level groups require an explicit group option." + ) + ) + .mockResolvedValueOnce(dataset); + + await expect(openDatasetFromCid("bafygrouped")).resolves.toBe(dataset); + + expect(openZarrMock).toHaveBeenNthCalledWith(1, store); + expect(openZarrMock).toHaveBeenNthCalledWith(2, store, { group: "0" }); + expect(dataset.attrs._ipfs_zarr_group).toBe("0"); }); it("uses caller supplied IPFS elements", async () => { diff --git a/tests/stac-version-discovery.test.ts b/tests/stac-version-discovery.test.ts index c6b2ea2..691cfe5 100644 --- a/tests/stac-version-discovery.test.ts +++ b/tests/stac-version-discovery.test.ts @@ -21,6 +21,7 @@ const properties = { "dclimate:version_label": "2026-08", "dclimate:is_citable": true, "dclimate:retention_class": "permanent", + "dclimate:default_zarr_group": "1", }; const item = { @@ -30,7 +31,12 @@ const item = { collection: "noaa_aigfs", properties, geometry: null, - assets: { data: { href: "ipfs://bafy-current" } }, + assets: { + data: { + href: "ipfs://bafy-current", + "dclimate:zarr_group": "/0/", + }, + }, links: [], }; @@ -67,6 +73,7 @@ describe("STAC release discovery", () => { commitId: "commit-1", isCitable: true, retentionClass: "permanent", + zarrGroup: "/0/", }); }); @@ -99,6 +106,40 @@ describe("STAC release discovery", () => { expect(resolved.versionsApi).toBe(properties["dclimate:versions_api"]); expect(resolved.provenanceApi).toBe(properties["dclimate:provenance_api"]); expect(resolved.citationApi).toBe(properties["dclimate:citation_api"]); + expect(resolved.zarrGroup).toBe("/0/"); + }); + + it("falls back to the item default when the data asset has no group", () => { + const unannotatedAssetItem = { + ...item, + assets: { data: { href: "ipfs://bafy-current" } }, + }; + const catalog: StacCatalog = { + type: "Catalog", + stac_version: "1.0.0", + id: "root", + links: [], + collections: [ + { + type: "Collection", + stac_version: "1.0.0", + id: "noaa_aigfs", + organizationId: "noaa", + links: [], + items: [unannotatedAssetItem], + }, + ], + }; + + expect( + resolveDatasetFromStac( + catalog, + "noaa_aigfs", + "wind_u_forecast", + "operational", + "noaa" + ).zarrGroup + ).toBe("1"); }); it("lists versions using the full URL advertised by STAC", async () => { From ec3f5e74fc0d38fcdc58b37452dfd717c8080692 Mon Sep 17 00:00:00 2001 From: eloramirez1356 Date: Wed, 5 Aug 2026 17:16:17 -0500 Subject: [PATCH 3/5] feat: add STAC-aware exact version lookup --- README.md | 9 ++++ package-lock.json | 4 +- package.json | 2 +- src/client.ts | 32 ++++++++++++- src/types.ts | 8 ++++ tests/stac-version-discovery.test.ts | 70 ++++++++++++++++++++++++++++ 6 files changed, 120 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f82d4bf..b68c0c3 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,15 @@ const versions = await client.listDatasetVersions({ for (const release of versions.versions) { console.log(release.versionLabel, release.cid); } + +const exactVersion = await client.getDatasetVersion({ + collection: "noaa_aigfs", + dataset: "wind_u_forecast", + variant: "operational", + commitId: "commit-id", +}); + +console.log(exactVersion.cid); ``` The low-level `listVersionsFromUrl`, `getExactVersionFromUrl`, and diff --git a/package-lock.json b/package-lock.json index cae5a03..f623d59 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dclimate/dclimate-client-js", - "version": "0.6.0", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dclimate/dclimate-client-js", - "version": "0.6.0", + "version": "0.7.0", "license": "MIT", "dependencies": { "@dclimate/jaxray": "^0.7.0", diff --git a/package.json b/package.json index 77b4778..126b92b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dclimate/dclimate-client-js", - "version": "0.6.0", + "version": "0.7.0", "description": "JavaScript client for dClimate datasets using jaxray and IPFS stores", "type": "module", "main": "./dist/node/index.js", diff --git a/src/client.ts b/src/client.ts index cdb4fa2..2755e55 100644 --- a/src/client.ts +++ b/src/client.ts @@ -4,6 +4,7 @@ import { ClientOptions, DatasetMetadata, DatasetRequest, + DatasetVersionRequest, DatasetVersionsRequest, GeoSelectionOptions, LoadDatasetOptions, @@ -34,8 +35,14 @@ import { DEFAULT_STAC_SERVER_URL, } from "./stac/stac-server.js"; import { SirenClient } from "./siren/siren-client.js"; -import { listVersionsFromUrl } from "./versions/version-client.js"; -import type { DatasetVersionListing } from "./versions/types.js"; +import { + getExactVersionFromUrl, + listVersionsFromUrl, +} from "./versions/version-client.js"; +import type { + DatasetVersion, + DatasetVersionListing, +} from "./versions/types.js"; function normalizeZarrGroup(group?: string): string | undefined { const normalized = group?.replace(/^\/+/, "").replace(/\/+$/, ""); @@ -165,6 +172,27 @@ export class DClimateClient { return listVersionsFromUrl(resolved.versionsApi, filters); } + async getDatasetVersion({ + collection, + dataset, + commitId, + variant, + organization, + }: DatasetVersionRequest): Promise { + const resolved = await this.resolveDatasetDetails({ + collection, + dataset, + variant, + organization, + }); + if (!resolved.versionsApi) { + throw new VersionHistoryUnavailableError( + `Version history is not available for ${collection}/${dataset}/${resolved.variant}.` + ); + } + return getExactVersionFromUrl(resolved.versionsApi, commitId); + } + /** * Access the Siren REST API client (metric data, regions, metrics). * Namespaced so Siren stays separate from the core dataset API: diff --git a/src/types.ts b/src/types.ts index dde6ce6..36e2501 100644 --- a/src/types.ts +++ b/src/types.ts @@ -125,6 +125,14 @@ export interface DatasetVersionsRequest { filters?: VersionFilters; } +export interface DatasetVersionRequest { + collection: string; + dataset: string; + commitId: string; + variant?: string; + organization?: string; +} + export interface DataArrayObject { data: unknown; dims: string[]; diff --git a/tests/stac-version-discovery.test.ts b/tests/stac-version-discovery.test.ts index 691cfe5..113bd59 100644 --- a/tests/stac-version-discovery.test.ts +++ b/tests/stac-version-discovery.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { DClimateClient } from "../src/client.js"; import { VersionHistoryUnavailableError } from "../src/errors.js"; +import { VersionApiError } from "../src/errors.js"; import { resolveDatasetFromStac, type StacCatalog, @@ -169,6 +170,66 @@ describe("STAC release discovery", () => { ); }); + it.each([ + "https://hydrogen.dclimate.net/api/datasets/aigfs-wind-u/versions", + "https://tritium.dclimate.net/api/datasets/aigfs-wind-u/versions", + ])("gets an exact version through the STAC-directed service %s", async (versionsApi) => { + const commitId = "commit/with spaces?and=query#fragment"; + const routedItem = { + ...item, + properties: { ...properties, "dclimate:versions_api": versionsApi }, + }; + const fetchMock = vi.fn(async (input: string | URL | Request) => { + const url = input.toString(); + if (url === "https://stac.test/search") { + return response({ + type: "FeatureCollection", + features: [routedItem], + links: [], + }); + } + return response({ dataset: "aigfs-wind-u", cid: "bafy-exact" }); + }); + vi.stubGlobal("fetch", fetchMock); + const client = new DClimateClient({ stacServerUrl: "https://stac.test" }); + + const result = await client.getDatasetVersion({ + collection: "noaa_aigfs", + dataset: "wind_u_forecast", + variant: "operational", + commitId, + }); + + expect(result.cid).toBe("bafy-exact"); + expect(fetchMock.mock.calls[1][0].toString()).toBe( + `${versionsApi}/commit%2Fwith%20spaces%3Fand%3Dquery%23fragment` + ); + }); + + it("propagates exact-version service errors", async () => { + const fetchMock = vi.fn(async (input: string | URL | Request) => { + if (input.toString() === "https://stac.test/search") { + return response({ type: "FeatureCollection", features: [item], links: [] }); + } + return { + ...response({ detail: "unavailable" }), + ok: false, + status: 503, + } as Response; + }); + vi.stubGlobal("fetch", fetchMock); + const client = new DClimateClient({ stacServerUrl: "https://stac.test" }); + + await expect( + client.getDatasetVersion({ + collection: "noaa_aigfs", + dataset: "wind_u_forecast", + variant: "operational", + commitId: "commit-1", + }) + ).rejects.toMatchObject>({ status: 503 }); + }); + it("reports when a STAC item has no version-history capability", async () => { vi.stubGlobal( "fetch", @@ -192,5 +253,14 @@ describe("STAC release discovery", () => { variant: "operational", }) ).rejects.toBeInstanceOf(VersionHistoryUnavailableError); + + await expect( + client.getDatasetVersion({ + collection: "noaa_aigfs", + dataset: "wind_u_forecast", + variant: "operational", + commitId: "commit-1", + }) + ).rejects.toBeInstanceOf(VersionHistoryUnavailableError); }); }); From 7ccfcfa5d7a7779ee21da29c613f3b05d126e382 Mon Sep 17 00:00:00 2001 From: eloramirez1356 Date: Thu, 6 Aug 2026 18:08:42 -0500 Subject: [PATCH 4/5] feat: require explicit multiresolution selection --- README.md | 34 ++++++++ src/client.ts | 81 ++++++++++++++++++- src/errors.ts | 14 ++++ src/index.ts | 1 + src/ipfs/open-dataset.ts | 6 +- src/stac/index.ts | 1 + src/stac/stac-catalog.ts | 59 ++++++++++---- src/stac/stac-server.ts | 14 +++- src/types.ts | 2 + tests/fetch-dataset-cid.test.ts | 70 ++++++++++++++-- tests/geotemporal-dataset.test.ts | 8 +- tests/open-dataset.test.ts | 9 ++- .../concat-items-resolved-id.test.ts | 3 + tests/stac-version-discovery.test.ts | 49 +++++++++-- 14 files changed, 308 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index b68c0c3..c85cbc5 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,40 @@ The low-level `listVersionsFromUrl`, `getExactVersionFromUrl`, and have the complete URLs. Items backed by hard-coded CIDs may not advertise a version-history service. +### Multiresolution datasets + +Pyramidal datasets require an explicit resolution (recommended) or raw Zarr +group. The client reports the available resolutions instead of silently +choosing between different precision, chunking, and fetching strategies. + +```typescript +const [data, metadata] = await client.loadDataset({ + request: { + collection: "copernicus_clms", + dataset: "fpar", + resolution: "2km", + }, +}); + +console.log(metadata.resolution, metadata.zarrGroup); +``` + +FPAR advertises `500m` → group `"0"`, `2km` → group `"1"`, and `8km` → +group `"2"`. Change `request.resolution` to select any of those levels. A raw +`options.zarrGroup` is supported for storage-aware callers, but must not be +combined with `request.resolution`. + +During migration, STAC may also contain a legacy `assets.data` alias for the +500 m asset. The client ignores it when building the three choices, and it is +neither a fourth resolution nor a default. Consumers relying on `assets.data` +or implicit group `"0"` should migrate before the alias is removed in a future +breaking release. + +Direct CID requests have no STAC resolution mapping and must use +`options.zarrGroup` when the store contains multiple groups; a human-readable +resolution is rejected. STAC's internal `metadataGroup` controls only catalog +metadata extraction and never selects a client resolution. + ### Siren REST API usage Use Siren methods by configuring `siren` in the client options. diff --git a/src/client.ts b/src/client.ts index 2755e55..b138dc9 100644 --- a/src/client.ts +++ b/src/client.ts @@ -13,6 +13,9 @@ import { DEFAULT_IPFS_GATEWAY } from "./constants.js"; import { openDatasetFromCid, IpfsElements } from "./ipfs/open-dataset.js"; import { DatasetNotFoundError, + ConflictingResolutionSelectionError, + MultiresolutionSelectionRequiredError, + ResolutionNotAvailableError, SirenNotConfiguredError, VersionHistoryUnavailableError, } from "./errors.js"; @@ -27,6 +30,7 @@ import { type StacCatalog, type ConcatenableStacItem, type ResolvedDatasetFromStac, + type StacZarrResolution, } from "./stac/index.js"; import { DatasetCatalog } from "./stac/stac-catalog.js"; import { @@ -49,6 +53,54 @@ function normalizeZarrGroup(group?: string): string | undefined { return normalized || undefined; } +function resolveZarrSelection( + choices: StacZarrResolution[], + resolution?: string, + zarrGroup?: string +): { zarrGroup?: string; resolution?: string } { + if (resolution && zarrGroup) { + throw new ConflictingResolutionSelectionError( + "Pass either request.resolution or options.zarrGroup, not both." + ); + } + if (resolution) { + const match = choices.find((choice) => choice.resolution === resolution); + if (!match) { + const available = choices.map((choice) => choice.resolution); + throw new ResolutionNotAvailableError( + `Resolution '${resolution}' is not available.` + + (available.length ? ` Choose one of: ${available.join(", ")}.` : "") + ); + } + return { zarrGroup: match.group, resolution: match.resolution }; + } + if (zarrGroup) { + const normalized = normalizeZarrGroup(zarrGroup); + const match = choices.find((choice) => choice.group === normalized); + if (choices.length && !match) { + throw new ResolutionNotAvailableError( + `Zarr group '${normalized}' is not available. Choose one of: ${choices + .map((choice) => choice.group) + .join(", ")}.` + ); + } + return { zarrGroup: normalized, resolution: match?.resolution }; + } + if (choices.length > 1) { + throw new MultiresolutionSelectionRequiredError( + `This dataset has multiple resolutions; pass request.resolution or options.zarrGroup. Available resolutions: ${choices + .map((choice) => choice.resolution) + .join(", ")}.`, + choices.map((choice) => choice.resolution), + choices.map((choice) => choice.group) + ); + } + if (choices.length === 1) { + return { zarrGroup: choices[0].group, resolution: choices[0].resolution }; + } + return {}; +} + export class DClimateClient { private gatewayUrl: string; private stacServerUrl: string | null; @@ -224,6 +276,16 @@ export class DClimateClient { if (request.cid) { + if (request.resolution && explicitZarrGroup) { + throw new ConflictingResolutionSelectionError( + "Pass either request.resolution or options.zarrGroup, not both." + ); + } + if (request.resolution) { + throw new ResolutionNotAvailableError( + "request.resolution requires STAC metadata; pass options.zarrGroup for a direct CID." + ); + } // Direct CID provided - bypass catalog const dataset = await openDatasetFromCid(request.cid, { gatewayUrl, @@ -330,7 +392,12 @@ export class DClimateClient { const metadataVariant = resolved.variant || ""; const metadataOrganization = resolved.organizationId ?? resolvedOrganization; - const zarrGroup = explicitZarrGroup ?? normalizeZarrGroup(resolved.zarrGroup); + const zarrSelection = resolveZarrSelection( + resolved.zarrResolutions, + request.resolution, + explicitZarrGroup + ); + const zarrGroup = zarrSelection.zarrGroup; // Build path from resolved names const pathParts = [metadataCollection, metadataDataset, metadataVariant].filter(Boolean); @@ -367,6 +434,9 @@ export class DClimateClient { ? { retentionClass: resolved.retentionClass } : {}), ...(zarrGroup ? { zarrGroup } : {}), + ...(zarrSelection.resolution + ? { resolution: zarrSelection.resolution } + : {}), }; if (!metadata.organization && metadata.collection?.includes("_")) { @@ -419,8 +489,12 @@ export class DClimateClient { // Load all variants in parallel const variantsToLoad: VariantToLoad[] = await Promise.all( orderedVariants.map(async (variantConfig) => { - const zarrGroup = - explicitZarrGroup ?? normalizeZarrGroup(variantConfig.zarrGroup); + const zarrSelection = resolveZarrSelection( + variantConfig.zarrResolutions, + request.resolution, + explicitZarrGroup + ); + const zarrGroup = zarrSelection.zarrGroup; // Load the dataset using the CID from STAC const dataset = await openDatasetFromCid(variantConfig.cid, { gatewayUrl, @@ -452,6 +526,7 @@ export class DClimateClient { source: "stac_concatenated", fetchedAt: new Date(), ...(explicitZarrGroup ? { zarrGroup: explicitZarrGroup } : {}), + ...(request.resolution ? { resolution: request.resolution } : {}), }; if (options.returnJaxrayDataset) { diff --git a/src/errors.ts b/src/errors.ts index 4d8b1d6..dca05a0 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -9,6 +9,20 @@ export class DatasetNotFoundError extends DClimateClientError {} export class InvalidSelectionError extends DClimateClientError {} +export class MultiresolutionSelectionRequiredError extends InvalidSelectionError { + constructor( + message: string, + public availableResolutions: string[] = [], + public availableGroups: string[] = [] + ) { + super(message); + } +} + +export class ResolutionNotAvailableError extends InvalidSelectionError {} + +export class ConflictingResolutionSelectionError extends InvalidSelectionError {} + export class NoDataFoundError extends DClimateClientError {} export class SirenApiError extends DClimateClientError {} diff --git a/src/index.ts b/src/index.ts index bfa064b..622c396 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,6 +23,7 @@ export { type ConcatenableStacItem, type ResolvedDatasetFromStac, type StacReleaseMetadata, + type StacZarrResolution, type StacOrganization, type SpatialExtent, type TemporalExtent, diff --git a/src/ipfs/open-dataset.ts b/src/ipfs/open-dataset.ts index 368fd49..949c082 100644 --- a/src/ipfs/open-dataset.ts +++ b/src/ipfs/open-dataset.ts @@ -2,6 +2,7 @@ import { Dataset, openIpfsStore } from "@dclimate/jaxray"; import type { IPFSELEMENTS_INTERFACE } from "@dclimate/jaxray"; import { DEFAULT_IPFS_GATEWAY } from "../constants.js"; import type { ShardReadMode } from "../types.js"; +import { MultiresolutionSelectionRequiredError } from "../errors.js"; import { classifyRetrievalError, recordDatasetOpen, @@ -100,8 +101,9 @@ export async function openDatasetFromCid( dataset = await Dataset.open_zarr(store); } catch (error) { if (!requiresExplicitZarrGroup(error)) throw error; - dataset = await Dataset.open_zarr(store, { group: "0" }); - dataset.attrs._ipfs_zarr_group = "0"; + throw new MultiresolutionSelectionRequiredError( + "This Zarr store has multiple groups; pass zarrGroup explicitly." + ); } } if (zarrGroup) dataset.attrs._ipfs_zarr_group = zarrGroup; diff --git a/src/stac/index.ts b/src/stac/index.ts index f9cc120..5a16d14 100644 --- a/src/stac/index.ts +++ b/src/stac/index.ts @@ -15,6 +15,7 @@ export { type ConcatenableStacItem, type ResolvedDatasetFromStac, type StacReleaseMetadata, + type StacZarrResolution, type StacOrganization, type SpatialExtent, type TemporalExtent, diff --git a/src/stac/stac-catalog.ts b/src/stac/stac-catalog.ts index 7f589f8..dbc0080 100644 --- a/src/stac/stac-catalog.ts +++ b/src/stac/stac-catalog.ts @@ -194,7 +194,7 @@ export interface ConcatenableStacItem { cid: string; concatPriority: number; concatDimension: string; - zarrGroup?: string; + zarrResolutions: StacZarrResolution[]; } export interface StacOrganization { @@ -210,16 +210,31 @@ export interface ResolvedDatasetFromStac extends StacReleaseMetadata { organizationId?: string; dataset: string; variant: string; - zarrGroup?: string; + zarrResolutions: StacZarrResolution[]; } -export function getStacZarrGroup( - asset: Record | undefined, - properties: Record | undefined -): string | undefined { - return ( - getStringProperty(asset, "dclimate:zarr_group") ?? - getStringProperty(properties, "dclimate:default_zarr_group") +export interface StacZarrResolution { + assetKey: string; + resolution: string; + group: string; +} + +export function getStacZarrResolutions( + assets: Record +): StacZarrResolution[] { + const choices = Object.entries(assets).flatMap(([assetKey, asset]) => { + if (assetKey === "data") return []; + const resolution = getStringProperty(asset, "dclimate:spatial_resolution"); + const group = getStringProperty(asset, "dclimate:zarr_group"); + return resolution && group ? [{ assetKey, resolution, group }] : []; + }); + return choices.filter( + (choice, index) => + choices.findIndex( + (candidate) => + candidate.resolution === choice.resolution && + candidate.group === choice.group + ) === index ); } @@ -723,13 +738,22 @@ export function resolveDatasetFromStac( } } - if (!selectedItem?.assets?.data) { + if (!selectedItem) { + throw new StacResolutionError("No STAC item was selected for this dataset"); + } + + const zarrResolutions = getStacZarrResolutions(selectedItem.assets); + const selectedAsset = + selectedItem.assets.data ?? + (zarrResolutions[0] + ? selectedItem.assets[zarrResolutions[0].assetKey] + : undefined); + if (!selectedAsset) { throw new StacResolutionError( - `No data asset found for item "${selectedItem?.id ?? "unknown"}"` + `No readable data asset found for item "${selectedItem.id}"` ); } - - const href = selectedItem.assets.data.href; + const href = selectedAsset.href; const cid = href.replace(/^ipfs:\/\//, ""); return { @@ -738,7 +762,7 @@ export function resolveDatasetFromStac( organizationId, dataset, variant: resolvedVariant || "default", - zarrGroup: getStacZarrGroup(selectedItem.assets.data, selectedItem.properties), + zarrResolutions, ...getStacReleaseMetadata(selectedItem.properties), }; } @@ -817,7 +841,10 @@ export function getConcatenableItemsFromStac( "time"; // Extract CID from assets - const dataAsset = item.assets.data; + const zarrResolutions = getStacZarrResolutions(item.assets); + const dataAsset = + item.assets.data ?? + (zarrResolutions[0] ? item.assets[zarrResolutions[0].assetKey] : undefined); if (!dataAsset) continue; const cid = dataAsset.href.replace(/^ipfs:\/\//, ""); @@ -827,7 +854,7 @@ export function getConcatenableItemsFromStac( cid, concatPriority: priority, concatDimension: dimension, - zarrGroup: getStacZarrGroup(dataAsset, item.properties), + zarrResolutions, }); } diff --git a/src/stac/stac-server.ts b/src/stac/stac-server.ts index 1b33ad5..9e9e110 100644 --- a/src/stac/stac-server.ts +++ b/src/stac/stac-server.ts @@ -11,10 +11,11 @@ import type { DatasetCatalog, DatasetVariantConfig, StacReleaseMetadata, + StacZarrResolution, } from "./stac-catalog.js"; import { getStacReleaseMetadata, - getStacZarrGroup, + getStacZarrResolutions, getStringProperty, } from "./stac-catalog.js"; @@ -51,7 +52,7 @@ export interface ResolvedCidFromServer extends StacReleaseMetadata { collectionId: string; dataset: string; variant: string; - zarrGroup?: string; + zarrResolutions: StacZarrResolution[]; } const MAX_STAC_SEARCH_PAGES = 50; @@ -223,7 +224,12 @@ export async function resolveCidFromStacServer( } // Extract CID from asset - const dataAsset = selectedItem.assets?.data; + const zarrResolutions = getStacZarrResolutions(selectedItem.assets); + const dataAsset = + selectedItem.assets?.data ?? + (zarrResolutions[0] + ? selectedItem.assets[zarrResolutions[0].assetKey] + : undefined); const href = dataAsset?.href || ""; const advertisedCid = getStringProperty( selectedItem.properties, @@ -243,7 +249,7 @@ export async function resolveCidFromStacServer( collectionId: collection, dataset, variant: resolvedVariant, - zarrGroup: getStacZarrGroup(dataAsset, selectedItem.properties), + zarrResolutions, ...getStacReleaseMetadata(selectedItem.properties), }; } diff --git a/src/types.ts b/src/types.ts index 36e2501..6b32f72 100644 --- a/src/types.ts +++ b/src/types.ts @@ -34,6 +34,7 @@ export interface LoadDatasetOptions { returnJaxrayDataset?: boolean; autoConcatenate?: boolean; zarrGroup?: string; + resolution?: string; /** Read only the requested shard entry on read-only sparse-store cache misses. */ shardReadMode?: ShardReadMode; } @@ -115,6 +116,7 @@ export interface DatasetRequest { variant?: string; organization?: string; cid?: string; + resolution?: string; } export interface DatasetVersionsRequest { diff --git a/tests/fetch-dataset-cid.test.ts b/tests/fetch-dataset-cid.test.ts index 52bc32d..6276a14 100644 --- a/tests/fetch-dataset-cid.test.ts +++ b/tests/fetch-dataset-cid.test.ts @@ -2,6 +2,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { Dataset } from "@dclimate/jaxray"; import { DClimateClient } from "../src/index.js"; import { StacResolutionError } from "../src/stac/index.js"; +import { + ConflictingResolutionSelectionError, + MultiresolutionSelectionRequiredError, + ResolutionNotAvailableError, +} from "../src/errors.js"; const openDatasetFromCidMock = vi.hoisted(() => vi.fn()); @@ -24,7 +29,7 @@ describe("loadDataset CID resolution", () => { }); describe("STAC catalog resolution", () => { - it("uses the STAC data asset group unless the caller overrides it", async () => { + it("requires and resolves an explicit pyramid resolution or group", async () => { vi.stubGlobal( "fetch", vi.fn(async () => ({ @@ -42,12 +47,17 @@ describe("loadDataset CID resolution", () => { properties: { "dclimate:dataset_id": "pyramid", "dclimate:variant": "default", - "dclimate:default_zarr_group": "1", }, assets: { - data: { + "data-500m": { href: "ipfs://bafygrouped", - "dclimate:zarr_group": "/0/", + "dclimate:zarr_group": "0", + "dclimate:spatial_resolution": "500m", + }, + "data-2km": { + href: "ipfs://bafygrouped", + "dclimate:zarr_group": "2", + "dclimate:spatial_resolution": "2km", }, }, }, @@ -57,14 +67,25 @@ describe("loadDataset CID resolution", () => { ); const client = new DClimateClient({ stacServerUrl: "https://stac.test" }); + await expect( + client.loadDataset({ + request: { collection: "test_grouped", dataset: "pyramid" }, + }) + ).rejects.toBeInstanceOf(MultiresolutionSelectionRequiredError); + const [, discoveredMetadata] = await client.loadDataset({ - request: { collection: "test_grouped", dataset: "pyramid" }, + request: { + collection: "test_grouped", + dataset: "pyramid", + resolution: "500m", + }, }); expect(openDatasetFromCidMock).toHaveBeenLastCalledWith( "bafygrouped", expect.objectContaining({ zarrGroup: "0" }) ); expect(discoveredMetadata.zarrGroup).toBe("0"); + expect(discoveredMetadata.resolution).toBe("500m"); const [, overriddenMetadata] = await client.loadDataset({ request: { collection: "test_grouped", dataset: "pyramid" }, @@ -75,6 +96,28 @@ describe("loadDataset CID resolution", () => { expect.objectContaining({ zarrGroup: "2" }) ); expect(overriddenMetadata.zarrGroup).toBe("2"); + expect(overriddenMetadata.resolution).toBe("2km"); + + await expect( + client.loadDataset({ + request: { + collection: "test_grouped", + dataset: "pyramid", + resolution: "8km", + }, + }) + ).rejects.toBeInstanceOf(ResolutionNotAvailableError); + + await expect( + client.loadDataset({ + request: { + collection: "test_grouped", + dataset: "pyramid", + resolution: "500m", + }, + options: { zarrGroup: "0" }, + }) + ).rejects.toBeInstanceOf(ConflictingResolutionSelectionError); }); it("resolves CID from STAC for known dataset", async () => { @@ -210,6 +253,23 @@ describe("loadDataset CID resolution", () => { expect(metadata.zarrGroup).toBe("0"); }); + it("requires raw groups instead of resolutions for direct CIDs", async () => { + const client = new DClimateClient(); + + await expect( + client.loadDataset({ + request: { cid: "bafygrouped", resolution: "500m" }, + }) + ).rejects.toBeInstanceOf(ResolutionNotAvailableError); + + await expect( + client.loadDataset({ + request: { cid: "bafygrouped", resolution: "500m" }, + options: { zarrGroup: "0" }, + }) + ).rejects.toBeInstanceOf(ConflictingResolutionSelectionError); + }); + it("passes sparse shard decoding through to the dataset opener", async () => { const client = new DClimateClient(); await client.loadDataset({ diff --git a/tests/geotemporal-dataset.test.ts b/tests/geotemporal-dataset.test.ts index 084321c..f03d164 100644 --- a/tests/geotemporal-dataset.test.ts +++ b/tests/geotemporal-dataset.test.ts @@ -6,7 +6,13 @@ describe("GeoTemporalDataset - Real Data Integration Tests", () => { const client = new DClimateClient(); const DATASET_REQUESTS: Record = { - fpar: { collection: "copernicus_clms", organization: "copernicus", dataset: "fpar", variant: "default" }, + fpar: { + collection: "copernicus_clms", + organization: "copernicus", + dataset: "fpar", + variant: "default", + resolution: "500m", + }, "ifs-temperature": { collection: "ifs", organization: "ecmwf", dataset: "temperature_forecast", variant: "default" }, "ifs-precip": { collection: "ifs", organization: "ecmwf", dataset: "precipitation_forecast", variant: "default" }, "aifs-single-temperature": { diff --git a/tests/open-dataset.test.ts b/tests/open-dataset.test.ts index be5de16..481f274 100644 --- a/tests/open-dataset.test.ts +++ b/tests/open-dataset.test.ts @@ -49,7 +49,7 @@ describe("openDatasetFromCid", () => { expect(dataset.attrs).toEqual({ _ipfs_zarr_group: "0" }); }); - it("safely retries group zero when jaxray reports an ambiguous grouped root", async () => { + it("requires an explicit group when jaxray reports an ambiguous grouped root", async () => { const store = { kind: "grouped-store" }; const dataset = { kind: "dataset", attrs: {} as Record }; openIpfsStoreMock.mockResolvedValue({ store }); @@ -61,11 +61,12 @@ describe("openDatasetFromCid", () => { ) .mockResolvedValueOnce(dataset); - await expect(openDatasetFromCid("bafygrouped")).resolves.toBe(dataset); + await expect(openDatasetFromCid("bafygrouped")).rejects.toThrow( + "pass zarrGroup explicitly" + ); expect(openZarrMock).toHaveBeenNthCalledWith(1, store); - expect(openZarrMock).toHaveBeenNthCalledWith(2, store, { group: "0" }); - expect(dataset.attrs._ipfs_zarr_group).toBe("0"); + expect(openZarrMock).toHaveBeenCalledTimes(1); }); it("uses caller supplied IPFS elements", async () => { diff --git a/tests/review-fixes/concat-items-resolved-id.test.ts b/tests/review-fixes/concat-items-resolved-id.test.ts index bc44dc8..c32f77d 100644 --- a/tests/review-fixes/concat-items-resolved-id.test.ts +++ b/tests/review-fixes/concat-items-resolved-id.test.ts @@ -73,12 +73,14 @@ describe("getConcatenableItemsFromStac collection resolution", () => { cid: "bafy-era5-part1-data", concatPriority: 0, concatDimension: "time", + zarrResolutions: [], }, { variant: "part2", cid: "bafy-era5-part2-data", concatPriority: 1, concatDimension: "time", + zarrResolutions: [], }, ]; @@ -116,6 +118,7 @@ describe("getConcatenableItemsFromStac collection resolution", () => { cid: "bafy-era5-part1-data", concatPriority: 0, concatDimension: "time", + zarrResolutions: [], }, ]); }); diff --git a/tests/stac-version-discovery.test.ts b/tests/stac-version-discovery.test.ts index 113bd59..aecd35d 100644 --- a/tests/stac-version-discovery.test.ts +++ b/tests/stac-version-discovery.test.ts @@ -3,6 +3,7 @@ import { DClimateClient } from "../src/client.js"; import { VersionHistoryUnavailableError } from "../src/errors.js"; import { VersionApiError } from "../src/errors.js"; import { + getStacZarrResolutions, resolveDatasetFromStac, type StacCatalog, } from "../src/stac/stac-catalog.js"; @@ -22,7 +23,6 @@ const properties = { "dclimate:version_label": "2026-08", "dclimate:is_citable": true, "dclimate:retention_class": "permanent", - "dclimate:default_zarr_group": "1", }; const item = { @@ -33,9 +33,10 @@ const item = { properties, geometry: null, assets: { - data: { + "data-500m": { href: "ipfs://bafy-current", - "dclimate:zarr_group": "/0/", + "dclimate:zarr_group": "0", + "dclimate:spatial_resolution": "500m", }, }, links: [], @@ -53,6 +54,34 @@ function response(body: unknown): Response { describe("STAC release discovery", () => { afterEach(() => vi.unstubAllGlobals()); + it.each([true, false])( + "treats transitional data alias=%s as three choices", + (includeAlias) => { + const namedAssets = { + "data-500m": { + href: "ipfs://bafy-fpar", + "dclimate:zarr_group": "0", + "dclimate:spatial_resolution": "500m", + }, + "data-2km": { + href: "ipfs://bafy-fpar", + "dclimate:zarr_group": "1", + "dclimate:spatial_resolution": "2km", + }, + "data-8km": { + href: "ipfs://bafy-fpar", + "dclimate:zarr_group": "2", + "dclimate:spatial_resolution": "8km", + }, + }; + const assets = includeAlias + ? { data: { ...namedAssets["data-500m"] }, ...namedAssets } + : namedAssets; + + expect(getStacZarrResolutions(assets)).toHaveLength(3); + } + ); + it("extracts release metadata from the hosted STAC server", async () => { vi.stubGlobal( "fetch", @@ -74,7 +103,9 @@ describe("STAC release discovery", () => { commitId: "commit-1", isCitable: true, retentionClass: "permanent", - zarrGroup: "/0/", + zarrResolutions: [ + { assetKey: "data-500m", resolution: "500m", group: "0" }, + ], }); }); @@ -107,10 +138,12 @@ describe("STAC release discovery", () => { expect(resolved.versionsApi).toBe(properties["dclimate:versions_api"]); expect(resolved.provenanceApi).toBe(properties["dclimate:provenance_api"]); expect(resolved.citationApi).toBe(properties["dclimate:citation_api"]); - expect(resolved.zarrGroup).toBe("/0/"); + expect(resolved.zarrResolutions).toEqual([ + { assetKey: "data-500m", resolution: "500m", group: "0" }, + ]); }); - it("falls back to the item default when the data asset has no group", () => { + it("keeps flat data assets ungrouped", () => { const unannotatedAssetItem = { ...item, assets: { data: { href: "ipfs://bafy-current" } }, @@ -139,8 +172,8 @@ describe("STAC release discovery", () => { "wind_u_forecast", "operational", "noaa" - ).zarrGroup - ).toBe("1"); + ).zarrResolutions + ).toEqual([]); }); it("lists versions using the full URL advertised by STAC", async () => { From afaff48a3085c3186f1dc6d570744b5b54d98008 Mon Sep 17 00:00:00 2001 From: eloramirez1356 Date: Fri, 7 Aug 2026 12:44:48 -0500 Subject: [PATCH 5/5] fix: complete multiresolution selection handling --- src/client.ts | 20 +++++++++++++--- src/stac/stac-catalog.ts | 18 ++++++++++++++- src/types.ts | 2 +- .../autoconcat-variant-crash.test.ts | 23 +++++++++++++++++-- tests/stac-version-discovery.test.ts | 17 ++++++++++++++ 5 files changed, 73 insertions(+), 7 deletions(-) diff --git a/src/client.ts b/src/client.ts index b138dc9..a16bc67 100644 --- a/src/client.ts +++ b/src/client.ts @@ -487,7 +487,7 @@ export class DClimateClient { ); // Load all variants in parallel - const variantsToLoad: VariantToLoad[] = await Promise.all( + const loadedVariants = await Promise.all( orderedVariants.map(async (variantConfig) => { const zarrSelection = resolveZarrSelection( variantConfig.zarrResolutions, @@ -506,12 +506,26 @@ export class DClimateClient { return { variant: variantConfig, dataset, + zarrSelection, }; }) ); + const variantsToLoad: VariantToLoad[] = loadedVariants; // Concatenate the variants const concatenatedDataset = await concatenateVariants(variantsToLoad); + const firstSelection = loadedVariants[0].zarrSelection; + const commonZarrGroup = loadedVariants.every( + ({ zarrSelection }) => zarrSelection.zarrGroup === firstSelection.zarrGroup + ) + ? firstSelection.zarrGroup + : undefined; + const commonResolution = loadedVariants.every( + ({ zarrSelection }) => + zarrSelection.resolution === firstSelection.resolution + ) + ? firstSelection.resolution + : undefined; // Build metadata for the concatenated dataset const pathParts = [request.collection, request.dataset].filter(Boolean); @@ -525,8 +539,8 @@ export class DClimateClient { cid: variantsToLoad[0].dataset.attrs._zarr_cid as string || "concatenated", source: "stac_concatenated", fetchedAt: new Date(), - ...(explicitZarrGroup ? { zarrGroup: explicitZarrGroup } : {}), - ...(request.resolution ? { resolution: request.resolution } : {}), + ...(commonZarrGroup ? { zarrGroup: commonZarrGroup } : {}), + ...(commonResolution ? { resolution: commonResolution } : {}), }; if (options.returnJaxrayDataset) { diff --git a/src/stac/stac-catalog.ts b/src/stac/stac-catalog.ts index dbc0080..6ebd347 100644 --- a/src/stac/stac-catalog.ts +++ b/src/stac/stac-catalog.ts @@ -228,7 +228,7 @@ export function getStacZarrResolutions( const group = getStringProperty(asset, "dclimate:zarr_group"); return resolution && group ? [{ assetKey, resolution, group }] : []; }); - return choices.filter( + const uniqueChoices = choices.filter( (choice, index) => choices.findIndex( (candidate) => @@ -236,6 +236,22 @@ export function getStacZarrResolutions( candidate.group === choice.group ) === index ); + + const advertisedCids = uniqueChoices.map((choice) => ({ + assetKey: choice.assetKey, + cid: assets[choice.assetKey].href + .replace(/^ipfs:\/\//, "") + .replace(/^\/+|\/+$/g, ""), + })); + const firstCid = advertisedCids[0]?.cid; + const mismatchedCid = advertisedCids.find(({ cid }) => cid !== firstCid); + if (mismatchedCid) { + throw new StacResolutionError( + `Selectable resolution assets must use the same dataset CID; asset '${mismatchedCid.assetKey}' advertises '${mismatchedCid.cid}' instead of '${firstCid}'.` + ); + } + + return uniqueChoices; } // ============================================================================ diff --git a/src/types.ts b/src/types.ts index 6b32f72..ff455b2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -34,7 +34,6 @@ export interface LoadDatasetOptions { returnJaxrayDataset?: boolean; autoConcatenate?: boolean; zarrGroup?: string; - resolution?: string; /** Read only the requested shard entry on read-only sparse-store cache misses. */ shardReadMode?: ShardReadMode; } @@ -92,6 +91,7 @@ export interface DatasetMetadata { * Dimension used for concatenation (e.g., "time") */ concatDimension?: string; + resolution?: string; zarrGroup?: string; path: string; cid: string; diff --git a/tests/review-fixes/autoconcat-variant-crash.test.ts b/tests/review-fixes/autoconcat-variant-crash.test.ts index ff735c3..9fb6d6f 100644 --- a/tests/review-fixes/autoconcat-variant-crash.test.ts +++ b/tests/review-fixes/autoconcat-variant-crash.test.ts @@ -114,7 +114,14 @@ describe("loadDataset auto-concatenation", () => { }, geometry: null, links: [], - assets: { data: { href: "ipfs://bafy-part1-data" } }, + assets: { + data: { href: "ipfs://bafy-part1-data" }, + "data-500m": { + href: "ipfs://bafy-part1-data", + "dclimate:zarr_group": "0", + "dclimate:spatial_resolution": "500m", + }, + }, }, ], [ @@ -129,7 +136,14 @@ describe("loadDataset auto-concatenation", () => { }, geometry: null, links: [], - assets: { data: { href: "ipfs://bafy-part2-data" } }, + assets: { + data: { href: "ipfs://bafy-part2-data" }, + "data-500m": { + href: "ipfs://bafy-part2-data", + "dclimate:zarr_group": "0", + "dclimate:spatial_resolution": "500m", + }, + }, }, ], ]); @@ -183,5 +197,10 @@ describe("loadDataset auto-concatenation", () => { expect(metadata.concatenatedVariants).toEqual(["part2", "part1"]); expect(metadata.cid).toBe("bafy-part2-data"); expect(metadata.concatDimension).toBe("time"); + expect(metadata.resolution).toBe("500m"); + expect(metadata.zarrGroup).toBe("0"); + for (const [, openOptions] of openDatasetFromCidMock.mock.calls) { + expect(openOptions).toEqual(expect.objectContaining({ zarrGroup: "0" })); + } }); }); diff --git a/tests/stac-version-discovery.test.ts b/tests/stac-version-discovery.test.ts index aecd35d..b8b2717 100644 --- a/tests/stac-version-discovery.test.ts +++ b/tests/stac-version-discovery.test.ts @@ -82,6 +82,23 @@ describe("STAC release discovery", () => { } ); + it("rejects selectable resolution assets with different CIDs", () => { + expect(() => + getStacZarrResolutions({ + "data-500m": { + href: "ipfs://bafy-fpar-500m", + "dclimate:zarr_group": "0", + "dclimate:spatial_resolution": "500m", + }, + "data-2km": { + href: "ipfs://bafy-fpar-2km", + "dclimate:zarr_group": "1", + "dclimate:spatial_resolution": "2km", + }, + }) + ).toThrow("Selectable resolution assets must use the same dataset CID"); + }); + it("extracts release metadata from the hosted STAC server", async () => { vi.stubGlobal( "fetch",