From 5b013281d928a78dc8756ea998d60f9005119814 Mon Sep 17 00:00:00 2001 From: Sagar Ghimire Date: Thu, 27 Aug 2026 14:48:07 +0545 Subject: [PATCH 1/5] feat: move download URLs under the versioned prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Downloads move from `/datastore/dump/…` to `/datastore/api/v2/dump/…`, reverting the direction of 34ec309. A dump URL's query params and output layout are part of the same compatibility contract as the actions, so it versions with them rather than sitting outside `/api`. `DUMP_PREFIX` is now built from `API_PREFIX`, which is the whole change — routes, analytics and tests all derive their paths from it. The analytics middleware still checks the dump prefixes before the action prefix, so a download is recorded as `datastore_dump` rather than an action named `dump`. Health probes stay unversioned. The Postman collection had no download requests at all; it gains a `dump` folder covering both routes across all four formats. Those requests take a path parameter and no body, so they are declared inline in the generator rather than driven by `example_payload/`. Co-Authored-By: Claude Opus 5 (1M context) --- API.md | 16 +- CLAUDE.md | 12 +- README.md | 2 +- datastore/api/routes.py | 5 +- datastore/core/constants.py | 8 +- .../infrastructure/engines/bigquery/export.py | 2 +- postman/README.md | 17 +- postman/collection.json | 184 +++++++++++++++++- postman/generate_postman.py | 92 +++++++++ tests/test_datastore_dump_sql.py | 2 +- 10 files changed, 314 insertions(+), 26 deletions(-) diff --git a/API.md b/API.md index 910e271..2a67271 100644 --- a/API.md +++ b/API.md @@ -75,9 +75,9 @@ token (except under `anonymous`). | POST | `/datastore/api/v2/datastore_delete` | Delete rows, drop columns, or drop the table | | GET | `/datastore/api/v2/datastore_search` | Search a resource (streaming) | | GET | `/datastore/api/v2/datastore_search_sql` | Run a read-only SQL `SELECT` (streaming) | -| GET | `/datastore/dump/query` | Download the result of a SQL `SELECT` as a file | +| GET | `/datastore/api/v2/dump/query` | Download the result of a SQL `SELECT` as a file | | GET | `/datastore/api/v2/datastore_info` | Schema + row stats for a resource | -| GET | `/datastore/dump/{resource_id}` | Download a whole resource (CSV/NDJSON/Parquet) | +| GET | `/datastore/api/v2/dump/{resource_id}` | Download a whole resource (CSV/NDJSON/Parquet) | | GET | `/datastore/api/health` · `/datastore/api/ready` | Liveness / readiness | --- @@ -314,7 +314,7 @@ GET /datastore/api/v2/datastore_search Run a single read-only `SELECT` / `WITH` statement and stream the result. Tables are referenced by `resource_id`; each is authorized individually, and functions are checked against the engine's allow-list. Include a `LIMIT` (required). -To export the result as a file instead, use [`GET /datastore/dump/query`](#get-datastoredumpquery). +To export the result as a file instead, use [`GET /datastore/api/v2/dump/query`](#get-datastoreapiv2dumpquery). ### Query parameters @@ -386,7 +386,7 @@ row stats — a column-level metadata catalog without a side store. --- -## `GET /datastore/dump/{resource_id}` +## `GET /datastore/api/v2/dump/{resource_id}` Download an entire resource. Pick the format with `?format=csv` (default), `gzip`, `ndjson`, or `parquet`. @@ -405,13 +405,13 @@ Download an entire resource. Pick the format with `?format=csv` (default), Requires `read` permission on the resource and a configured export bucket (`BIGQUERY_EXPORT_BUCKET`). -`query` is a **reserved name** on this route — `/datastore/dump/query` is the SQL +`query` is a **reserved name** on this route — `/datastore/api/v2/dump/query` is the SQL download endpoint below, so a resource literally named `query` can't be dumped by this URL. --- -## `GET /datastore/dump/query` +## `GET /datastore/api/v2/dump/query` Download the result of a **SQL `SELECT`** as a single file — filtered downloads at any size. Same validation as `datastore_search_sql` (single @@ -428,14 +428,14 @@ file itself, not the CKAN envelope. ### Example ```http -GET /datastore/dump/query +GET /datastore/api/v2/dump/query ?sql=SELECT * FROM "c6153a74-43cb-4edf-8bdf-bb664feca937" WHERE accepted = true &format=csv ``` ### Response -Identical to `/datastore/dump/{resource_id}` above: +Identical to `/datastore/api/v2/dump/{resource_id}` above: - **csv / gzip / ndjson** — `302` to a signed GCS URL at any size (shards are composed into one object). The URL expires after diff --git a/CLAUDE.md b/CLAUDE.md index 3e14233..e5deb7a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -430,9 +430,9 @@ Each endpoint takes a single `ContextDep`. The handler calls `context.authorize( | POST | `/datastore/api/v2/datastore_delete` | **implemented** | `DatastoreDeleteRequest` | `DatastoreDeleteResponse` | | GET | `/datastore/api/v2/datastore_search` | **implemented** (streaming) | `DatastoreSearchRequest` | `DatastoreSearchResponse` | | GET | `/datastore/api/v2/datastore_search_sql` | **implemented** (streaming) | `DatastoreSearchSQLRequest` | `DatastoreSearchResponse` | -| GET | `/datastore/dump/query` | **implemented** | `sql=`, `format=csv\|gzip\|ndjson\|parquet` | 302 → GCS *or* streaming body (see §5.3) | +| GET | `/datastore/api/v2/dump/query` | **implemented** | `sql=`, `format=csv\|gzip\|ndjson\|parquet` | 302 → GCS *or* streaming body (see §5.3) | | GET | `/datastore/api/v2/datastore_info` | **implemented** | `DatastoreInfoRequest` | `DatastoreInfoResponse` | -| GET | `/datastore/dump/{resource_id}` | **implemented** | `format=csv\|ndjson\|parquet` | 302 → GCS *or* streaming body (see §5.3) | +| GET | `/datastore/api/v2/dump/{resource_id}` | **implemented** | `format=csv\|ndjson\|parquet` | 302 → GCS *or* streaming body (see §5.3) | The BigQuery engine is wired end-to-end: DDL, MERGE-based upsert, DML delete, parameterised search, native table-level metadata (the Frictionless schema + unique_key are JSON-encoded into the table's own `description` OPTION) for the schema round-trip, a row-count fast path via `INFORMATION_SCHEMA.TABLE_STORAGE`, and `EXPORT DATA`-backed dump with `table.modified`-keyed GCS caching. The DuckLake engine is the next concrete adapter — see §7. @@ -443,7 +443,7 @@ The BigQuery engine is wired end-to-end: DDL, MERGE-based upsert, DML delete, pa **Read-only guard (`AUTH_TYPE=ckan` only).** `datastore_create`, `datastore_upsert`, and `datastore_delete` refuse to write a resource whose CKAN record carries `url_type="datastore"` unless the request sets `force: true` — a `Validation Error` ("Cannot update a read-only resource. Use \"force\" to force update.") otherwise. This mirrors CKAN's protection against clobbering datastore-managed data by accident. The guard is gated on `AUTH_TYPE=ckan` and skipped entirely under any other provider (only the CKAN provider attaches a resource record). -### 5.3 `GET /datastore/dump/{resource_id}` +### 5.3 `GET /datastore/api/v2/dump/{resource_id}` Full-table download, **one URL → one file** from the caller's point of view. Bytes never pass through API memory — the one exception is a @@ -497,9 +497,9 @@ A single SA works if both perm sets land on the same identity — `BIGQUERY_CRED A 24h object-lifecycle rule on the bucket is **required** in practice: the engine GCs older revs already, but lifecycle is the only thing that cleans abandoned `dumps//` prefixes (SQL downloads whose query is never re-issued — see below) and anything stranded by a crashed dump. -### SQL download (`GET /datastore/dump/query`) +### SQL download (`GET /datastore/api/v2/dump/query`) -`GET /datastore/dump/query?sql=&format=csv|gzip|ndjson|parquet` exports the result of an arbitrary vetted SELECT through the same pipeline as `/datastore/dump/{resource_id}` — engine method `dump_sql` in [bigquery/export.py](datastore/infrastructure/engines/bigquery/export.py), response shaping shared via `download_response` in [api/endpoints/dump.py](datastore/api/endpoints/dump.py) (302 for the composed file · gzip streamed · JSON URL list for multi-file parquet). Same SQL validation + per-table auth as `datastore_search_sql` (`DatastoreDumpSQLRequest` subclasses its request schema); the action API itself stays pure JSON envelope. The route is declared before `/datastore/dump/{resource_id}`, making `query` a reserved resource name on the dump family. +`GET /datastore/api/v2/dump/query?sql=&format=csv|gzip|ndjson|parquet` exports the result of an arbitrary vetted SELECT through the same pipeline as `/datastore/api/v2/dump/{resource_id}` — engine method `dump_sql` in [bigquery/export.py](datastore/infrastructure/engines/bigquery/export.py), response shaping shared via `download_response` in [api/endpoints/dump.py](datastore/api/endpoints/dump.py) (302 for the composed file · gzip streamed · JSON URL list for multi-file parquet). Same SQL validation + per-table auth as `datastore_search_sql` (`DatastoreDumpSQLRequest` subclasses its request schema); the action API itself stays pure JSON envelope. The route is declared before `/datastore/api/v2/dump/{resource_id}`, making `query` a reserved resource name on the dump family. Deltas vs the whole-table dump: @@ -799,7 +799,7 @@ Optional fields appear in `result` only when requested: ### 6.4 `GET /datastore/api/v2/datastore_search_sql` -**Query params**: `sql` (required; must carry a `LIMIT` literal). To export the result as a file instead of the JSON envelope, use `GET /datastore/dump/query?sql=…&format=…` (LIMIT optional + uncapped there — see §5.3 "SQL download"). +**Query params**: `sql` (required; must carry a `LIMIT` literal). To export the result as a file instead of the JSON envelope, use `GET /datastore/api/v2/dump/query?sql=…&format=…` (LIMIT optional + uncapped there — see §5.3 "SQL download"). **Example request — daily clearing-price summary** ``` diff --git a/README.md b/README.md index 374ef4e..b08c23d 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Copy [.env.example](.env.example) and fill it in. The essentials: | `AUTH_TYPE` | `ckan` | Auth provider: `ckan` · `jwt` · `anonymous` | | `CKAN_URL` | — | CKAN base URL (required when `AUTH_TYPE=ckan`) | | `BIGQUERY_PROJECT` / `BIGQUERY_DATASET` | — | Required when `DATASTORE_ENGINE=bigquery` | -| `BIGQUERY_EXPORT_BUCKET` | — | GCS bucket for downloads (`/datastore/dump/{resource_id}`, `/datastore/dump/query`) | +| `BIGQUERY_EXPORT_BUCKET` | — | GCS bucket for downloads (`/datastore/api/v2/dump/{resource_id}`, `/datastore/api/v2/dump/query`) | | `REDIS_URL` | — | Cache backend; empty → in-process cache | | `API_URL` | `https://example.com` | Public base URL; sets the host in the OpenAPI examples | | `DOCS_PRIMARY_COLOR` / `DOCS_HEADER_COLOR` | — | Swagger UI branding (see [Documentation](#documentation)) | diff --git a/datastore/api/routes.py b/datastore/api/routes.py index 4dad230..89e33bc 100644 --- a/datastore/api/routes.py +++ b/datastore/api/routes.py @@ -8,8 +8,9 @@ api_router = APIRouter() # Unversioned: a probe URL is stable regardless of the action contract. api_router.include_router(health.probe_router, prefix=API_BASE_PREFIX) -# Downloads sit outside `/api` — a dump URL is opened in a browser or fed to -# `curl`, not driven by an API client. +# Downloads: versioned alongside the actions, since their query params and +# output layout are part of the same contract. Mounted before the action +# router so `/dump/...` can't be read as an action name. api_router.include_router(dump.router, prefix=DUMP_PREFIX) # Versioned: the datastore actions are the compatibility contract. api_router.include_router(datastore.router, prefix=API_PREFIX) diff --git a/datastore/core/constants.py b/datastore/core/constants.py index 11ff7be..7f54593 100644 --- a/datastore/core/constants.py +++ b/datastore/core/constants.py @@ -14,8 +14,10 @@ # the compatibility contract; # - the health probes sit outside it, so an orchestrator's probe URL # doesn't churn when the contract gets a new version; -# - downloads sit outside the `/api` segment entirely, since a dump URL is -# handed to a browser or a `curl` rather than driven by an API client. +# - downloads sit under the versioned prefix alongside the actions: a dump +# URL is handed to a browser or a `curl` rather than driven by an API +# client, but its query params and output layout are part of the same +# compatibility contract, so it versions with them. # # Note this is NOT the prefix used to *call* an upstream CKAN — those # requests go to CKAN's own `/api/3/action/` (see @@ -24,7 +26,7 @@ API_BASE_PREFIX = f"{SERVICE_PREFIX}/api" API_VERSION = "v2" API_PREFIX = f"{API_BASE_PREFIX}/{API_VERSION}" -DUMP_PREFIX = f"{SERVICE_PREFIX}/dump" +DUMP_PREFIX = f"{API_PREFIX}/dump" # Download formats served by the export pipeline (`/…` and # `datastore_search_sql?download=…`). Lives here — not in `api/` — because # both the request schemas (pydantic layer) and the endpoints (starlette diff --git a/datastore/infrastructure/engines/bigquery/export.py b/datastore/infrastructure/engines/bigquery/export.py index 74c3bde..94e4f43 100644 --- a/datastore/infrastructure/engines/bigquery/export.py +++ b/datastore/infrastructure/engines/bigquery/export.py @@ -648,7 +648,7 @@ def _get_export_bucket(backend: Any) -> str: if not bucket: raise ServerError( "BIGQUERY_EXPORT_BUCKET is not configured — " - "/datastore/dump cannot run without an export bucket." + "/datastore/api/v2/dump cannot run without an export bucket." ) return bucket diff --git a/postman/README.md b/postman/README.md index 52915c6..d935d33 100644 --- a/postman/README.md +++ b/postman/README.md @@ -6,9 +6,9 @@ Auto-generated from [`example_payload/`](../example_payload/) by ## Import -In Postman: **File → Import** → `collection.json`. Seven folders appear: +In Postman: **File → Import** → `collection.json`. Eight folders appear: `health`, `datastore_create`, `datastore_upsert`, `datastore_info`, -`datastore_search`, `datastore_search_sql`, `datastore_delete`. +`datastore_search`, `datastore_search_sql`, `datastore_delete`, `dump`. ## Variables @@ -29,10 +29,19 @@ Run folders top-to-bottom on a fresh resource: 3. **`datastore_info`** — confirm schema + row count. 4. **`datastore_search`** — filter / full-text / paginated. 5. **`datastore_search_sql`** — raw SQL; `LIMIT` required. JOIN/UNION variants need a second resource `balancing_auction_results_2024`. -6. **`datastore_delete`** — row delete (`auction_id=1`) → drop column (`bidder_metadata`) → drop table. +6. **`dump`** — download the table or a SQL result as a file. Run before + `datastore_delete`, while the rows still exist. +7. **`datastore_delete`** — row delete (`auction_id=1`) → drop column (`bidder_metadata`) → drop table. `health` is independent — hit any time to check the server. +**On the `dump` folder:** these return `302` to a signed GCS URL rather than a +body. Postman follows the redirect by default and downloads the file; switch +off **Settings → Automatically follow redirects** to inspect the `Location` +header instead. A sharded parquet export returns `200` + a zip of the parts +instead of a redirect. The server needs `BIGQUERY_EXPORT_BUCKET` set — +without it every dump request is a `500`. + ## Regenerate ```sh @@ -40,6 +49,8 @@ python postman/generate_postman.py ``` Drop new files under `example_payload//.json` to add requests. +The `dump` folder is the exception: its requests take a path parameter and no +body, so they're declared inline as `DUMP_REQUESTS` in the generator. ## Auth diff --git a/postman/collection.json b/postman/collection.json index bea69b2..b95274d 100644 --- a/postman/collection.json +++ b/postman/collection.json @@ -1,6 +1,6 @@ { "info": { - "_postman_id": "fa56adc4-1610-477b-b095-713e228cad51", + "_postman_id": "bfb8a805-c676-4c5b-9832-520972b8d184", "name": "Datastore API", "description": "CKAN-compatible datastore API \u2014 auto-generated from `example_payload/`. Set `baseUrl` to your server, `apiKey` to a CKAN API key (anonymous reads are allowed; writes require a key), and `resourceId` to the table you want to hit.", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" @@ -920,6 +920,188 @@ "response": [] } ] + }, + { + "name": "dump", + "description": "File downloads: `/datastore/api/v2/dump/{resource_id}` for a whole table, `/datastore/api/v2/dump/query` for a SQL result. One URL always yields one file \u2014 a 302 to a signed GCS URL for csv / gzip / ndjson, and for parquet either a 302 (single shard) or a streamed zip of the parts. Needs `BIGQUERY_EXPORT_BUCKET` set on the server.", + "item": [ + { + "name": "Dump - whole table (CSV)", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/datastore/api/v2/dump/{{resourceId}}?format=csv", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "datastore", + "api", + "v2", + "dump", + "{{resourceId}}" + ], + "query": [ + { + "key": "format", + "value": "csv" + } + ] + }, + "description": "Export the whole table. Responds 302 to a signed GCS URL \u2014 the bytes go straight from GCS to the client. Turn OFF Postman's \"Automatically follow redirects\" to inspect the `Location` header instead of downloading." + }, + "response": [] + }, + { + "name": "Dump - whole table (gzip)", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/datastore/api/v2/dump/{{resourceId}}?format=gzip", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "datastore", + "api", + "v2", + "dump", + "{{resourceId}}" + ], + "query": [ + { + "key": "format", + "value": "gzip" + } + ] + }, + "description": "Same export, gzipped by BigQuery. Shards compose into one `.csv.gz` carrying exactly one header member." + }, + "response": [] + }, + { + "name": "Dump - whole table (NDJSON)", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/datastore/api/v2/dump/{{resourceId}}?format=ndjson", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "datastore", + "api", + "v2", + "dump", + "{{resourceId}}" + ], + "query": [ + { + "key": "format", + "value": "ndjson" + } + ] + }, + "description": "Newline-delimited JSON, one object per row. Composes to a single object when the export shards." + }, + "response": [] + }, + { + "name": "Dump - whole table (Parquet)", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/datastore/api/v2/dump/{{resourceId}}?format=parquet", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "datastore", + "api", + "v2", + "dump", + "{{resourceId}}" + ], + "query": [ + { + "key": "format", + "value": "parquet" + } + ] + }, + "description": "Parquet can't be composed (footer + magic bytes), so a sharded export returns 200 + a streamed zip of the parts rather than a redirect." + }, + "response": [] + }, + { + "name": "Dump SQL - query result (CSV)", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/datastore/api/v2/dump/query?sql=SELECT auction_id, product_code, clearing_price_gbp_per_mwh FROM \"{{resourceId}}\" WHERE accepted = true&format=csv", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "datastore", + "api", + "v2", + "dump", + "query" + ], + "query": [ + { + "key": "sql", + "value": "SELECT auction_id, product_code, clearing_price_gbp_per_mwh FROM \"{{resourceId}}\" WHERE accepted = true" + }, + { + "key": "format", + "value": "csv" + } + ] + }, + "description": "Export an arbitrary vetted SELECT. Unlike `datastore_search_sql`, `LIMIT` is optional and uncapped here. `query` is a reserved resource name on this route." + }, + "response": [] + }, + { + "name": "Dump SQL - aggregate (Parquet)", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/datastore/api/v2/dump/query?sql=SELECT product_code, AVG(clearing_price_gbp_per_mwh) AS avg_price, SUM(volume_mwh) AS total_volume FROM \"{{resourceId}}\" GROUP BY product_code&format=parquet", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "datastore", + "api", + "v2", + "dump", + "query" + ], + "query": [ + { + "key": "sql", + "value": "SELECT product_code, AVG(clearing_price_gbp_per_mwh) AS avg_price, SUM(volume_mwh) AS total_volume FROM \"{{resourceId}}\" GROUP BY product_code" + }, + { + "key": "format", + "value": "parquet" + } + ] + }, + "description": "Aggregate export. Results cache per (SQL, table version); a query calling `now()` / `current_date` bypasses the cache and re-exports." + }, + "response": [] + } + ] } ] } diff --git a/postman/generate_postman.py b/postman/generate_postman.py index 36d02c5..bf31da5 100644 --- a/postman/generate_postman.py +++ b/postman/generate_postman.py @@ -73,6 +73,69 @@ ] +# Download requests. These aren't driven by `example_payload/` — the table +# dump takes its resource on the path rather than in a body, and neither +# request has a JSON payload. Each entry: name, path, query params, blurb. +DUMP_REQUESTS: list[tuple[str, str, list[tuple[str, str]], str]] = [ + ( + "Dump - whole table (CSV)", + "datastore/api/v2/dump/{{resourceId}}", + [("format", "csv")], + "Export the whole table. Responds 302 to a signed GCS URL — the " + "bytes go straight from GCS to the client. Turn OFF Postman's " + "\"Automatically follow redirects\" to inspect the `Location` " + "header instead of downloading.", + ), + ( + "Dump - whole table (gzip)", + "datastore/api/v2/dump/{{resourceId}}", + [("format", "gzip")], + "Same export, gzipped by BigQuery. Shards compose into one " + "`.csv.gz` carrying exactly one header member.", + ), + ( + "Dump - whole table (NDJSON)", + "datastore/api/v2/dump/{{resourceId}}", + [("format", "ndjson")], + "Newline-delimited JSON, one object per row. Composes to a " + "single object when the export shards.", + ), + ( + "Dump - whole table (Parquet)", + "datastore/api/v2/dump/{{resourceId}}", + [("format", "parquet")], + "Parquet can't be composed (footer + magic bytes), so a sharded " + "export returns 200 + a streamed zip of the parts rather than a " + "redirect.", + ), + ( + "Dump SQL - query result (CSV)", + "datastore/api/v2/dump/query", + [ + ("sql", "SELECT auction_id, product_code, " + "clearing_price_gbp_per_mwh " + "FROM \"{{resourceId}}\" WHERE accepted = true"), + ("format", "csv"), + ], + "Export an arbitrary vetted SELECT. Unlike `datastore_search_sql`, " + "`LIMIT` is optional and uncapped here. `query` is a reserved " + "resource name on this route.", + ), + ( + "Dump SQL - aggregate (Parquet)", + "datastore/api/v2/dump/query", + [ + ("sql", "SELECT product_code, AVG(clearing_price_gbp_per_mwh) " + "AS avg_price, SUM(volume_mwh) AS total_volume " + "FROM \"{{resourceId}}\" GROUP BY product_code"), + ("format", "parquet"), + ], + "Aggregate export. Results cache per (SQL, table version); a query " + "calling `now()` / `current_date` bypasses the cache and re-exports.", + ), +] + + def _request_url(path: str, query: list[dict[str, str]] | None = None) -> dict[str, Any]: """Postman v2.1 structured URL — lets the Postman UI edit params.""" parts = path.strip("/").split("/") @@ -355,10 +418,39 @@ def _build_health_folder() -> dict[str, Any]: } +def _build_dump_folder() -> dict[str, Any]: + items = [] + for name, path, params, desc in DUMP_REQUESTS: + query = [{"key": k, "value": v} for k, v in params] + items.append({ + "name": name, + "request": { + "method": "GET", + "header": [], + "url": _request_url(path, query=query), + "description": desc, + }, + "response": [], + }) + return { + "name": "dump", + "description": ( + "File downloads: `/datastore/api/v2/dump/{resource_id}` for a " + "whole table, `/datastore/api/v2/dump/query` for a SQL result. " + "One URL always yields one file — a 302 to a signed GCS URL for " + "csv / gzip / ndjson, and for parquet either a 302 (single " + "shard) or a streamed zip of the parts. Needs " + "`BIGQUERY_EXPORT_BUCKET` set on the server." + ), + "item": items, + } + + def build_collection() -> dict[str, Any]: folders: list[dict[str, Any]] = [_build_health_folder()] for action, method, description in ENDPOINTS: folders.append(_build_endpoint_folder(action, method, description)) + folders.append(_build_dump_folder()) return { "info": { "_postman_id": str(uuid.uuid4()), diff --git a/tests/test_datastore_dump_sql.py b/tests/test_datastore_dump_sql.py index 33d5824..d7546eb 100644 --- a/tests/test_datastore_dump_sql.py +++ b/tests/test_datastore_dump_sql.py @@ -418,7 +418,7 @@ def test_parquet_export_casts_json_columns_from_dry_run_schema() -> None: def test_csv_export_iso_casts_timestamps_from_dry_run_schema() -> None: """CSV downloads render TIMESTAMP identically to `datastore_search` - and `/datastore/dump` (shared `format_select_column`).""" + and `/datastore/api/v2/dump` (shared `format_select_column`).""" new_blob = _blob("z_000.csv", "https://fresh") backend, storage_client = _engine_with_storage([]) bucket_obj = storage_client.bucket.return_value From 4fe772a609ab6fffeebfb0105485d3324f8c6a21 Mon Sep 17 00:00:00 2001 From: Sagar Ghimire Date: Thu, 27 Aug 2026 15:02:04 +0545 Subject: [PATCH 2/5] fix: stop documenting downloads as JSON responses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dump routes declared no `response_class`, so FastAPI added its default `application/json` body to the 200 for both. Swagger rendered that as the primary response, and a generated client would have parsed a parquet file or a redirect as the CKAN envelope. Both routes now declare `response_class=RedirectResponse` with an explicit `status_code=302`, which is what the handler actually returns — without the pin FastAPI documents RedirectResponse's own 307 default as a "Successful Response" that never occurs. The 302 gains its `Location` header and the per-format content types, and the 200 stays the sharded-parquet zip. `DUMP_MEDIA_TYPES` puts the format → content-type mapping next to the extensions it parallels; API.md gains the same table. Error responses are untouched: those really are JSON envelopes. Co-Authored-By: Claude Opus 5 (1M context) --- API.md | 11 +++++++ datastore/api/endpoints/dump.py | 56 +++++++++++++++++++++++---------- datastore/core/constants.py | 11 +++++++ tests/test_openapi.py | 38 +++++++++++++++++++++- 4 files changed, 99 insertions(+), 17 deletions(-) diff --git a/API.md b/API.md index 2a67271..3a9e1f1 100644 --- a/API.md +++ b/API.md @@ -391,6 +391,17 @@ row stats — a column-level metadata catalog without a side store. Download an entire resource. Pick the format with `?format=csv` (default), `gzip`, `ndjson`, or `parquet`. +This route returns a **file**, never the JSON envelope — the only JSON it can +produce is an error. The content type comes from the signed GCS URL and +follows `format`: + +| `format` | Content-Type | Extension | +|---|---|---| +| `csv` | `text/csv` | `.csv` | +| `gzip` | `application/gzip` | `.csv.gz` | +| `ndjson` | `application/x-ndjson` | `.json` | +| `parquet` | `application/vnd.apache.parquet` | `.parquet` | + - **csv / gzip / ndjson** — `302` redirect to a signed GCS URL, at any size. Shards from a large export are stitched into one object server-side, so the bytes go straight from storage to the client (resumable, no server diff --git a/datastore/api/endpoints/dump.py b/datastore/api/endpoints/dump.py index 1d652eb..a9ac0f2 100644 --- a/datastore/api/endpoints/dump.py +++ b/datastore/api/endpoints/dump.py @@ -9,7 +9,7 @@ from __future__ import annotations -from typing import Annotated +from typing import Annotated, Any from fastapi import APIRouter, Query from starlette.requests import Request @@ -17,7 +17,7 @@ from datastore.api.context import Context from datastore.api.responses import ERROR_RESPONSES -from datastore.core.constants import DUMP_EXTENSIONS, DumpFormat +from datastore.core.constants import DUMP_EXTENSIONS, DUMP_MEDIA_TYPES, DumpFormat from datastore.core.exceptions import ServerError from datastore.infrastructure.engines import get_datastore_engine from datastore.schemas.request import DatastoreDumpSQLRequest @@ -27,6 +27,38 @@ router = APIRouter(tags=["Datastore Download"], responses=ERROR_RESPONSES) +# What a download actually returns, for OpenAPI. Without an explicit +# `response_class` FastAPI documents a JSON body on the 200 — wrong for +# every path here, and it would have a generated client parsing a parquet +# file as an envelope. The 302 is the usual answer; the 200 only happens +# when a parquet export shards. +_DOWNLOAD_RESPONSES: dict[int | str, dict[str, Any]] = { + 302: { + "description": ( + "The download. Redirects to a short-lived signed GCS URL — the " + "bytes stream from GCS, not through this API, so the transfer " + "is resumable. Content type follows `format`: " + + ", ".join(f"`{f}` → `{m}`" for f, m in DUMP_MEDIA_TYPES.items()) + + ". The URL carries a `Content-Disposition` naming the file." + ), + "headers": { + "Location": { + "description": "Signed GCS URL holding the exported file.", + "schema": {"type": "string", "format": "uri"}, + }, + }, + }, + 200: { + "description": ( + "A parquet export that sharded: one streamed zip of the parts, " + "served by this API rather than redirected. Chunked, so there " + "is no `Content-Length` and no range support." + ), + "content": {"application/zip": {}}, + }, +} + + def download_response( request: Request, urls: list[str], @@ -67,13 +99,9 @@ def download_response( @router.get( "/query", summary="Download the result of a SQL SELECT", - responses={ - 302: {"description": "Redirect to the signed download URL."}, - 200: { - "description": ("Sharded parquet export - one streamed zip of the parts."), - "content": {"application/zip": {}}, - }, - }, + response_class=RedirectResponse, + status_code=302, + responses=_DOWNLOAD_RESPONSES, ) async def dump_sql( request: Request, @@ -99,13 +127,9 @@ async def dump_sql( @router.get( "/{resource_id}", summary="Download an entire table", - responses={ - 302: {"description": "Redirect to the signed Download URL."}, - 200: { - "description": ("Multi-file parquet export — one streamed zip."), - "content": {"application/zip": {}}, - }, - }, + response_class=RedirectResponse, + status_code=302, + responses=_DOWNLOAD_RESPONSES, ) async def dump( request: Request, diff --git a/datastore/core/constants.py b/datastore/core/constants.py index 7f54593..6dc402a 100644 --- a/datastore/core/constants.py +++ b/datastore/core/constants.py @@ -43,6 +43,17 @@ "parquet": "parquet", } +# Media type per dump format. Only ever reaches the client through the +# signed GCS URL's own headers — the API redirects rather than serving the +# bytes — but OpenAPI needs them to document what a download actually is, +# so a generated client doesn't assume the JSON envelope. +DUMP_MEDIA_TYPES: dict[str, str] = { + "csv": "text/csv", + "gzip": "application/gzip", + "ndjson": "application/x-ndjson", + "parquet": "application/vnd.apache.parquet", +} + POSTGRES_TYPES: dict[str, str] = { # integer "int2": "int2", diff --git a/tests/test_openapi.py b/tests/test_openapi.py index 05cf93e..c57b1d9 100644 --- a/tests/test_openapi.py +++ b/tests/test_openapi.py @@ -17,7 +17,12 @@ from datastore.api import docs as docs_module from datastore.api.docs import api_description from datastore.core.config import Config, get_config -from datastore.core.constants import API_PREFIX, API_VERSION, DEFAULT_API_URL +from datastore.core.constants import ( + API_PREFIX, + API_VERSION, + DEFAULT_API_URL, + DUMP_PREFIX, +) from datastore.main import create_app from fastapi.testclient import TestClient from pydantic import ValidationError @@ -476,3 +481,34 @@ def test_docs_page_falls_back_to_openapi_title( get_config.cache_clear() assert '

Datastore API

' in body + + +# Downloads are files, not envelopes --------------------------------------- + +def test_download_routes_do_not_advertise_a_json_body() -> None: + """A dump returns a file, so its success responses must not be typed as + JSON. + + FastAPI documents `application/json` on the 200 for any route without an + explicit `response_class`. That default is wrong here — it would have a + generated client parsing a parquet file (or a redirect) as the CKAN + envelope. The error responses stay JSON: those really are envelopes. + """ + schema = create_app().openapi() + + for path in (f"{DUMP_PREFIX}/query", f"{DUMP_PREFIX}/{{resource_id}}"): + responses = schema["paths"][path]["get"]["responses"] + + # 302 is the usual answer and carries no body at all. + assert "content" not in responses["302"] + assert "Location" in responses["302"]["headers"] + + # 200 happens only for a sharded parquet export: a zip, never JSON. + assert list(responses["200"]["content"]) == ["application/zip"] + + # Errors are the one place the JSON envelope belongs. + assert list(responses["400"]["content"]) == ["application/json"] + + # RedirectResponse's own 307 default must not leak in as a + # documented "Successful Response" the route never returns. + assert "307" not in responses From 7c4df803aa058535bca699427a3d687086a24560 Mon Sep 17 00:00:00 2001 From: Sagar Ghimire Date: Thu, 27 Aug 2026 15:09:39 +0545 Subject: [PATCH 3/5] docs: present downloads as download endpoints, not an API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "Datastore Download" section read like another JSON action group, which is misleading: those two routes hand back a file and never the CKAN envelope. It now says so first — open one in a browser or curl it, there is nothing to parse — before covering the formats, the signed-URL redirect and the sharded-parquet zip. JSON is named only as what an error looks like. Renamed to "Datastore Downloads" in both the tag list and the router, since the two strings have to match or Swagger renders a second, untitled section. The "Datastore" group gets the same treatment: it now states that it is the JSON action API and what its envelope looks like, replacing a description that said "API endpoint" and carried a "searchsearch_sql" typo. Co-Authored-By: Claude Opus 5 (1M context) --- datastore/api/docs.py | 23 ++++++++++++++++++++--- datastore/api/endpoints/dump.py | 2 +- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/datastore/api/docs.py b/datastore/api/docs.py index d7ef8dd..cc519cf 100644 --- a/datastore/api/docs.py +++ b/datastore/api/docs.py @@ -284,10 +284,27 @@ def api_description(auth_type: str) -> str: }, { "name": "Datastore", - "description": ("API endpoint - create, upsert, delete, searchsearch_sql, and info."), + "description": ( + "JSON action API — create, upsert, delete, search, search_sql " + "and info. Every response is the CKAN envelope: `help`, " + "`success`, and either `result` or `error`." + ), }, { - "name": "Datastore Download", - "description": "Bulk download of an entire resource in available formats.", + "name": "Datastore Downloads", + "description": ( + "**Download endpoints — these return a file, not JSON.** Open " + "one in a browser or hand it to `curl -L`; there is no " + "envelope to parse and no `result` object.\n\n" + "Export a whole resource or the result of a SQL `SELECT` as " + "csv, gzip, ndjson or parquet. A download answers `302` with " + "a short-lived signed storage URL, so the bytes stream " + "straight from storage rather than through this API and the " + "transfer is resumable. The one exception is a parquet export " + "large enough to shard: those parts cannot be merged, so they " + "come back as a single streamed zip.\n\n" + "JSON appears only when something fails — an error is the " + "usual CKAN envelope." + ), }, ] diff --git a/datastore/api/endpoints/dump.py b/datastore/api/endpoints/dump.py index a9ac0f2..47e4a73 100644 --- a/datastore/api/endpoints/dump.py +++ b/datastore/api/endpoints/dump.py @@ -24,7 +24,7 @@ from datastore.services.read import dump_sql_datastore from datastore.services.streaming import zip_archive_writer -router = APIRouter(tags=["Datastore Download"], responses=ERROR_RESPONSES) +router = APIRouter(tags=["Datastore Downloads"], responses=ERROR_RESPONSES) # What a download actually returns, for OpenAPI. Without an explicit From 608956af4a389096a627251155b3bc8a467e8b49 Mon Sep 17 00:00:00 2001 From: Sagar Ghimire Date: Thu, 27 Aug 2026 15:11:39 +0545 Subject: [PATCH 4/5] docs: tighten the downloads blurb and the sql parameter text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The section description ran to three paragraphs, which left the header block taller than the operation it introduced. It says the essential thing in one paragraph now — these return a file, here are the formats, the answer is a 302 to a signed URL. The zip case and the error envelope are already documented on the responses themselves, where a reader looks for them. The `sql` parameter described itself as "A Datastore read API with `SELECT` / `WITH` statement" on both routes — reads as though the parameter were an API, with a doubled space and, on the dump route, a missing one. It is just a statement, so it says that. Co-Authored-By: Claude Opus 5 (1M context) --- datastore/api/docs.py | 16 ++++------------ datastore/schemas/request.py | 4 ++-- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/datastore/api/docs.py b/datastore/api/docs.py index cc519cf..2688e87 100644 --- a/datastore/api/docs.py +++ b/datastore/api/docs.py @@ -293,18 +293,10 @@ def api_description(auth_type: str) -> str: { "name": "Datastore Downloads", "description": ( - "**Download endpoints — these return a file, not JSON.** Open " - "one in a browser or hand it to `curl -L`; there is no " - "envelope to parse and no `result` object.\n\n" - "Export a whole resource or the result of a SQL `SELECT` as " - "csv, gzip, ndjson or parquet. A download answers `302` with " - "a short-lived signed storage URL, so the bytes stream " - "straight from storage rather than through this API and the " - "transfer is resumable. The one exception is a parquet export " - "large enough to shard: those parts cannot be merged, so they " - "come back as a single streamed zip.\n\n" - "JSON appears only when something fails — an error is the " - "usual CKAN envelope." + "**These return a file, not JSON** — open one in a browser or " + "hand it to `curl -L`. Export a whole resource or a SQL " + "`SELECT` as csv, gzip, ndjson or parquet; the response is a " + "`302` to a short-lived signed storage URL." ), }, ] diff --git a/datastore/schemas/request.py b/datastore/schemas/request.py index c0fa9e1..51ac014 100644 --- a/datastore/schemas/request.py +++ b/datastore/schemas/request.py @@ -327,7 +327,7 @@ class DatastoreSearchSQLRequest(BaseModel): _REQUIRE_LIMIT: ClassVar[bool] = True sql: str = Field( - description=("A Datastore read API with `SELECT` / `WITH` statement."), + description=("A read-only `SELECT` / `WITH` statement."), examples=['SELECT * FROM "balancing_auction_results_2025" WHERE accepted = true LIMIT 100'], ) @@ -434,7 +434,7 @@ class DatastoreDumpSQLRequest(DatastoreSearchSQLRequest): _REQUIRE_LIMIT: ClassVar[bool] = False sql: str = Field( - description=("A Datastore read API with`SELECT` / `WITH` statement."), + description=("A read-only `SELECT` / `WITH` statement to export."), examples=['SELECT * FROM "balancing_auction_results_2025" WHERE accepted = true'], ) From 74f995b2b6e447a521f4396b76e0b8a0fd19b37a Mon Sep 17 00:00:00 2001 From: Sagar Ghimire Date: Thu, 27 Aug 2026 15:16:27 +0545 Subject: [PATCH 5/5] docs: cut the section blurbs to one line each Both descriptions still ran long enough to wrap over the operations they introduce. One line each now, matching Health: what the group returns, and nothing else. The formats, the zip case and the error envelope are on the responses themselves. Co-Authored-By: Claude Opus 5 (1M context) --- datastore/api/docs.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/datastore/api/docs.py b/datastore/api/docs.py index 2688e87..6d78c30 100644 --- a/datastore/api/docs.py +++ b/datastore/api/docs.py @@ -284,19 +284,12 @@ def api_description(auth_type: str) -> str: }, { "name": "Datastore", - "description": ( - "JSON action API — create, upsert, delete, search, search_sql " - "and info. Every response is the CKAN envelope: `help`, " - "`success`, and either `result` or `error`." - ), + "description": "JSON action API — every response is the CKAN envelope.", }, { "name": "Datastore Downloads", "description": ( - "**These return a file, not JSON** — open one in a browser or " - "hand it to `curl -L`. Export a whole resource or a SQL " - "`SELECT` as csv, gzip, ndjson or parquet; the response is a " - "`302` to a short-lived signed storage URL." + "File downloads — a `302` to a signed URL, not the JSON envelope." ), }, ]