diff --git a/.lychee.toml b/.lychee.toml index b99ea03..acab7d3 100644 --- a/.lychee.toml +++ b/.lychee.toml @@ -10,5 +10,7 @@ exclude = [ # GitHub issue creation requires login "^https://github\\.com/.*/issues/new", # DICOM NEMA server - "^https://dicom\\.nema\\.org" + "^https://dicom\\.nema\\.org", + # IDC MCP endpoint: an MCP transport, not a page (405 to HEAD, 406 to GET) + "^https://api\\.imaging\\.datacommons\\.cancer\\.gov/mcp" ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f7c788..3e423e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,46 @@ All notable changes to the Imaging Data Commons Skill are documented in this fil The format is based on [Keep a Changelog](https://keepachangelog.com/), and this project adheres to [Semantic Versioning](https://semver.org/). +## [1.9.0] - 2026-10-08 + +IDC v25, `idc-index` 0.13.0 (`idc-index-data` 25.0.0), REST/MCP API `3.0.0b5`. The package bump +carries no API change; the substance of the release is **data provenance**. + +### Added + +- **Provenance** (`index_tables_guide.md`, `sql_patterns.md`, `SKILL.md`): who contributed, + de-identified, and DICOM-converted each part of the archive. Recorded **per source, keyed on + `source_DOI`** — `collections_index.sources` is a list of structs (unnest it) with a nested + `provenance`; `analysis_results_index.provenance` is a new column. A `NULL` struct means that + source is an analysis result, so look it up in the other table — across the pair, coverage is + complete. `NOT_APPLICABLE` / `UNKNOWN` / `NOT_DOCUMENTED` are values, not nulls +- Four `index` columns the guides never carried: `sop_class_name`, `transfer_syntax_name`, + `PhotometricInterpretation`, `PixelRepresentation`. All four are comma-separated when instances + in a series disagree (~72 000 series) and the latter two are `NULL` for SEG/SR/RTSTRUCT, so `=` + silently drops mixed-encoding series. `sop_class_name`, not `SOPClassUID`, is filterable +- Hugging Face mirror of the Parquet artifacts + ([`ImagingDataCommons/idc-index-data`](https://huggingface.co/datasets/ImagingDataCommons/idc-index-data), + public, no token). A fallback only — revision tags are incomplete, so pin via GCS +- Four missing Parquet artifacts: `sm_instance_index`, `version_metadata_index`, + `gdc_idc_mapping` (the imaging/genomics join), `tcia_idc_subset` +- `TestProvenance`, covering the unnest path, the NULL-is-a-pointer rule, and `tcga_brca` + +### Changed + +- Pinned minimum `idc-index` **0.12.5 → 0.13.0**; documented data version **v24 → v25**. Older + installs report `v24` and cannot download series added in v25 +- Archive totals **176 → 179 collections**, ~99 → **~100 TB**; mixed-license collections still 39 +- API claims **`3.0.0b3` → `3.0.0b5`**; tool inventory, the 19 filterable attributes, and all REST + limits re-verified unchanged +- BigQuery example `idc_v24` → `idc_v25`. The DICOMweb store stays `idc-store-v24` — stores lag a + release by 1-2 weeks and `idc-store-v25` is not populated yet, which `dicomweb_guide.md` now + warns about at the point of use + +### Fixed + +- `test_snippets.py` hardcoded `"v24"`, needing a hand edit every data release. It now reads + `idc-data-version` from the frontmatter, so a stale pin fails CI instead of stale guides + ## [1.8.3] - 2026-09-03 A stale index returns zero rows rather than an error, and `scripts/check_version.py` was diff --git a/README.md b/README.md index 54e666f..4483afb 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ An AI agent skill for querying and downloading public cancer imaging data from t - Check data licenses and generate proper citations - Generate download commands, either with the `idc-index` package or as direct transfers from the public S3/GCS buckets - Provide links to browser-based DICOM viewers for data preview +- Trace data provenance — who contributed each part of a dataset, who de-identified it, and whether IDC converted it to DICOM - Answer questions about IDC data structure and DICOM metadata ## What You Need @@ -24,6 +25,7 @@ Once the skill is loaded, you can ask questions like: - "How do I download all breast MRI data with commercial-use licenses?" - "Show me the available collections in IDC and their sizes" - "Generate a citation for the TCGA-BRCA collection" +- "Who de-identified the pathology images in TCGA-BRCA, and did IDC convert them to DICOM?" ## Reporting Issues diff --git a/SKILL.md b/SKILL.md index ecd64ab..b1383de 100644 --- a/SKILL.md +++ b/SKILL.md @@ -3,10 +3,10 @@ name: imaging-data-commons description: Query and download public cancer imaging data from NCI Imaging Data Commons. Invoke for any question about IDC collections, cancer imaging datasets, DICOM data access, radiology (CT, MR, PET) or pathology AI training sets, metadata queries, visualization, or license checks — even when the user doesn't explicitly mention "IDC". No authentication required. license: This skill is provided under the MIT License. IDC data itself has individual licensing (mostly CC-BY, some CC-NC) that must be respected when using the data. metadata: - version: 1.8.3 + version: 1.9.0 skill-author: Andrey Fedorov, @fedorov - idc-index: "0.12.5" - idc-data-version: "v24" + idc-index: "0.13.0" + idc-data-version: "v25" repository: https://github.com/ImagingDataCommons/imaging-data-commons-skill --- @@ -18,7 +18,7 @@ Query and download public cancer imaging data from the National Cancer Institute **Expected network access:** IDC metadata is reachable three ways — a local DuckDB index shipped with the `idc-index` Python package (no network), or the hosted IDC service over MCP or REST (`api.imaging.datacommons.cancer.gov`, no authentication). File downloads use public GCS (`storage.googleapis.com`) and AWS S3 (`s3.amazonaws.com`) — no authentication required. DICOMweb access uses either the public IDC proxy (`proxy.imaging.datacommons.cancer.gov`, no auth) or the Google Cloud Healthcare API (`healthcare.googleapis.com`, requires GCP authentication). Optional BigQuery queries (`bigquery.googleapis.com`) also require GCP authentication. No credentials or environment variables are accessed by this skill. -**Current IDC Data Version: v24** (always verify — see *Best Practices*) +**Current IDC Data Version: v25** (always verify — see *Best Practices*) **Choose the access path first.** There is no single default: the cheapest correct path depends on the session and the task. @@ -50,12 +50,12 @@ calls for it. `check_version.py` never installs anything itself — it also flag import re, idc_index from idc_index import IDCClient # Repeat the startup check where it cannot be skipped — see references/cli_guide.md. -if [int(re.sub(r"\D.*", "", p) or 0) for p in idc_index.__version__.split(".")[:3]] < [0, 12, 5]: - raise RuntimeError(f"idc-index {idc_index.__version__} is stale (need 0.12.5): a stale " +if [int(re.sub(r"\D.*", "", p) or 0) for p in idc_index.__version__.split(".")[:3]] < [0, 13, 0]: + raise RuntimeError(f"idc-index {idc_index.__version__} is stale (need 0.13.0): a stale " "index returns zero rows, not an error. Run scripts/check_version.py.") client = IDCClient() -print(f"IDC data version: {client.get_idc_version()}") # should be "v24" +print(f"IDC data version: {client.get_idc_version()}") # should be "v25" ``` **Core workflow:** query metadata with `client.sql_query()` → download with @@ -158,14 +158,12 @@ Always call `client.fetch_index("table_name")` before querying any index table | Microscopy | `sm_index`, `sm_instance_index` | 1 row = 1 SM series / instance | | Geometry, clinical, history | `volume_geometry_index`, `clinical_index`, `version_metadata_index`, `prior_versions_index` | see guide | -`references/index_tables_guide.md` has the full inventory with each table's columns and -contents — load it when you need to know what a specialized table actually holds. +`references/index_tables_guide.md` has the full inventory, each table's columns, and join examples. -**`prior_versions_index` is for reproducibility only.** It contains series permanently *removed* -from IDC, with zero overlap with `index`. Use it only to reproduce work against a prior IDC -version. Do NOT use it for version history or "what's new" questions — those use -`series_init_idc_version` / `series_revised_idc_version` in the main `index` table, which are -not equivalent to this table's `min_idc_version` / `max_idc_version`. +**`prior_versions_index` is for reproducibility only** — series permanently *removed* from IDC, +zero overlap with `index`. Do NOT use it for version history or "what's new": those use +`series_init_idc_version` / `series_revised_idc_version` in `index`, which are not equivalent to +this table's `min_idc_version` / `max_idc_version`. ### Joining Tables @@ -175,13 +173,15 @@ not equivalent to this table's `min_idc_version` / `max_idc_version`. |-------------|--------|----------| | `collection_id` | index, prior_versions_index, collections_index, clinical_index | Link series to collection metadata or clinical data | | `analysis_result_id` | index, analysis_results_index | Link series to analysis result metadata (annotations, segmentations) | -| `source_DOI` | index, analysis_results_index | Link by publication DOI | +| `source_DOI` | index, analysis_results_index, `collections_index.sources` | Link by publication DOI; also the key for provenance (below) | | `segmented_SeriesInstanceUID` | seg_index → index | Link segmentation to its source image series (`seg_index.segmented_SeriesInstanceUID = index.SeriesInstanceUID`) | | `referenced_SeriesInstanceUID` | ann_index → index, rtstruct_index → index | Link annotation or RTSTRUCT to its source image series | -**Note:** `subjects`, `updated`, and `description` appear in multiple tables but have different meanings (counts vs identifiers, different update contexts). Joining `prior_versions_index` to `index` on `SeriesInstanceUID` always returns zero rows — see the warning above. +**Note:** `subjects`, `updated`, and `description` mean different things in different tables; joining `prior_versions_index` to `index` on `SeriesInstanceUID` always returns zero rows. -For detailed join examples, schema discovery patterns, key columns reference, and DataFrame access, see `references/index_tables_guide.md`. +**Provenance (new in v25) is recorded per source, not per collection**, keyed on `source_DOI`. `collections_index.sources` is a list of structs (unnest it) whose `provenance` names who contributed, de-identified, and DICOM-converted that component; `analysis_results_index.provenance` is a plain struct. + +A NULL provenance struct in `collections_index` means that source is an analysis result — look it up in `analysis_results_index`, where it is always populated. `NOT_APPLICABLE` and `UNKNOWN` are values, not nulls. See `references/index_tables_guide.md`. ### Clinical Data Access @@ -244,8 +244,8 @@ v3 only:** V1 and V2 are superseded and scheduled for shutdown, so port any `/v1 `Modality_btw`-style example a user brings rather than extending it. Both sides build on `idc-index-data`, so compare the API's `idc_index_data_version` against local -`idc_index_data.__version__` before mixing them: the **major is the IDC data release** (`24.x.y` -serves `v24`), so differing minor/patch means the series are identical. If the API is a whole +`idc_index_data.__version__` before mixing them: the **major is the IDC data release** (`25.x.y` +serves `v25`), so differing minor/patch means the series are identical. If the API is a whole release ahead, `idc-index` **cannot download the extra series** — it silently skips what its own index does not list — so either upgrade it (run `scripts/check_version.py` for the right command) or transfer directly from the bucket with `s5cmd --no-sign-request`. @@ -417,7 +417,7 @@ for citation in client.citations_from_selection(collection_id="rider_pilot"): ``` About 97% of IDC data is CC BY (commercial use allowed with attribution) and about 3% is -CC BY-NC (non-commercial only). **Licenses attach to series, not collections** — 39 of 176 +CC BY-NC (non-commercial only). **Licenses attach to series, not collections** — 39 of 179 collections carry more than one — so check the selection you actually intend to use, and note that the most restrictive term governs a mixed cohort. @@ -443,7 +443,7 @@ idc-index equivalent. - **Check schema before writing queries** — Use `client.get_index_schema('index')` (reads cached metadata, no SQL executed) or `client.indices_overview` to see all available columns and their descriptions. The version-tracking columns `series_init_idc_version` and `series_revised_idc_version` in the main `index` table directly answer "what's new / when was this added" questions without touching `prior_versions_index`. - **Never use web search for IDC data content questions** - Always query the IDC index directly, via `client.sql_query()` locally or `POST /v3/sql` over HTTP. Web sources (release notes, blog posts, documentation pages) are frequently out of date and will produce incorrect answers. The index is the authoritative source; use it even when web search is available. -- **Verify the IDC data version at the start of a session** - `client.get_idc_version()`, `GET /v3/version`, or the MCP `get_idc_version` tool, depending on the path in use (currently v24). For a stale local index, run `scripts/check_version.py` and use the upgrade command it prints +- **Verify the IDC data version at the start of a session** - `client.get_idc_version()`, `GET /v3/version`, or the MCP `get_idc_version` tool, depending on the path in use (currently v25). For a stale local index, run `scripts/check_version.py` and use the upgrade command it prints - **Check licenses and generate citations** - Query `license_short_name` and respect CC BY vs CC BY-NC terms; use `citations_from_selection()` to produce citations from `source_DOI` for publications - **Explore small, then commit** - Use `LIMIT` (or a low `max_rows`) while exploring, and check collection size before downloading — some collections are terabytes. See `references/cli_guide.md` - **Keep downloads reproducible** - Organize with `dirTemplate` (e.g. `%collection_id/%PatientID/%Modality`) and save the Series UIDs or manifest behind any dataset you build diff --git a/references/bigquery_guide.md b/references/bigquery_guide.md index 6401c2e..c6a7ee6 100644 --- a/references/bigquery_guide.md +++ b/references/bigquery_guide.md @@ -1,6 +1,6 @@ # BigQuery Guide for IDC -**Tested with:** `bigquery-public-data.idc_current` and idc-index 0.12.5 (IDC data version v24) +**Tested with:** `bigquery-public-data.idc_current` and idc-index 0.13.0 (IDC data version v25) For most queries and downloads, use `idc-index` (see main SKILL.md). This guide covers BigQuery for advanced use cases requiring full DICOM metadata or complex joins. diff --git a/references/cli_guide.md b/references/cli_guide.md index eda4b54..cec9887 100644 --- a/references/cli_guide.md +++ b/references/cli_guide.md @@ -30,7 +30,7 @@ resolve and still produces no check. **Confirm the outcome, not the command.** A pass prints ``` -idc-index meets pinned minimum (0.12.5) +idc-index meets pinned minimum (0.13.0) ``` and exits 0. Anything else — `command not found`, a permission error (the bundled script is not diff --git a/references/clinical_data_guide.md b/references/clinical_data_guide.md index 3b2d9e3..3282214 100644 --- a/references/clinical_data_guide.md +++ b/references/clinical_data_guide.md @@ -1,6 +1,6 @@ # Clinical Data Guide for IDC -**Tested with:** idc-index 0.12.5 (IDC data version v24) +**Tested with:** idc-index 0.13.0 (IDC data version v25) Clinical data (demographics, diagnoses, therapies, lab tests, staging) accompanies many IDC imaging collections. This guide covers how to discover, access, and integrate clinical data with imaging data using `idc-index`. diff --git a/references/cloud_storage_guide.md b/references/cloud_storage_guide.md index 6342d0f..716d0ec 100644 --- a/references/cloud_storage_guide.md +++ b/references/cloud_storage_guide.md @@ -205,7 +205,7 @@ IDC releases new data versions every 2-4 months. The versioning system ensures r ### How Versioning Works -1. **Snapshots**: Each IDC version (v1, v2, ..., v24, etc.) represents a complete snapshot of all data at release time +1. **Snapshots**: Each IDC version (v1, v2, ..., v25, etc.) represents a complete snapshot of all data at release time 2. **UUID-based**: When data changes, new CRDC UUIDs are assigned; old UUIDs remain accessible 3. **Cumulative buckets**: All versions coexist in the same buckets—old series folders @@ -223,7 +223,7 @@ IDC releases new data versions every 2-4 months. The versioning system ensures r For querying version-specific metadata, BigQuery provides versioned tables. See `bigquery_guide.md` for details. - `bigquery-public-data.idc_current` — alias to latest version -- `bigquery-public-data.idc_v24` — specific version (replace 24 with desired version) +- `bigquery-public-data.idc_v25` — specific version (replace 25 with desired version) ### Reproducing a Previous Analysis diff --git a/references/dicomweb_guide.md b/references/dicomweb_guide.md index 8eab369..53f11a0 100644 --- a/references/dicomweb_guide.md +++ b/references/dicomweb_guide.md @@ -34,14 +34,19 @@ https://proxy.imaging.datacommons.cancer.gov/current/viewer-only-no-downloads-se https://healthcare.googleapis.com/v1/projects/nci-idc-data/locations/us-central1/datasets/idc/dicomStores/idc-store-v{VERSION}/dicomWeb ``` -Replace `{VERSION}` with the IDC release number. To find the current version: +Replace `{VERSION}` with the IDC release number: ```python from idc_index import IDCClient client = IDCClient() -print(client.get_idc_version()) # e.g., "v24" for current version +print(client.get_idc_version()) # e.g., "v25" for current version ``` +**The store lags the data release — do not assume `get_idc_version()` names an existing store.** +Stores are populated 1-2 weeks after a release, so the newest store is often the previous +version (as of IDC v25, `idc-store-v24` is the latest populated one). Treat the current data +version as an upper bound, and fall back to the preceding version if a store 404s. + - **~96% data coverage** - Only replicates data from `idc-open-data` bucket (missing ~4% from other buckets) - **Updates 1-2 weeks after** IDC releases - Requires authentication and provides higher quotas diff --git a/references/digital_pathology_guide.md b/references/digital_pathology_guide.md index e2232f0..941ff16 100644 --- a/references/digital_pathology_guide.md +++ b/references/digital_pathology_guide.md @@ -1,6 +1,6 @@ # Digital Pathology Guide for IDC -**Tested with:** idc-index 0.12.5 (IDC data version v24) +**Tested with:** idc-index 0.13.0 (IDC data version v25) For general IDC queries and downloads, use `idc-index` (see main SKILL.md). This guide covers slide microscopy (SM) imaging, microscopy bulk simple annotations (ANN), and segmentations (SEG) in the context of digital pathology in IDC. diff --git a/references/index_tables_guide.md b/references/index_tables_guide.md index d89ff9b..651c51d 100644 --- a/references/index_tables_guide.md +++ b/references/index_tables_guide.md @@ -1,6 +1,6 @@ # Index Tables Guide for IDC -**Tested with:** idc-index 0.12.5 (IDC data version v24) +**Tested with:** idc-index 0.13.0 (IDC data version v25) This guide covers the structure and access patterns for IDC index tables: programmatic schema discovery, DataFrame access, and join column references. For the overview of available tables and their purposes, see the "Index Tables" section in the main SKILL.md. @@ -160,9 +160,97 @@ Most common columns in the primary `index` table (use `indices_overview` for com | `instanceCount` | INTEGER | No | Number of DICOM instances in series | | `SOPClassUID` | STRING | Yes | DICOM SOP Class UID (identifies the object/service class, e.g., CT Image Storage) | | `TransferSyntaxUID` | STRING | Yes | DICOM Transfer Syntax UID (encoding/compression method) | +| `sop_class_name` | STRING | No | Human-readable `SOPClassUID` (e.g., "CT Image Storage", "Segmentation Storage"); the filterable form, and more specific than `Modality` for distinguishing object types | +| `transfer_syntax_name` | STRING | No | Human-readable `TransferSyntaxUID` (e.g., "JPEG 2000", "Explicit VR Little Endian") | +| `PhotometricInterpretation` | STRING | Yes | Intended interpretation of pixel data — `MONOCHROME2` (grayscale), `RGB` / `YBR_FULL_422` (color) | +| `PixelRepresentation` | STRING | Yes | Pixel sample representation: `"0"` unsigned, `"1"` signed | + +**Pixel-encoding columns are per-series summaries, not per-instance values.** `TransferSyntaxUID`, +`transfer_syntax_name`, `PhotometricInterpretation`, and `PixelRepresentation` are +**comma-separated** when instances within one series differ — about 72 000 series in v25, mostly +SM — and the latter two are `NULL` for non-image objects (SEG, SR, RTSTRUCT). Match them with +`LIKE '%...%'` or split before comparing; an `=` test silently drops every mixed-encoding series. +Filter on `sop_class_name` rather than parsing `SOPClassUID`: it is one of the 19 attributes the +cohort and REST filter APIs accept, while the raw UID is not. **DICOM = Yes**: Column value extracted from the DICOM attribute with the same name. Refer to the [DICOM standard](https://dicom.nema.org/medical/dicom/current/output/chtml/part06/chapter_6.html) for numeric tag mappings. Use standard DICOM knowledge for expected values and formats. +## Data Provenance (new in v25) + +IDC v25 added a structured, queryable record of **where each piece of data came from and who +handled it** — previously answerable only by heuristics over `collection_id` and DOI strings. +It lives in two places, both as a `provenance` STRUCT with the same four fields: + +| Field | Meaning | +|-------|---------| +| `data_contributor` | Who contributed the data to IDC (`TCIA`, `IDC`) | +| `source_data_provider` | Who held the upstream source material (`GDC`, `TCIA`, `HTAN`, `NLM`, …) | +| `deidentification_party` | Who performed de-identification (`TCIA`, `GDC`, `HTAN`, `NCH`, …) | +| `dicom_conversion_by` | Who produced the DICOM representation | + +**Provenance is per-source, not per-collection.** A collection routinely mixes components with +different lineage — TCGA radiology arrives as DICOM from TCIA while its pathology originates as +vendor SVS from GDC and is converted by IDC under a separate Zenodo DOI. The unit is therefore +`source_DOI`: + +- `collections_index.sources` is a **list of structs**, one per source, each carrying its own + `license`, `citation`, and `provenance`. Unnest before reaching into it. +- `analysis_results_index.provenance` is a **plain struct** on the row (new column in v25). + +### Reading it + +```python +client.fetch_index("collections_index") + +# Who de-identified each component of a collection, and did IDC convert it? +client.sql_query(""" + SELECT collection_id, + src.source_doi, + src.provenance.data_contributor, + src.provenance.deidentification_party, + src.provenance.dicom_conversion_by + FROM (SELECT collection_id, unnest(sources) AS src FROM collections_index) + WHERE collection_id = 'tcga_brca' +""") +``` + +Attach provenance to actual series by joining `index.source_DOI` to the unnested `source_doi` +(the DOIs match exactly; lowercase both sides to be safe): + +```python +client.sql_query(""" + SELECT i.collection_id, i.Modality, COUNT(*) AS series, + s.src.provenance.data_contributor AS contributed_by, + s.src.provenance.dicom_conversion_by AS converted_by + FROM index i + JOIN (SELECT collection_id, unnest(sources) AS src FROM collections_index) s + ON s.collection_id = i.collection_id + AND lower(s.src.source_doi) = lower(i.source_DOI) + GROUP BY ALL +""") +``` + +### Two traps + +**`NULL` in `collections_index` does not mean "unknown".** 92 of the 308 sources in v25 have a +`NULL` provenance struct, and all 92 are analysis-result DOIs whose provenance lives in +`analysis_results_index.provenance` instead — which is populated for all 26 rows. Across the two +tables coverage is complete, so a `NULL` here is a **pointer to the other table**, not missing +data. Look the `source_DOI` up there before reporting provenance as unavailable. + +**Absent and unknown are different values.** The vocabulary uses explicit sentinels, so test for +them rather than treating any non-empty string as an answer: + +| Sentinel | Means | +|----------|-------| +| `NOT_APPLICABLE` | The step did not happen — e.g. `dicom_conversion_by` for the 141 sources that arrived already as DICOM | +| `UNKNOWN` | The step happened but the party is not recorded | +| `NOT_DOCUMENTED` | Seen in `analysis_results_index.dicom_conversion_by` | + +In v25, `data_contributor` is `TCIA` for 141 sources and `IDC` for 75; the 75 IDC-contributed +sources are exactly those with `dicom_conversion_by = 'IDC'`, since IDC contributing a component +implies IDC converted it from a non-DICOM upstream source. + ## Join Column Reference Use this table to identify join columns between index tables. Always call `client.fetch_index("table_name")` before using a table in SQL. @@ -170,6 +258,8 @@ Use this table to identify join columns between index tables. Always call `clien | Table A | Table B | Join Condition | |---------|---------|----------------| | `index` | `collections_index` | `index.collection_id = collections_index.collection_id` | +| `index` | `collections_index.sources` (provenance) | `index.collection_id = collections_index.collection_id` AND `lower(index.source_DOI) = lower(src.source_doi)` after `unnest(sources) AS src` | +| `index` | `analysis_results_index` | `lower(index.source_DOI) = lower(analysis_results_index.source_DOI)` | | `index` | `sm_index` | `index.SeriesInstanceUID = sm_index.SeriesInstanceUID` | | `index` | `seg_index` | `index.SeriesInstanceUID = seg_index.segmented_SeriesInstanceUID` | | `index` | `ann_index` | `index.SeriesInstanceUID = ann_index.SeriesInstanceUID` | diff --git a/references/licensing_and_citation.md b/references/licensing_and_citation.md index 9891c88..651bc82 100644 --- a/references/licensing_and_citation.md +++ b/references/licensing_and_citation.md @@ -42,7 +42,7 @@ About 97% of IDC data by size permits commercial reuse; just under 3% is non-com any `license_short_name` that is not a recognizable Creative Commons string as custom, and report the exact value to the user rather than assuming it permits reuse. -**Licenses attach to individual series, not to whole collections.** 39 of IDC's 176 collections +**Licenses attach to individual series, not to whole collections.** 39 of IDC's 179 collections carry more than one license — analysis results and original images within one collection can differ, as can series from different sources. Never conclude that a collection is commercially usable from one series, or from the collection's headline license: group by diff --git a/references/mcp_guide.md b/references/mcp_guide.md index 7f3ad99..462ced9 100644 --- a/references/mcp_guide.md +++ b/references/mcp_guide.md @@ -49,7 +49,7 @@ needs downloads or local analysis. Tool names may change as the server matures. ## Tool inventory -Verified against server version `3.0.0b3`. Treat this as a snapshot, not a contract — call +Verified against server version `3.0.0b5`. Treat this as a snapshot, not a contract — call the server's own listing rather than assuming this list is current. | Group | Tools | diff --git a/references/parquet_access_guide.md b/references/parquet_access_guide.md index caa25b1..a62cb94 100644 --- a/references/parquet_access_guide.md +++ b/references/parquet_access_guide.md @@ -1,6 +1,6 @@ # Direct Parquet Access Guide for IDC -**Tested with:** idc-index-data 24.2.2 (IDC data version v24), DuckDB 1.5 +**Tested with:** idc-index-data 25.0.0 (IDC data version v25), DuckDB 1.5 All idc-index metadata tables are published as Parquet files to a public GCS bucket with unrestricted CORS access. This enables metadata queries with DuckDB or pandas without installing idc-index. @@ -45,9 +45,38 @@ https://storage.googleapis.com/idc-index-data-artifacts/current/release_artifact | `mr_index.parquet` | — | MR sequence/acquisition parameters | | `pt_index.parquet` | — | PET acquisition/radiopharmaceutical parameters | | `prior_versions_index.parquet` | — | Series from previous IDC releases | +| `sm_instance_index.parquet` | ~18 MB | Instance-level slide microscopy metadata | +| `version_metadata_index.parquet` | — | IDC release version timestamps | + +Two artifacts in the same directory are **not** index tables and have no `list_tables` / +`fetch_index` equivalent, so they are reachable only here and from BigQuery: + +| File | Approximate Size | Description | +|------|-----------------|-------------| +| `gdc_idc_mapping.parquet` | ~1 MB | Maps IDC studies to GDC `gdc_case_id`, keyed by `collection_id` / `PatientID` / `StudyInstanceUID` — the join to use when pairing imaging with genomics or other GDC data | +| `tcia_idc_subset.parquet` | ~46 MB | The subset of IDC series originating from TCIA, with `series_aws_url` for direct transfer | **Note:** the main index file is named `idc_index.parquet`, not `index.parquet`. Reference it with an alias in SQL queries (e.g., `FROM read_parquet(...) AS index`). +## Hugging Face Mirror + +Since `idc-index-data` 25.0.0 the same artifacts are also published as a Hugging Face dataset, +[`ImagingDataCommons/idc-index-data`](https://huggingface.co/datasets/ImagingDataCommons/idc-index-data) +— public, no token, CC BY. The files carry the same names, flat at the repository root: + +``` +https://huggingface.co/datasets/ImagingDataCommons/idc-index-data/resolve/main/{filename}.parquet +``` + +Each index also ships a `{filename}_schema.json` alongside it, which is the cheapest way to read +column names and descriptions without fetching the Parquet itself. + +`main` tracks the current release. Revision tags are published for index releases but coverage is +incomplete, so **use the GCS versioned path, not a Hugging Face revision, when a query must be +pinned** — see *Pinning to a Specific Version*. Prefer this mirror when GCS is blocked, or when +the surrounding work already uses `datasets` / `polars`, which read it natively; otherwise GCS +remains the primary. + ## Prerequisites Install the Python `duckdb` package, using whatever installer manages the environment you are diff --git a/references/rest_api_guide.md b/references/rest_api_guide.md index 6b10a89..3afc010 100644 --- a/references/rest_api_guide.md +++ b/references/rest_api_guide.md @@ -1,6 +1,6 @@ # IDC REST API Guide -**Tested with:** API `3.0.0b3` (build `0640860`), IDC data version v24, `idc_index_data_version` 24.2.2 +**Tested with:** API `3.0.0b5` (build `a7abd9e`), IDC data version v25, `idc_index_data_version` 25.0.0 IDC operates a hosted REST API that exposes discovery, cohort building, metadata SQL, and download manifests over plain HTTP. No authentication, account, or credentials are required — @@ -57,7 +57,7 @@ running build rather than assuming the values in this guide: ```bash curl -s https://api.imaging.datacommons.cancer.gov/v3/version -# {"idc_version":"v24","idc_index_data_version":"24.2.2","api_version":"3.0.0b3","build":"0640860"} +# {"idc_version":"v25","idc_index_data_version":"25.0.0","api_version":"3.0.0b5","build":"a7abd9e"} ``` `idc_version` is the IDC data release the API serves and is the authority when the API is in @@ -101,7 +101,7 @@ So read a mismatch by its position: | Minor or patch (24.2.0 vs 24.2.2) | Same data release, different index build | Same series everywhere; downloads are unaffected. A metadata query can still differ if it touches a column that was added or corrected | Comparing `idc_version` alone cannot make this distinction in the other direction either — the -`vNN` label is exactly the major, so matching `v24` on both sides tells you the release agrees +`vNN` label is exactly the major, so matching `v25` on both sides tells you the release agrees but says nothing about the index build. When the two disagree, say so and name both versions, then reconcile rather than mixing @@ -199,7 +199,7 @@ of them. It has two parts: } ``` -Filters operate only on the `index` table's filterable attributes — 19 of them as of `3.0.0b3`: +Filters operate only on the `index` table's filterable attributes — 19 of them as of `3.0.0b5`: | Kind | Attributes | |------|------------| @@ -258,7 +258,7 @@ B=https://api.imaging.datacommons.cancer.gov/v3 curl -s $B/version # data release + API build curl -s $B/stats # headline totals -curl -s $B/collections # all 176 collections +curl -s $B/collections # all 179 collections curl -s $B/collections/rider_pilot # one collection: counts, modalities, licenses curl -s $B/analysis_results # derived datasets (segmentations, annotations) curl -s $B/attributes # what can be filtered, and how @@ -366,7 +366,7 @@ reason before retrying. Ground the schema with `GET /tables` and `GET /tables/{table}` — do not guess table or column names. -**Guardrails** (verified against `3.0.0b3`): +**Guardrails** (verified against `3.0.0b5`): - Only single read-only `SELECT` / `WITH … SELECT` statements are accepted. Anything else is rejected with `{"error": {"code": "invalid_query", "message": "Only read-only SELECT (or WITH ... SELECT) statements are allowed."}}` and HTTP 400. @@ -468,7 +468,7 @@ with the rest: ``` The total of N copy commands are not recognized as referencing any associated series in the -main index. ... they may correspond to files available in a release of IDC different from v24 +main index. ... they may correspond to files available in a release of IDC different from v25 used in this version of idc-index. ... The corresponding files could not be downloaded. @@ -510,12 +510,12 @@ S3-compatible endpoint, never a `gs://` URL. That is why `idc download-from-mani recognizes `s3://` lines. Driving `s5cmd` yourself, use `--no-sign-request`, and for `source=gcs` add `--endpoint-url https://storage.googleapis.com`. -IDC is ~99 TB across 176 collections. Always report `series` and `size_TB` from +IDC is ~100 TB across 179 collections. Always report `series` and `size_TB` from `cohort/counts` and confirm with the user before starting a broad download. ## Limits, Defaults, and Errors -Measured against `3.0.0b3`. Values above a cap are silently clamped — the response echoes the +Measured against `3.0.0b5`. Values above a cap are silently clamped — the response echoes the value actually used (`max_rows`, `page_size`), so read it back rather than assuming the request was honored. diff --git a/references/sql_patterns.md b/references/sql_patterns.md index 05691fe..593db72 100644 --- a/references/sql_patterns.md +++ b/references/sql_patterns.md @@ -1,6 +1,6 @@ # SQL Query Patterns for IDC -**Tested with:** idc-index 0.12.5 (IDC data version v24) +**Tested with:** idc-index 0.13.0 (IDC data version v25) Quick reference for common SQL query patterns when working with IDC data. For detailed examples with context, see the "Core Capabilities" section in the main SKILL.md. @@ -285,6 +285,65 @@ client.sql_query(""" """) ``` +## Data Provenance — "Where did this come from, and who de-identified it?" + +New in IDC v25. Provenance is recorded per **source** (`source_DOI`), not per collection, because +collections mix components with different lineage. `collections_index.sources` is a list of +structs, so unnest it; `analysis_results_index.provenance` is a plain struct. See +`references/index_tables_guide.md` for the field definitions and sentinel values. + +```python +client.fetch_index("collections_index") + +# Which datasets did IDC itself convert to DICOM (rather than receive as DICOM)? +client.sql_query(""" + SELECT collection_id, src.source_doi, + src.provenance.source_data_provider AS converted_from + FROM (SELECT collection_id, unnest(sources) AS src FROM collections_index) + WHERE src.provenance.dicom_conversion_by = 'IDC' + ORDER BY collection_id +""") + +# Who performed de-identification, across the archive? +client.sql_query(""" + SELECT src.provenance.deidentification_party AS party, COUNT(*) AS sources + FROM (SELECT unnest(sources) AS src FROM collections_index) + WHERE src.provenance.deidentification_party IS NOT NULL + GROUP BY party ORDER BY sources DESC +""") +``` + +Provenance for the series you actually selected — join on `source_DOI`: + +```python +client.sql_query(""" + SELECT i.collection_id, i.Modality, COUNT(*) AS series, + s.src.provenance.data_contributor AS contributed_by, + s.src.provenance.deidentification_party AS deidentified_by + FROM index i + JOIN (SELECT collection_id, unnest(sources) AS src FROM collections_index) s + ON s.collection_id = i.collection_id + AND lower(s.src.source_doi) = lower(i.source_DOI) + WHERE i.collection_id = 'tcga_brca' + GROUP BY ALL ORDER BY series DESC +""") +# tcga_brca splits by lineage: MR/MG contributed by TCIA (dicom_conversion_by NOT_APPLICABLE), +# SM converted by IDC from GDC source material under a separate Zenodo DOI. The SEG/ANN/SR rows +# come back NULL because they are analysis results — resolve those via analysis_results_index. +``` + +A `NULL` provenance struct in `collections_index` means the source is an **analysis result** — +look it up in `analysis_results_index`, where provenance is fully populated: + +```python +client.fetch_index("analysis_results_index") +client.sql_query(""" + SELECT analysis_result_id, source_DOI, + provenance.data_contributor, provenance.dicom_conversion_by + FROM analysis_results_index ORDER BY analysis_result_id +""") +``` + ## Troubleshooting **Issue:** Query returns error "table not found" diff --git a/references/use_cases.md b/references/use_cases.md index 46f5687..959e0ab 100644 --- a/references/use_cases.md +++ b/references/use_cases.md @@ -1,6 +1,6 @@ # Common Use Cases for IDC -**Tested with:** idc-index 0.12.5 (IDC data version v24) +**Tested with:** idc-index 0.13.0 (IDC data version v25) This guide provides complete end-to-end workflow examples for common IDC use cases. Each use case demonstrates the full workflow from query to download with best practices. diff --git a/scripts/check_version.py b/scripts/check_version.py index 921a31f..67129d5 100644 --- a/scripts/check_version.py +++ b/scripts/check_version.py @@ -21,8 +21,8 @@ import shutil import sys -MIN_VERSION = "0.12.5" # keep in sync with metadata.idc-index in SKILL.md -SKILL_VERSION = "1.8.3" # keep in sync with metadata.version in SKILL.md +MIN_VERSION = "0.13.0" # keep in sync with metadata.idc-index in SKILL.md +SKILL_VERSION = "1.9.0" # keep in sync with metadata.version in SKILL.md REPO = "ImagingDataCommons/imaging-data-commons-skill" _LEADING_DIGITS = re.compile(r"\d+") diff --git a/tests/requirements-test.txt b/tests/requirements-test.txt index 6cb28e4..8ffb632 100644 --- a/tests/requirements-test.txt +++ b/tests/requirements-test.txt @@ -1,4 +1,4 @@ -idc-index==0.12.5 +idc-index==0.13.0 pandas pytest pytest-timeout diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index b833cdf..0a197f7 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -7,7 +7,7 @@ independently of this repository, so they can go stale silently. These tests read the expectations out of the documentation and check them against the live server. -The server is at beta (3.0.0b3) and its tool set may still move. A drift failure here means +The server is at beta (3.0.0b5) and its tool set may still move. A drift failure here means the docs need updating, not that the skill is broken — the skill falls back to idc-index whenever identification fails. diff --git a/tests/test_rest_api.py b/tests/test_rest_api.py index b9ffac5..d6f119e 100644 --- a/tests/test_rest_api.py +++ b/tests/test_rest_api.py @@ -3,7 +3,7 @@ The guide documents an API that versions independently of this repository — endpoint paths, filterable attributes, request body shapes, and row caps are all a contract with a beta -service (3.0.0b3) that can move without notice. These tests parse the expectations out of the +service (3.0.0b5) that can move without notice. These tests parse the expectations out of the guide and check them against the live API, so drift shows up as a CI failure rather than as wrong instructions to an agent. diff --git a/tests/test_snippets.py b/tests/test_snippets.py index f6716ad..3063498 100644 --- a/tests/test_snippets.py +++ b/tests/test_snippets.py @@ -16,6 +16,8 @@ """ import os +import pathlib +import re import sys import duckdb @@ -77,8 +79,19 @@ def test_package_version_meets_minimum(self): >= check_version.parse_version(check_version.MIN_VERSION) ), f"idc-index {idc_index.__version__} < pinned minimum {check_version.MIN_VERSION}" - def test_idc_data_version_is_v24(self, client): - assert client.get_idc_version() == "v24" + def test_idc_data_version_matches_frontmatter(self, client): + """The installed index must serve the release SKILL.md claims to document. + + Reading the frontmatter rather than hardcoding the version means a data + release bumps one place (SKILL.md) instead of silently leaving the docs + describing an older release than the pinned idc-index actually serves. + """ + skill = pathlib.Path(__file__).resolve().parents[1] / "SKILL.md" + declared = re.search( + r'^ idc-data-version:\s*"?(v\d+)"?', skill.read_text(), re.M + ) + assert declared, "SKILL.md frontmatter has no idc-data-version" + assert client.get_idc_version() == declared.group(1) def test_series_version_columns_present(self, client): cols = client.index.columns.tolist() @@ -1046,6 +1059,8 @@ def test_listed_files_exist(self): "sm_index", "contrast_index", "ann_index", "ann_group_index", "collections_index", "analysis_results_index", "clinical_index", "ct_index", "mr_index", "pt_index", "prior_versions_index", + "sm_instance_index", "version_metadata_index", + "gdc_idc_mapping", "tcia_idc_subset", ]: url = f"{PARQUET_BASE}/{name}.parquet" request = urllib.request.Request(url, method="HEAD") @@ -1142,3 +1157,74 @@ def test_rtstruct_collections(self): LIMIT 10 """).df() assert len(df) == 10 + + +class TestProvenance: + """references/index_tables_guide.md + sql_patterns.md — provenance, new in IDC v25. + + Guards the two claims the guides make that a reader would otherwise have to + take on faith: that provenance is reachable by unnesting collections_index, + and that a NULL struct there is a pointer to analysis_results_index rather + than missing data. + """ + + PROV_FIELDS = ( + "data_contributor", + "source_data_provider", + "deidentification_party", + "dicom_conversion_by", + ) + + def test_provenance_struct_fields_present(self, client_with_all_indices): + fields = ", ".join(f"src.provenance.{f}" for f in self.PROV_FIELDS) + df = client_with_all_indices.sql_query( + f"SELECT {fields} " + "FROM (SELECT unnest(sources) AS src FROM collections_index) LIMIT 5" + ) + assert list(df.columns) == list(self.PROV_FIELDS) + + def test_analysis_results_provenance_fully_populated(self, client_with_all_indices): + df = client_with_all_indices.sql_query( + "SELECT COUNT(*) AS total, " + "COUNT(provenance.data_contributor) AS with_prov " + "FROM analysis_results_index" + ) + assert df["total"][0] > 0 + assert df["with_prov"][0] == df["total"][0] + + def test_null_collection_provenance_is_an_analysis_result(self, client_with_all_indices): + """The guides tell readers to treat NULL as a pointer, not as unknown.""" + df = client_with_all_indices.sql_query(""" + WITH s AS (SELECT unnest(sources) AS src FROM collections_index), + n AS (SELECT lower(src.source_doi) AS doi FROM s + WHERE src.provenance.data_contributor IS NULL) + SELECT COUNT(*) AS unexplained FROM n + WHERE doi NOT IN (SELECT lower(source_DOI) FROM analysis_results_index) + """) + assert df["unexplained"][0] == 0 + + def test_idc_contributed_sources_are_idc_converted(self, client_with_all_indices): + """SKILL guides state the two coincide; a release that breaks it fails here.""" + df = client_with_all_indices.sql_query(""" + SELECT COUNT(*) AS mismatched + FROM (SELECT unnest(sources) AS src FROM collections_index) + WHERE src.provenance.data_contributor = 'IDC' + AND src.provenance.dicom_conversion_by IS DISTINCT FROM 'IDC' + """) + assert df["mismatched"][0] == 0 + + def test_series_join_to_source_provenance(self, client_with_all_indices): + """tcga_brca mixes TCIA-contributed radiology with IDC-converted pathology.""" + df = client_with_all_indices.sql_query(""" + SELECT DISTINCT s.src.provenance.data_contributor AS contributed_by, + s.src.provenance.dicom_conversion_by AS converted_by + FROM index i + JOIN (SELECT collection_id, unnest(sources) AS src + FROM collections_index) s + ON s.collection_id = i.collection_id + AND lower(s.src.source_doi) = lower(i.source_DOI) + WHERE i.collection_id = 'tcga_brca' + """) + pairs = set(zip(df["contributed_by"], df["converted_by"])) + assert ("TCIA", "NOT_APPLICABLE") in pairs + assert ("IDC", "IDC") in pairs