diff --git a/.changeset/variantkey-imported-sets.md b/.changeset/variantkey-imported-sets.md new file mode 100644 index 0000000..11a197b --- /dev/null +++ b/.changeset/variantkey-imported-sets.md @@ -0,0 +1,49 @@ +--- +'@platforma-open/milaboratories.generation-probability.workflow': minor +'@platforma-open/milaboratories.generation-probability.model': minor +'@platforma-open/milaboratories.generation-probability.block': minor +--- + +Score imported receptor sets, and fix swapped chain labels on single-cell TCR data + +**Imported receptor sets.** A set from Import VDJ Data was offered nowhere and scored nothing. +Four separate things stopped it, each failing differently. + +The dataset dropdown filtered on the presence of a per-record `pl7.app/vdj/chain` column, which +an imported set does not have — so it never appeared as an option at all. Its locus is a property +of the whole set, recorded on the key axis, and that is now accepted as an alternative. + +The CDR3 alphabet was read from `pl7.app/alphabet` on the key axis or from a `/structure` +domain key. An imported set states it in neither — the structure key belongs to the +`scClonotypeKey` vocabulary its axis does not use — so the run died on "Cannot determine CDR3 +alphabet". Its sequence columns do carry the alphabet, so that is where it is now read from when +the axis says nothing. The axis stays authoritative when it does say something: a MiXCR set emits +both nucleotide and amino-acid CDR3 columns, and the axis is what says which of them defines the +clonotype. + +Scoring needed a per-row locus column and quietly produced an empty result without one. The +scorer's input table is now assembled in a single pass that appends a constant locus column for a +unit whose locus is the same for every row, so the scoring script is unchanged and still simply +reads a locus per row. + +Whether the two chains are scored separately followed the key axis being +`pl7.app/vdj/scClonotypeKey`. A paired imported set carries both chains in one frame under the +`pl7.app/vdj/scClonotypeChain` column domain on a `variantKey` axis, so it took the bulk path, +where the first CDR3 column found is scored and the other chain is silently dropped. That now +follows the column domain. + +**A light chain from an imported set is reported as skipped, not scored.** OLGA needs IGK and IGL +as separate models and an imported set records only "IG Light", so the locus is genuinely unknown. +Rather than guess, the light unit keeps its chain name, gets no Pgen, and is named in the block's +existing skipped-chains banner. Heavy chains (IGH) and TCR alpha/beta (TRA, TRB) score normally. + +**Swapped chain labels.** MiXCR fixes "A" as the more diverse chain — the one that recombines a D +segment — so a receptor's chain order is TCRBeta/TCRAlpha and TCRDelta/TCRGamma, not alphabetical. +The label table read A as Alpha and B as Beta, so on a single-cell TCR alpha/beta dataset the alpha +column was labelled "Generation probability (Beta)" and the beta column "(Alpha)". Gamma/delta was +swapped the same way. The Pgen values were always right — the locus came from the per-record chain +column, never from the A/B letter — but a mislabelled column reads as the wrong chain's result. +Labels are now derived from the chain that the A/B slot resolves to, through the single table that +also decides the locus, so a label cannot disagree with the model that produced the value beside it. + +Bulk, single-cell IG, peptide and amplicon inputs are unaffected. diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7a6c4c9..83c9fe3 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -24,7 +24,7 @@ jobs: with: app-name: 'Block: Generation Probability' app-name-slug: 'block-generation-probability' - node-version: '20.x' + node-version: '22.x' gha-runner-label: hz-ubuntu-dind build-script-name: 'build:dev-local' build-before-publish-script-name: 'build:release' diff --git a/.github/workflows/mark-stable.yaml b/.github/workflows/mark-stable.yaml index abb4f76..39e07c3 100644 --- a/.github/workflows/mark-stable.yaml +++ b/.github/workflows/mark-stable.yaml @@ -15,7 +15,7 @@ jobs: uses: milaboratory/github-ci/.github/workflows/block-mark-stable.yaml@v4 with: app-name: 'Block: Generation Probability - Mark Stable' - node-version: '20.x' + node-version: '22.x' npmrc-config: | { "registries": { diff --git a/model/src/index.ts b/model/src/index.ts index a07feff..968f9c5 100644 --- a/model/src/index.ts +++ b/model/src/index.ts @@ -32,21 +32,24 @@ export type BlockData = { distributionGraphState: GraphMakerState; }; -const inputSelectors = [ - { - axes: [{ name: "pl7.app/vdj/clonotypeKey" }], - annotations: { "pl7.app/isAnchor": "true" }, - }, - { - axes: [{ name: "pl7.app/vdj/scClonotypeKey" }], - annotations: { "pl7.app/isAnchor": "true" }, - }, - { - axes: [{ name: "pl7.app/variantKey" }], - annotations: { "pl7.app/isAnchor": "true" }, - }, +// The axes a dataset can be keyed on, mirroring ENTITY_KEY_NAMES in main.tpl.tengo. Both the +// selectors and the scorability check below are derived from this one list so they cannot drift. +const ENTITY_KEY_NAMES = [ + "pl7.app/vdj/clonotypeKey", + "pl7.app/vdj/scClonotypeKey", + "pl7.app/variantKey", ]; +const inputSelectors = ENTITY_KEY_NAMES.map((name) => ({ + axes: [{ name }], + annotations: { "pl7.app/isAnchor": "true" }, +})); + +// The key axis is found by name, never by position: which index it sits at is a property of the +// producer, and main.tpl.tengo searches for it the same way rather than assuming one. +const keyAxisOf = (spec: { axesSpec: { name: string; domain?: Record }[] }) => + spec.axesSpec.find((axis) => ENTITY_KEY_NAMES.includes(axis.name)); + const dataModel = new DataModelBuilder().from("v1").init(() => ({ datasetLabel: "", tableState: createPlDataTableStateV2(), @@ -74,19 +77,34 @@ export const platforma = BlockModelV3.create(dataModel) const collection = ColumnsCollection(["result_pool"]).filter({ include: inputSelectors, }); + // A dataset is scorable when the locus can be established. Normally that means a per-record + // pl7.app/vdj/chain column. An imported receptor set has none: the locus is a property of + // the whole set and is read from the key axis instead -- pl7.app/vdj/chain for a single + // mapped chain, or pl7.app/vdj/receptor plus the chain column domain for a paired one -- + // so requiring the column would keep every imported set out of this dropdown. const scorableIds = new Set( collection .getColumns() - .filter( - (anchor) => - !ColumnsCollection(["result_pool"]) - .discover({ - anchors: { main: anchor.getSpec() }, - include: [{ name: [{ type: "exact", value: CHAIN_NAME }] }], - mode: "enrichment", - }) - .isEmpty(), - ) + .filter((anchor) => { + const keyAxis = keyAxisOf(anchor.getSpec()); + const keyDomain = keyAxis?.domain ?? {}; + if ( + keyAxis?.name === "pl7.app/variantKey" && + keyDomain["pl7.app/vdj/clonotypingRunId"] !== undefined + ) { + return ( + keyDomain["pl7.app/vdj/chain"] !== undefined || + keyDomain["pl7.app/vdj/receptor"] !== undefined + ); + } + return !ColumnsCollection(["result_pool"]) + .discover({ + anchors: { main: anchor.getSpec() }, + include: [{ name: [{ type: "exact", value: CHAIN_NAME }] }], + mode: "enrichment", + }) + .isEmpty(); + }) .map((anchor) => anchor.id), ); return deriveColumnOptions(collection) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 91790f8..ff2648b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -25,8 +25,8 @@ catalogs: specifier: 1.8.2 version: 1.8.2 '@platforma-sdk/block-tools': - specifier: 2.12.9 - version: 2.12.9 + specifier: 2.14.3 + version: 2.14.3 '@platforma-sdk/model': specifier: 1.80.10 version: 1.80.10 @@ -74,10 +74,10 @@ importers: version: 2.29.8(@types/node@26.1.1) '@milaboratories/ts-builder': specifier: 'catalog:' - version: 1.6.1(@types/node@26.1.1)(esbuild@0.28.1)(rollup@4.62.2)(vue@3.5.24(typescript@5.9.3))(yaml@2.9.0) + version: 1.6.1(@types/node@26.1.1)(esbuild@0.28.1)(rollup@4.62.2)(vue@3.5.24(typescript@7.0.2))(yaml@2.9.0) '@platforma-sdk/block-tools': specifier: 'catalog:' - version: 2.12.9(@types/node@26.1.1) + version: 2.14.3(@types/node@26.1.1) shx: specifier: 'catalog:' version: 0.4.0 @@ -104,7 +104,7 @@ importers: version: link:../workflow '@platforma-sdk/block-tools': specifier: 'catalog:' - version: 2.12.9(@types/node@26.1.1) + version: 2.14.3(@types/node@26.1.1) '@platforma-sdk/model': specifier: 'catalog:' version: 1.80.10 @@ -119,7 +119,7 @@ importers: dependencies: '@milaboratories/graph-maker': specifier: 'catalog:' - version: 1.6.1(@milaboratories/pl-model-common@1.47.3)(@platforma-sdk/model@1.80.10)(@platforma-sdk/ui-vue@1.80.10(@bytecodealliance/preview2-shim@0.17.9)(@vue/compiler-dom@3.5.40)(@vue/server-renderer@3.5.24(vue@3.5.24(typescript@7.0.2)))(typescript@7.0.2))(d3-dispatch@3.0.1)(d3-path@3.1.0)(d3-scale-chromatic@3.1.0)(typescript@7.0.2) + version: 1.6.1(@milaboratories/pl-model-common@1.48.0)(@platforma-sdk/model@1.80.10)(@platforma-sdk/ui-vue@1.80.10(@bytecodealliance/preview2-shim@0.17.9)(@vue/compiler-dom@3.5.40)(@vue/server-renderer@3.5.24(vue@3.5.24(typescript@7.0.2)))(typescript@7.0.2))(d3-dispatch@3.0.1)(d3-path@3.1.0)(d3-scale-chromatic@3.1.0)(typescript@7.0.2) '@milaboratories/helpers': specifier: 'catalog:' version: 1.14.5 @@ -141,7 +141,7 @@ importers: version: 1.3.1 '@platforma-sdk/block-tools': specifier: 'catalog:' - version: 2.12.9(@types/node@26.1.1) + version: 2.14.3(@types/node@26.1.1) software: devDependencies: @@ -150,7 +150,7 @@ importers: version: 1.8.2 '@platforma-sdk/block-tools': specifier: 'catalog:' - version: 2.12.9(@types/node@26.1.1) + version: 2.14.3(@types/node@26.1.1) test: dependencies: @@ -833,11 +833,20 @@ packages: resolution: {integrity: sha512-ILtqflBY9tzd79bKa87eS98vZFCso5/uYxuArfcKrmRDatRg7KWmI/Vr9+xOEf2Y2E/44HXMCx3JKZe/Y4yF6g==} engines: {node: '>=12.10.0'} + '@grpc/grpc-js@1.14.4': + resolution: {integrity: sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==} + engines: {node: '>=12.10.0'} + '@grpc/proto-loader@0.7.15': resolution: {integrity: sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==} engines: {node: '>=6'} hasBin: true + '@grpc/proto-loader@0.8.1': + resolution: {integrity: sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==} + engines: {node: '>=6'} + hasBin: true + '@inquirer/ansi@1.0.2': resolution: {integrity: sha512-S8qNSZiYzFd0wAcyG5AXCvUHC5Sr7xpZ9wZ2py9XR88jUz8wooStVx5M6dRzczbBWjic9NP7+rY0Xi7qqK/aMQ==} engines: {node: '>=18'} @@ -1100,6 +1109,10 @@ packages: resolution: {integrity: sha512-LljKMbKa8zl2lNFkg3VthMx9E6308B5vSH7dtWnlVz20ZU1GF7steToGv5gWs1lwUtm4GRU7kmmsMzU9a7l2yg==} engines: {node: '>=22.19.0'} + '@milaboratories/pl-client@3.14.7': + resolution: {integrity: sha512-HIjPfGAYRRD0hbq5YSTkWt5XgBiv+yYtw73EjWLxui8eUEeB2ffdgsvH7IhVx2oXjUOL4p7M4i8PBnmdJ3830w==} + engines: {node: '>=22.19.0'} + '@milaboratories/pl-config@1.8.5': resolution: {integrity: sha512-XnfYXSSkRxeImQ21k6I8y5apisvagcSgMGfEeyRxNdSGwUVJbbI8TwJk+XBEDQ4lErW8oqtLxKjFQuHJMzRUoQ==} @@ -1131,18 +1144,27 @@ packages: '@milaboratories/pl-model-backend@1.4.18': resolution: {integrity: sha512-cDSreI5DV25v9JmiRDMdp8c2+ZxJgeIjnybsuToVbBwDHoANFqB+2Nsxos5jzU2nxCx5kt2KYJw9/tweuSC4xQ==} + '@milaboratories/pl-model-backend@1.4.21': + resolution: {integrity: sha512-Z2z5J8bglslgXXoi1aySi1ho+/d+ylJiuEaRw9XTNuE5iBM3EQPe0rw095pXGOinTlMpmvUcCI2wkVCraUQ1dA==} + '@milaboratories/pl-model-common@1.46.2': resolution: {integrity: sha512-VEeauisApYScvCS8lnK3zpFJ520xuTAodKJmjR8ulHcMrWMyWMfHEdGb7j5OMD0mM/OwTgmQrrJ5eB7Xd+xoOQ==} '@milaboratories/pl-model-common@1.47.3': resolution: {integrity: sha512-tXmKujm+6ru/Fh9hr9H3iRBWbS+JE0Q7FNualtzJpijaB3SNtScR4erLTamdANv5RYNn7kbYpDWr6wAjAOajrg==} + '@milaboratories/pl-model-common@1.48.0': + resolution: {integrity: sha512-oCVrjFNmjQolb7YWnbSGHnp6GGVm1PhG1lnNABp9lqYjvA2ISjIPQLIo/vT2ca0mqwLrEvbuRqiqSFOg7sQ5tQ==} + '@milaboratories/pl-model-middle-layer@1.30.15': resolution: {integrity: sha512-oBkVlKR+qgsQR74N8G3aW3wBaJw78+9HJjnaNbH7QrTjKq4pno7wTXdrz7Ptk6Tu4SWfkcecXY6+xCnkDKr4eQ==} '@milaboratories/pl-model-middle-layer@1.30.7': resolution: {integrity: sha512-rs9x3Ron4ujR/UOdEgB8WUB1SvZ8ZAScT1Av/e4or+iiQ/CzhmK9nqtYamHVwKV+JgwIFbXQwxGvIHDVz955dQ==} + '@milaboratories/pl-model-middle-layer@1.32.0': + resolution: {integrity: sha512-X1iLGgOwzkw8mQ/GfTxfOTJakBJ26np85h5HqUziMbVkFL0SR7wpd7xpgUs6TKVkYNM8viAv3iA2k63Yc5SahQ==} + '@milaboratories/pl-tree@1.13.3': resolution: {integrity: sha512-39QT6opCX4ejjT0UI7dq4HMEs/NBXEI2nQyyaHbJBLKGKr6DKhUAUd0mLqeIjsEdH0B4eo3bKMlgry5PaPSlpQ==} engines: {node: '>=22.19.0'} @@ -1762,6 +1784,10 @@ packages: resolution: {integrity: sha512-wtqkPsjMpan+XjP0GHa8pcJbj3CX0Jmzxjxez0sz+jGMiIqpsNV3sSTQGYj1zkgYA9Wi0lusKiNoVOYB7Jtmbw==} hasBin: true + '@platforma-sdk/block-tools@2.14.3': + resolution: {integrity: sha512-CD0NPfUoXiJhl6JArC057oCXbqKkJf5HJsmrlZShnh6lKRaIQlibF8VUqbpBi9+PopsGCQ4/s5HnLR7wQoWDzg==} + hasBin: true + '@platforma-sdk/blocks-deps-updater@2.2.0': resolution: {integrity: sha512-p9lBxhFXM9WoRsrJO7dfkiXSK+1m63yIn1sKhBO71eMbhrLMyVYHEOeNf3w5OCdbRF5QsNhXzWuiTmFK3zHFsA==} hasBin: true @@ -1772,6 +1798,9 @@ packages: '@platforma-sdk/package-builder-lib@1.2.1': resolution: {integrity: sha512-H6weitj7JxbiJSlteEFLafTJ+tfty6iv/imf3ysy8oCS8AZIRJk2VMW3M/aAc+xVkQeX7oVIwMwFMYrJIoFsAg==} + '@platforma-sdk/package-builder-lib@1.3.0': + resolution: {integrity: sha512-CdBjmNo6E1fBxKYWaXa49L/L2WLURxs2f1TAqxLIZlHRE4DZ6E1TEj3jNNKESWp+/9rwtLkTAzmTzNPrDgz+2Q==} + '@platforma-sdk/tengo-builder@4.0.20': resolution: {integrity: sha512-4zf38sLzctOgbwym39+YdrE07/W61zzNaeZebZd0p2QDYNacAf83hRlD5KlDvUZtrDbhYNMgoZtghw61XAchog==} engines: {node: '>=22'} @@ -7754,6 +7783,11 @@ snapshots: '@grpc/proto-loader': 0.7.15 '@js-sdsl/ordered-map': 4.4.2 + '@grpc/grpc-js@1.14.4': + dependencies: + '@grpc/proto-loader': 0.8.1 + '@js-sdsl/ordered-map': 4.4.2 + '@grpc/proto-loader@0.7.15': dependencies: lodash.camelcase: 4.3.0 @@ -7761,6 +7795,13 @@ snapshots: protobufjs: 7.6.5 yargs: 17.7.3 + '@grpc/proto-loader@0.8.1': + dependencies: + lodash.camelcase: 4.3.0 + long: 5.3.2 + protobufjs: 7.6.5 + yargs: 17.7.3 + '@inquirer/ansi@1.0.2': {} '@inquirer/checkbox@4.3.2(@types/node@26.1.1)': @@ -8040,6 +8081,30 @@ snapshots: - supports-color - typescript + '@milaboratories/graph-maker@1.6.1(@milaboratories/pl-model-common@1.48.0)(@platforma-sdk/model@1.80.10)(@platforma-sdk/ui-vue@1.80.10(@bytecodealliance/preview2-shim@0.17.9)(@vue/compiler-dom@3.5.40)(@vue/server-renderer@3.5.24(vue@3.5.24(typescript@7.0.2)))(typescript@7.0.2))(d3-dispatch@3.0.1)(d3-path@3.1.0)(d3-scale-chromatic@3.1.0)(typescript@7.0.2)': + dependencies: + '@ag-grid-community/core': 32.3.9 + '@milaboratories/helpers': 1.14.5 + '@milaboratories/miplots4': 1.4.0(d3-dispatch@3.0.1)(d3-path@3.1.0)(d3-scale-chromatic@3.1.0) + '@milaboratories/pf-plots': 1.5.0(@milaboratories/pl-model-common@1.48.0)(@platforma-sdk/model@1.80.10) + '@platforma-sdk/model': 1.80.10 + '@platforma-sdk/ui-vue': 1.80.10(@bytecodealliance/preview2-shim@0.17.9)(@vue/compiler-dom@3.5.40)(@vue/server-renderer@3.5.24(vue@3.5.24(typescript@7.0.2)))(typescript@7.0.2) + '@types/d3-hierarchy': 3.1.7 + '@types/d3-scale': 4.0.9 + '@vueuse/core': 13.9.0(vue@3.5.24(typescript@7.0.2)) + ag-grid-vue3: 34.1.2(vue@3.5.24(typescript@7.0.2)) + canonicalize: 2.1.0 + d3-hierarchy: 3.1.2 + d3-scale: 4.0.2 + vue: 3.5.24(typescript@7.0.2) + transitivePeerDependencies: + - '@milaboratories/pl-model-common' + - d3-dispatch + - d3-path + - d3-scale-chromatic + - supports-color + - typescript + '@milaboratories/helpers@1.14.2': {} '@milaboratories/helpers@1.14.5': {} @@ -8105,6 +8170,14 @@ snapshots: canonicalize: 2.1.0 lodash: 4.18.1 + '@milaboratories/pf-plots@1.5.0(@milaboratories/pl-model-common@1.48.0)(@platforma-sdk/model@1.80.10)': + dependencies: + '@milaboratories/helpers': 1.14.5 + '@milaboratories/pl-model-common': 1.48.0 + '@platforma-sdk/model': 1.80.10 + canonicalize: 2.1.0 + lodash: 4.18.1 + '@milaboratories/pf-spec-driver@1.4.24(@bytecodealliance/preview2-shim@0.17.9)': dependencies: '@milaboratories/helpers': 1.14.5 @@ -8162,6 +8235,24 @@ snapshots: utility-types: 3.11.0 yaml: 2.9.0 + '@milaboratories/pl-client@3.14.7': + dependencies: + '@grpc/grpc-js': 1.14.4 + '@milaboratories/pl-http': 1.2.4 + '@milaboratories/pl-model-common': 1.48.0 + '@milaboratories/ts-helpers': 1.8.6 + '@protobuf-ts/grpc-transport': 2.11.1(@grpc/grpc-js@1.14.4) + '@protobuf-ts/runtime': 2.11.1 + '@protobuf-ts/runtime-rpc': 2.11.1 + canonicalize: 2.1.0 + denque: 2.1.0 + long: 5.3.2 + lru-cache: 11.5.2 + openapi-fetch: 0.15.2 + undici: 7.16.0 + utility-types: 3.11.0 + yaml: 2.9.0 + '@milaboratories/pl-config@1.8.5': dependencies: '@milaboratories/ts-helpers': 1.8.6 @@ -8281,6 +8372,12 @@ snapshots: canonicalize: 2.1.0 zod: 3.25.76 + '@milaboratories/pl-model-backend@1.4.21': + dependencies: + '@milaboratories/pl-client': 3.14.7 + canonicalize: 2.1.0 + zod: 3.25.76 + '@milaboratories/pl-model-common@1.46.2': dependencies: '@milaboratories/helpers': 1.14.2 @@ -8296,6 +8393,14 @@ snapshots: es-toolkit: 1.50.0 zod: 3.25.76 + '@milaboratories/pl-model-common@1.48.0': + dependencies: + '@milaboratories/helpers': 1.14.5 + '@milaboratories/pl-error-like': 1.12.10 + canonicalize: 2.1.0 + es-toolkit: 1.50.0 + zod: 3.25.76 + '@milaboratories/pl-model-middle-layer@1.30.15': dependencies: '@milaboratories/helpers': 1.14.5 @@ -8312,6 +8417,14 @@ snapshots: utility-types: 3.11.0 zod: 3.25.76 + '@milaboratories/pl-model-middle-layer@1.32.0': + dependencies: + '@milaboratories/helpers': 1.14.5 + '@milaboratories/pl-model-common': 1.48.0 + es-toolkit: 1.50.0 + utility-types: 3.11.0 + zod: 3.25.76 + '@milaboratories/pl-tree@1.13.3': dependencies: '@milaboratories/computable': 2.9.8 @@ -8332,47 +8445,6 @@ snapshots: '@milaboratories/tengo-tester@1.6.4': {} - '@milaboratories/ts-builder@1.6.1(@types/node@26.1.1)(esbuild@0.28.1)(rollup@4.62.2)(vue@3.5.24(typescript@5.9.3))(yaml@2.9.0)': - dependencies: - '@milaboratories/ts-configs': 1.3.1 - '@vitejs/plugin-vue': 6.0.8(vite@8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0))(vue@3.5.24(typescript@5.9.3)) - commander: 15.0.0 - jsonc-parser: 3.3.1 - oxfmt: 0.35.0 - oxlint: 1.63.0 - oxlint-plugin-eslint: 1.63.0 - rolldown: 1.2.0 - rolldown-plugin-dts: 0.26.0(rolldown@1.2.0)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@5.9.3)) - rollup-plugin-copy: 3.5.0 - rollup-plugin-sourcemaps2: 0.5.8(@types/node@26.1.1)(rollup@4.62.2) - typescript: 5.9.3 - vite: 8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0) - vite-plugin-commonjs: 0.10.4 - vite-plugin-dts: 4.5.4(@types/node@26.1.1)(rollup@4.62.2)(typescript@5.9.3)(vite@8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0)) - vite-plugin-externalize-deps: 0.10.0(vite@8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0)) - vite-plugin-lib-inject-css: 2.2.2(vite@8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0)) - vue-tsc: 3.3.5(typescript@5.9.3) - transitivePeerDependencies: - - '@ts-macro/tsc' - - '@types/node' - - '@typescript/native-preview' - - '@vitejs/devtools' - - esbuild - - jiti - - less - - oxc-resolver - - oxlint-tsgolint - - rollup - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - - tsx - - vue - - yaml - '@milaboratories/ts-builder@1.6.1(@types/node@26.1.1)(esbuild@0.28.1)(rollup@4.62.2)(vue@3.5.24(typescript@7.0.2))(yaml@2.9.0)': dependencies: '@milaboratories/ts-configs': 1.3.1 @@ -8383,7 +8455,7 @@ snapshots: oxlint: 1.63.0 oxlint-plugin-eslint: 1.63.0 rolldown: 1.2.0 - rolldown-plugin-dts: 0.26.0(rolldown@1.2.0)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@5.9.3)) + rolldown-plugin-dts: 0.26.0(rolldown@1.2.0)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@7.0.2)) rollup-plugin-copy: 3.5.0 rollup-plugin-sourcemaps2: 0.5.8(@types/node@26.1.1)(rollup@4.62.2) typescript: 5.9.3 @@ -8897,6 +8969,34 @@ snapshots: - bare-buffer - react-native-b4a + '@platforma-sdk/block-tools@2.14.3(@types/node@26.1.1)': + dependencies: + '@aws-sdk/client-ecr-public': 3.859.0 + '@aws-sdk/client-s3': 3.859.0 + '@inquirer/prompts': 7.10.1(@types/node@26.1.1) + '@milaboratories/pl-http': 1.2.4 + '@milaboratories/pl-model-backend': 1.4.21 + '@milaboratories/pl-model-common': 1.48.0 + '@milaboratories/pl-model-middle-layer': 1.32.0 + '@milaboratories/resolve-helper': 1.1.3 + '@milaboratories/ts-helpers': 1.8.6 + '@platforma-sdk/blocks-deps-updater': 2.2.0 + '@platforma-sdk/package-builder-lib': 1.3.0 + canonicalize: 2.1.0 + commander: 15.0.0 + lru-cache: 11.5.2 + mime-types: 2.1.35 + tar: 7.5.21 + undici: 7.16.0 + yaml: 2.9.0 + zod: 3.25.76 + transitivePeerDependencies: + - '@types/node' + - aws-crt + - bare-abort-controller + - bare-buffer + - react-native-b4a + '@platforma-sdk/blocks-deps-updater@2.2.0': dependencies: yaml: 2.9.0 @@ -8931,6 +9031,22 @@ snapshots: - bare-buffer - react-native-b4a + '@platforma-sdk/package-builder-lib@1.3.0': + dependencies: + '@aws-sdk/client-s3': 3.859.0 + '@aws-sdk/lib-storage': 3.859.0(@aws-sdk/client-s3@3.859.0) + '@milaboratories/resolve-helper': 1.1.3 + archiver: 7.0.1 + undici: 7.16.0 + winston: 3.19.0 + yaml: 2.9.0 + zod: 3.25.76 + transitivePeerDependencies: + - aws-crt + - bare-abort-controller + - bare-buffer + - react-native-b4a + '@platforma-sdk/tengo-builder@4.0.20': dependencies: '@milaboratories/pl-model-backend': 1.4.18 @@ -9053,6 +9169,12 @@ snapshots: '@protobuf-ts/runtime': 2.11.1 '@protobuf-ts/runtime-rpc': 2.11.1 + '@protobuf-ts/grpc-transport@2.11.1(@grpc/grpc-js@1.14.4)': + dependencies: + '@grpc/grpc-js': 1.14.4 + '@protobuf-ts/runtime': 2.11.1 + '@protobuf-ts/runtime-rpc': 2.11.1 + '@protobuf-ts/plugin@2.11.1': dependencies: '@bufbuild/protobuf': 2.13.0 @@ -12099,12 +12221,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@vitejs/plugin-vue@6.0.8(vite@8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0))(vue@3.5.24(typescript@5.9.3))': - dependencies: - '@rolldown/pluginutils': 1.0.1 - vite: 8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0) - vue: 3.5.24(typescript@5.9.3) - '@vitejs/plugin-vue@6.0.8(vite@8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0))(vue@3.5.24(typescript@7.0.2))': dependencies: '@rolldown/pluginutils': 1.0.1 @@ -12330,12 +12446,6 @@ snapshots: '@vue/shared': 3.5.24 csstype: 3.2.3 - '@vue/server-renderer@3.5.24(vue@3.5.24(typescript@5.9.3))': - dependencies: - '@vue/compiler-ssr': 3.5.24 - '@vue/shared': 3.5.24 - vue: 3.5.24(typescript@5.9.3) - '@vue/server-renderer@3.5.24(vue@3.5.24(typescript@7.0.2))': dependencies: '@vue/compiler-ssr': 3.5.24 @@ -13943,7 +14053,7 @@ snapshots: reusify@1.1.0: {} - rolldown-plugin-dts@0.26.0(rolldown@1.2.0)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@5.9.3)): + rolldown-plugin-dts@0.26.0(rolldown@1.2.0)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@7.0.2)): dependencies: '@babel/generator': 8.0.0 '@babel/helper-validator-identifier': 8.0.4 @@ -14591,16 +14701,6 @@ snapshots: '@vue/language-core': 3.3.5 typescript: 5.9.3 - vue@3.5.24(typescript@5.9.3): - dependencies: - '@vue/compiler-dom': 3.5.24 - '@vue/compiler-sfc': 3.5.24 - '@vue/runtime-dom': 3.5.24 - '@vue/server-renderer': 3.5.24(vue@3.5.24(typescript@5.9.3)) - '@vue/shared': 3.5.24 - optionalDependencies: - typescript: 5.9.3 - vue@3.5.24(typescript@7.0.2): dependencies: '@vue/compiler-dom': 3.5.24 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 6d53f82..5b2809c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -12,7 +12,7 @@ catalog: "@milaboratories/ts-builder": 1.6.1 "@milaboratories/ts-configs": 1.3.1 "@platforma-open/milaboratories.runenv-python-3": 1.8.2 - "@platforma-sdk/block-tools": 2.12.9 + "@platforma-sdk/block-tools": 2.14.3 "@platforma-sdk/model": 1.80.10 "@platforma-sdk/package-builder": 3.14.2 "@platforma-sdk/tengo-builder": 4.0.20 diff --git a/workflow/src/main.tpl.tengo b/workflow/src/main.tpl.tengo index ed696e6..22653f2 100644 --- a/workflow/src/main.tpl.tengo +++ b/workflow/src/main.tpl.tengo @@ -12,25 +12,6 @@ ENTITY_KEY_NAMES := { "pl7.app/variantKey": true } -// The alphabet may live directly on the key axis domain, or nested in the structured -// key's `/structure` domain under the CDR3 sequence entry. -alphabetFromStructure := func(structureStr) { - if is_undefined(structureStr) { - return undefined - } - for entry in json.decode(structureStr) { - if len(entry) == 0 || entry[0] != "pl7.app/vdj/sequence" { - continue - } - for idx, item in entry { - if idx > 0 && item[0] == "pl7.app/alphabet" { - return item[1] - } - } - } - return undefined -} - // Resolve only the anchor spec here; CDR3/chain columns are resolved in the child (score), // which anchors them by the entity-key axis name — unknown until the anchor spec resolves. wf.prepare(func(args) { @@ -57,15 +38,6 @@ wf.body(func(args) { ll.panic("Selected dataset has no clonotype or variant key axis") } - keyDomain := is_undefined(keySpec.domain) ? {} : keySpec.domain - alphabet := keyDomain["pl7.app/alphabet"] - if is_undefined(alphabet) { - alphabet = alphabetFromStructure(keyDomain[keySpec.name + "/structure"]) - } - if is_undefined(alphabet) { - ll.panic("Cannot determine CDR3 alphabet") - } - // Forward only trace + label, so an unrelated datasetSpec change doesn't bust the // downstream cache. datasetLabel := "" @@ -84,7 +56,6 @@ wf.body(func(args) { anchorRef: anchorRef, keyName: keySpec.name, keySpec: keySpec, - alphabet: alphabet, species: args.species, datasetLabel: datasetLabel, traceAnnotations: traceAnnotations diff --git a/workflow/src/score.tpl.tengo b/workflow/src/score.tpl.tengo index 5d467ab..5ef2404 100644 --- a/workflow/src/score.tpl.tengo +++ b/workflow/src/score.tpl.tengo @@ -4,9 +4,31 @@ smart := import("@platforma-sdk/workflow-tengo:smart") assets := import("@platforma-sdk/workflow-tengo:assets") pframes := import("@platforma-sdk/workflow-tengo:pframes") pBundle := import("@platforma-sdk/workflow-tengo:pframes.bundle") +pt := import("@platforma-sdk/workflow-tengo:pt") +ll := import("@platforma-sdk/workflow-tengo:ll") +json := import("json") pgenTpl := assets.importTemplate(":pgen-process") +// The alphabet may sit directly on the key axis domain, or nested in the structured key's +// `/structure` domain under the CDR3 sequence entry. +alphabetFromStructure := func(structureStr) { + if is_undefined(structureStr) { + return undefined + } + for entry in json.decode(structureStr) { + if len(entry) == 0 || entry[0] != "pl7.app/vdj/sequence" { + continue + } + for idx, item in entry { + if idx > 0 && item[0] == "pl7.app/alphabet" { + return item[1] + } + } + } + return undefined +} + CHAIN_LABELS := { "IGHeavy": "Heavy", "IGLight": "Light", @@ -16,23 +38,40 @@ CHAIN_LABELS := { "TCRDelta": "Delta" } -SC_CHAIN_LABELS := { - "IG": { "A": "Heavy", "B": "Light" }, - "TCRAB": { "A": "Alpha", "B": "Beta" }, - "TCRGD": { "A": "Gamma", "B": "Delta" } +// Which chain each A/B slot holds. MiXCR fixes "A" as the more diverse chain — the one that +// recombines a D segment — so a receptor's chain order is TCRBeta/TCRAlpha and +// TCRDelta/TCRGamma, not alphabetical (mixcr-clonotyping/workflow/src/process.tpl.tengo:43,695). +RECEPTOR_CHAINS := { + "IG": { "A": "IGHeavy", "B": "IGLight" }, + "TCRAB": { "A": "TCRBeta", "B": "TCRAlpha" }, + "TCRGD": { "A": "TCRDelta", "B": "TCRGamma" } +} + +// The OLGA locus each chain maps to. A chain is absent here only when no locus can be named +// for it at all: an imported light chain is "IG Light", and OLGA needs IGK and IGL as separate +// models, so which one it is simply is not recorded. +CHAIN_LOCUS := { + "IGHeavy": "IGH", + "TCRAlpha": "TRA", + "TCRBeta": "TRB", + "TCRGamma": "TRG", + "TCRDelta": "TRD" } +// Labels come from the chain, and the A/B slot resolves to a chain through RECEPTOR_CHAINS — +// the same table the locus is derived from, so a label can never disagree with the model that +// produced the value beside it. chainLabel := func(keySpec, cdr3Col, scChain) { if is_undefined(scChain) { return CHAIN_LABELS[keySpec.domain["pl7.app/vdj/chain"]] } axis := cdr3Col.spec.axesSpec[0] axisDomain := is_undefined(axis.domain) ? {} : axis.domain - receptor := axisDomain["pl7.app/vdj/receptor"] - if is_undefined(receptor) || is_undefined(SC_CHAIN_LABELS[receptor]) { + byLetter := RECEPTOR_CHAINS[axisDomain["pl7.app/vdj/receptor"]] + if is_undefined(byLetter) { return undefined } - return SC_CHAIN_LABELS[receptor][scChain] + return CHAIN_LABELS[byLetter[scChain]] } matchDomain := func(col, required) { @@ -73,6 +112,22 @@ scSlot := func(scChain) { } BULK_SLOT := { scChain: undefined, chainDomain: {} } + +// An imported receptor set carries no per-record pl7.app/vdj/chain column: the locus is a +// property of the whole unit. Resolve it from the chain the scientist mapped — on the key axis +// for a single-chain set, from the receptor plus the A/B slot for a paired one. +bareChain := func(keySpec, scChain) { + keyDomain := is_undefined(keySpec.domain) ? {} : keySpec.domain + if is_undefined(scChain) { + return keyDomain["pl7.app/vdj/chain"] + } + byLetter := RECEPTOR_CHAINS[keyDomain["pl7.app/vdj/receptor"]] + if is_undefined(byLetter) { + return undefined + } + return byLetter[scChain] +} + self.defineOutputs("pgenPf", "exportPf", "progress") self.prepare(func(args) { @@ -108,15 +163,55 @@ self.prepare(func(args) { self.body(func(args) { columns := args.columns - alphabet := args.alphabet keySpec := args.keySpec - isSingleCell := keySpec.name == "pl7.app/vdj/scClonotypeKey" // CDR3 lives under one of two namespaces (pl7.app/vdj/sequence for clonotype keys, // pl7.app/sequence for variantKey); merge both — only one is present, the other returns []. cdr3Cols := columns.getColumns("cdr3") + columns.getColumns("cdr3Plain") chainCols := columns.getColumns("chain") + // Which alphabet the CDR3 is scored in. The key axis is authoritative wherever it speaks, + // directly or inside its structure key: a MiXCR set emits both nt and aa CDR3 columns, and + // the axis is what says which of them defines the clonotype. An imported set's axis states + // neither, so fall back to the columns, which do carry pl7.app/alphabet. + keyDomain := is_undefined(keySpec.domain) ? {} : keySpec.domain + alphabet := keyDomain["pl7.app/alphabet"] + if is_undefined(alphabet) { + alphabet = alphabetFromStructure(keyDomain[keySpec.name + "/structure"]) + } + if is_undefined(alphabet) { + for col in cdr3Cols { + if is_undefined(col.spec.domain) { + continue + } + colAlphabet := col.spec.domain["pl7.app/alphabet"] + if is_undefined(colAlphabet) { + continue + } + if is_undefined(alphabet) { + alphabet = colAlphabet + } else if alphabet != colAlphabet { + ll.panic("Cannot determine CDR3 alphabet: the key axis states none and the CDR3 columns disagree (%v vs %v)", alphabet, colAlphabet) + } + } + } + if is_undefined(alphabet) { + ll.panic("Cannot determine CDR3 alphabet: neither the key axis nor any CDR3 column states one") + } + + // A paired imported set carries both chains in one frame under the scClonotypeChain column + // domain, on a variantKey axis — so the axis name alone would call it bulk, and the bulk slot + // takes the first CDR3 column it finds and drops the other chain. + isSingleCell := keySpec.name == "pl7.app/vdj/scClonotypeKey" + if keySpec.name == "pl7.app/variantKey" { + for col in cdr3Cols { + if !is_undefined(col.spec.domain) && + !is_undefined(col.spec.domain["pl7.app/vdj/scClonotypeChain/index"]) { + isSingleCell = true + } + } + } + units := [] addUnit := func(slot) { domain := { "pl7.app/alphabet": alphabet } @@ -124,16 +219,39 @@ self.body(func(args) { domain[k] = v } cdr3Col := firstWithDomain(cdr3Cols, domain) - chainCol := firstWithDomain(chainCols, slot.chainDomain) - if is_undefined(cdr3Col) || is_undefined(chainCol) { + if is_undefined(cdr3Col) { return } + + // Preferred: the per-row locus column. An imported set has none, so fall back to the + // unit's single mapped chain and materialize it as a constant column below. Without a + // locus from either source there is nothing to score against. + chainCol := firstWithDomain(chainCols, slot.chainDomain) + chainLiteral := undefined + label := chainLabel(keySpec, cdr3Col, slot.scChain) + if is_undefined(chainCol) { + chain := bareChain(keySpec, slot.scChain) + if is_undefined(chain) { + return + } + // With no locus for this chain, send its own name: the scorer has no model under + // that name either, so it leaves the score null and reports the name — which the + // block shows as a banner. Dropping the unit instead would lose the chain with + // nothing on screen to say so. + chainLiteral = CHAIN_LOCUS[chain] + if is_undefined(chainLiteral) { + chainLiteral = chain + } + label = CHAIN_LABELS[chain] + } + units += [{ suffix: is_undefined(slot.scChain) ? "bulk" : slot.scChain, domain: domain, - label: chainLabel(keySpec, cdr3Col, slot.scChain), + label: label, cdr3Col: cdr3Col, chainCol: chainCol, + chainLiteral: chainLiteral, seqCols: allWithDomain(cdr3Cols, slot.chainDomain) }] } @@ -154,21 +272,48 @@ self.body(func(args) { } } - tableBuilder := pframes.parquetFileBuilder() - tableBuilder.setAxisHeader(keySpec, "clonotypeKey") + // The scorer's input table, assembled in one pass. pt joins the bundle columns on the key + // axis under the header names the scorer expects, and appends a constant column for a unit + // whose locus is one value for every row. Assembling with parquetFileBuilder and then + // running pt to add that constant wrote the whole table twice. + // + // The columns are named here rather than via frameFromColumnBundle: that helper decodes + // each pool key as JSON, and an addMulti result is keyed "." + // (pframes/bundle.lib.tengo:399), which is not JSON — it fails inside the helper. + ptWf := pt.workflow().mem("4GiB").cpu(1) + + entries := [] + selects := [pt.sc.axis(keySpec).alias("clonotypeKey")] + chainExprs := [] unitParams := [] for unit in units { - tableBuilder.add(unit.cdr3Col, { header: "cdr3_" + unit.suffix }) - tableBuilder.add(unit.chainCol, { header: "chain_" + unit.suffix }) + cdr3Name := "cdr3_" + unit.suffix + entries += [pt.p.column(cdr3Name, { spec: unit.cdr3Col.spec, data: unit.cdr3Col.data })] + selects += [pt.col(cdr3Name)] + + chainName := "chain_" + unit.suffix + if is_undefined(unit.chainCol) { + // No column to join: this unit's locus is a property of the whole set. Appending it + // here keeps main.py a pure per-row scorer and the software package unchanged. + chainExprs += [pt.lit(unit.chainLiteral).alias(chainName)] + } else { + entries += [pt.p.column(chainName, { spec: unit.chainCol.spec, data: unit.chainCol.data })] + selects += [pt.col(chainName)] + } + unitParams += [{ suffix: unit.suffix, domain: unit.domain, label: unit.label }] } - tableBuilder.cpu(1) - tableBuilder.mem("4GiB") - inputTable := tableBuilder.build() + + df := ptWf.frame(pt.p.full(entries...)).select(selects...) + if len(chainExprs) > 0 { + df = df.withColumns(chainExprs...) + } + df.save("input.parquet") + inputTable := ptWf.run().getFile("input.parquet") // Specs in a keyed map so ordering can't perturb the params hash; data passed separately. renderInputs := { inputTable: inputTable }