From 5be7d5d58504904c42a0936d4e5306b55313f138 Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 11:49:10 +0200 Subject: [PATCH 01/17] MILAB-6720: score imported receptor sets keyed on variantKey MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four independent things stopped an imported set from Import VDJ Data, and they fail at four different points — so fixing only what a user reported would fix one of them. MODEL, inputOptions. The dropdown filtered candidates down to those with a discoverable per-record pl7.app/vdj/chain column. An imported set has none, so it was never offered: the failure was invisible, with no error to report. Its locus is a property of the whole set and lives on the key axis, so that is accepted as an alternative — pl7.app/vdj/chain for a single mapped chain, pl7.app/vdj/receptor for a paired one. MAIN.TPL, alphabet. Read from pl7.app/alphabet on the key axis, else from the /structure domain key. An imported set carries neither: the structure key belongs to the scClonotypeKey vocabulary its axis no longer uses, and no variantKey producer emits one. The run died on "Cannot determine CDR3 alphabet". Every sequence and region column an imported set emits is pl7.app/alphabet: aminoacid, so that is the answer. SCORE.TPL, locus. addUnit required a per-row chain column and returned early without one, leaving units empty — an empty p-frame and a null progress resource, which the model reads as not-running. So the block sat there having silently scored nothing. A unit whose locus is constant now carries it as "=LOCUS". SCORE.TPL, arrangement. isSingleCell keyed off the axis being scClonotypeKey. A paired imported set carries both chains in one frame under the scClonotypeChain column domain but on a variantKey axis, so it took the bulk path, where firstWithDomain returns the first CDR3 column and the second chain is dropped with the result still looking complete. It now widens from the column domain, gated on the axis being variantKey, testing /index because that is what the scSlot domains match on. SOFTWARE. --chain-column accepts "=LOCUS" and materialises it as a constant column before anything else runs, so group_by, the join and the unsupported-chain report are untouched. Args is a frozen dataclass, so the resolved names travel as a local rather than being assigned back. Verified against real polars: literals become constant String columns, plain column names pass through with nothing added, and the two forms mix. The light chain is deliberately not scored. OLGA needs IGK and IGL as separate models and an imported set records only IGLight — CHAINS in bare-set-specs.lib.tengo has no kappa or lambda — so the locus is unknown, not merely unstated. It is passed through as IGLight, which is not in CHAIN_MODELS, so the scorer skips it, leaves Pgen null and names it in the "skipped: " line the model's skippedChains output already reads. No new warning machinery. IGHeavy, TCRAlpha and TCRBeta resolve to IGH, TRA and TRB and are scored normally. A NEW table rather than reuse of SC_CHAIN_LABELS. Its two TCR rows are inverted against MiXCR's convention, which fixes A as the more diverse chain and so orders TCRAB as TCRBeta/TCRAlpha (mixcr-clonotyping/workflow/src/process.tpl.tengo:43,695). Today that is only a labelling bug, because the locus comes from the per-row column; reusing the table here would have picked the wrong OLGA model and produced wrong numbers. Imported units take their label from the resolved chain for the same reason, so their labels agree with their loci. The existing inversion is left alone — correcting it changes user-visible labels on single-cell TCR datasets and belongs in its own change. --- .changeset/variantkey-imported-sets.md | 38 ++++++++++++ model/src/index.ts | 35 ++++++++--- software/src/main.py | 33 ++++++++-- workflow/src/main.tpl.tengo | 8 +++ workflow/src/pgen-process.tpl.tengo | 8 ++- workflow/src/score.tpl.tengo | 86 ++++++++++++++++++++++++-- 6 files changed, 187 insertions(+), 21 deletions(-) create mode 100644 .changeset/variantkey-imported-sets.md diff --git a/.changeset/variantkey-imported-sets.md b/.changeset/variantkey-imported-sets.md new file mode 100644 index 0000000..f268dcf --- /dev/null +++ b/.changeset/variantkey-imported-sets.md @@ -0,0 +1,38 @@ +--- +'@platforma-open/milaboratories.generation-probability.software': minor +'@platforma-open/milaboratories.generation-probability.workflow': minor +'@platforma-open/milaboratories.generation-probability.model': minor +'@platforma-open/milaboratories.generation-probability.block': minor +--- + +Score imported receptor sets keyed on pl7.app/variantKey + +An imported set from Import VDJ Data was offered nowhere and scored nothing. Four separate +things stopped it, each failing differently: + +The dataset dropdown filtered on the presence of a per-record `pl7.app/vdj/chain` column, which +an imported set does not have — so it never appeared as an option at all. Its locus is a property +of the whole set, recorded on the key axis, and that is now accepted as an alternative. + +The CDR3 alphabet was read from `pl7.app/alphabet` on the key axis or from a `/structure` +domain key. An imported set carries neither — the structure key belongs to the `scClonotypeKey` +vocabulary its axis does not use — so the run died on "Cannot determine CDR3 alphabet". Such a +set is amino acid throughout, which is now the answer. + +Scoring needed a per-row locus column and quietly produced an empty result without one. A unit +whose locus is constant now passes it to the scorer as `=LOCUS`, which materialises it as a +column, leaving the grouping, joining and unsupported-chain reporting unchanged. + +Whether the two chains are scored separately followed the key axis being +`pl7.app/vdj/scClonotypeKey`. A paired imported set carries both chains in one frame under the +`pl7.app/vdj/scClonotypeChain` column domain on a `variantKey` axis, so it took the bulk path, +where the first CDR3 column found is scored and the other chain is silently dropped. That now +follows the column domain. + +**A light chain from an imported set is reported as skipped, not scored.** OLGA needs IGK and IGL +as separate models and an imported set records only "IG Light", so the locus is genuinely unknown. +Rather than guess, the light unit is passed through as `IGLight`, gets no Pgen, and is named in +the block's existing skipped-chains message. Heavy chains (IGH) and TCR alpha/beta (TRA, TRB) are +scored normally. + +Bulk, single-cell, peptide and amplicon inputs are unaffected. diff --git a/model/src/index.ts b/model/src/index.ts index a07feff..d8e105c 100644 --- a/model/src/index.ts +++ b/model/src/index.ts @@ -74,19 +74,34 @@ export const platforma = BlockModelV3.create(dataModel) const collection = ColumnsCollection(["result_pool"]).filter({ include: inputSelectors, }); + // A dataset is scorable when the locus can be established. Normally that means a per-record + // pl7.app/vdj/chain column. An imported receptor set has none: the locus is a property of + // the whole set and is read from the key axis instead -- pl7.app/vdj/chain for a single + // mapped chain, or pl7.app/vdj/receptor plus the chain column domain for a paired one -- + // so requiring the column would keep every imported set out of this dropdown. const scorableIds = new Set( collection .getColumns() - .filter( - (anchor) => - !ColumnsCollection(["result_pool"]) - .discover({ - anchors: { main: anchor.getSpec() }, - include: [{ name: [{ type: "exact", value: CHAIN_NAME }] }], - mode: "enrichment", - }) - .isEmpty(), - ) + .filter((anchor) => { + const keyAxis = anchor.getSpec().axesSpec[1]; + const keyDomain = keyAxis?.domain ?? {}; + if ( + keyAxis?.name === "pl7.app/variantKey" && + keyDomain["pl7.app/vdj/clonotypingRunId"] !== undefined + ) { + return ( + keyDomain["pl7.app/vdj/chain"] !== undefined || + keyDomain["pl7.app/vdj/receptor"] !== undefined + ); + } + return !ColumnsCollection(["result_pool"]) + .discover({ + anchors: { main: anchor.getSpec() }, + include: [{ name: [{ type: "exact", value: CHAIN_NAME }] }], + mode: "enrichment", + }) + .isEmpty(); + }) .map((anchor) => anchor.id), ); return deriveColumnOptions(collection) diff --git a/software/src/main.py b/software/src/main.py index 420ff61..7e0328d 100644 --- a/software/src/main.py +++ b/software/src/main.py @@ -114,6 +114,7 @@ def parse_args() -> Args: required=True, action="append", dest="chain_columns", + help="column holding each row's locus, or =LOCUS for a unit with a constant locus", ) parser.add_argument( "--pgen-column", @@ -131,15 +132,39 @@ def parse_args() -> Args: return Args(**vars(parser.parse_args())) +def resolve_chain_literals(df: pl.LazyFrame, chain_args: list[str]) -> tuple[pl.LazyFrame, list[str]]: + """Materialise `=LOCUS` chain arguments as constant columns. + + A dataset whose locus is a property of the whole unit rather than of each row -- an + imported receptor set, which carries no per-record `pl7.app/vdj/chain` column -- passes + `=IGH` instead of a column name. Adding it as a real column here keeps every step + downstream (group_by, join, the unsupported-chain report) working unchanged. + """ + literals: dict[str, str] = {} + resolved: list[str] = [] + for index, argument in enumerate(chain_args): + if not argument.startswith("="): + resolved.append(argument) + continue + name = f"_chain_literal_{index}" + literals[name] = argument[1:] + resolved.append(name) + if literals: + df = df.with_columns([pl.lit(v).alias(k) for k, v in literals.items()]) + return df, resolved + + def main() -> None: args = parse_args() df = pl.scan_parquet(args.input) + # Args is frozen, so the resolved names travel as a local. + df, chain_columns = resolve_chain_literals(df, args.chain_columns) columns = set(df.collect_schema().names()) missing_columns = { args.key_column, *args.sequence_columns, - *args.chain_columns, + *chain_columns, } - columns if missing_columns: msg = f"columns not found in {args.input}: {missing_columns}" @@ -147,10 +172,10 @@ def main() -> None: if args.workers < 1: msg = f"--workers must be >= 1, got {args.workers}" raise SystemExit(msg) - if len(args.sequence_columns) != len(args.chain_columns): + if len(args.sequence_columns) != len(chain_columns): msg = ( "number of sequence columns and chain columns do not match " - f"({len(args.sequence_columns)} ≠ {len(args.chain_columns)})" + f"({len(args.sequence_columns)} ≠ {len(chain_columns)})" ) raise SystemExit(msg) @@ -194,7 +219,7 @@ def process_batch(batch: pl.Series) -> pl.Series: for sequence, chain, pgen, neg_log_pgen in zip( args.sequence_columns, - args.chain_columns, + chain_columns, args.pgen_columns, args.neg_log_pgen_columns, strict=True, diff --git a/workflow/src/main.tpl.tengo b/workflow/src/main.tpl.tengo index ed696e6..c581c9b 100644 --- a/workflow/src/main.tpl.tengo +++ b/workflow/src/main.tpl.tengo @@ -62,6 +62,14 @@ wf.body(func(args) { if is_undefined(alphabet) { alphabet = alphabetFromStructure(keyDomain[keySpec.name + "/structure"]) } + // An imported receptor set carries neither source: no pl7.app/alphabet on the key axis, and + // no /structure -- that key belongs to the scClonotypeKey vocabulary this axis does not + // use, and no variantKey producer emits one. Its sequences are amino acid throughout, so + // that is the answer rather than a failure. + if is_undefined(alphabet) && keySpec.name == "pl7.app/variantKey" && + !is_undefined(keyDomain["pl7.app/vdj/clonotypingRunId"]) { + alphabet = "aminoacid" + } if is_undefined(alphabet) { ll.panic("Cannot determine CDR3 alphabet") } diff --git a/workflow/src/pgen-process.tpl.tengo b/workflow/src/pgen-process.tpl.tengo index 7af8828..482237a 100644 --- a/workflow/src/pgen-process.tpl.tengo +++ b/workflow/src/pgen-process.tpl.tengo @@ -24,7 +24,13 @@ self.body(func(args) { arg("--key-column").arg("clonotypeKey") for unit in units { cmd = cmd.arg("--sequence-column").arg("cdr3_" + unit.suffix) - cmd = cmd.arg("--chain-column").arg("chain_" + unit.suffix) + // A unit with a constant locus (an imported set, which has no per-row chain column) + // carries it as "=LOCUS"; the scorer materialises that as a column. + if is_undefined(unit.chainArg) { + cmd = cmd.arg("--chain-column").arg("chain_" + unit.suffix) + } else { + cmd = cmd.arg("--chain-column").arg(unit.chainArg) + } cmd = cmd.arg("--pgen-column").arg("pgen_" + unit.suffix) cmd = cmd.arg("--neg-log-pgen-column").arg("neglog_" + unit.suffix) } diff --git a/workflow/src/score.tpl.tengo b/workflow/src/score.tpl.tengo index 5d467ab..2cdfb4a 100644 --- a/workflow/src/score.tpl.tengo +++ b/workflow/src/score.tpl.tengo @@ -73,6 +73,47 @@ scSlot := func(scChain) { } BULK_SLOT := { scChain: undefined, chainDomain: {} } +// MiXCR's A/B convention: "A" is always the more diverse chain, so a receptor's chain order is +// TCRBeta/TCRAlpha and TCRDelta/TCRGamma rather than alphabetical +// (mixcr-clonotyping/workflow/src/process.tpl.tengo:43,695). +// +// SC_CHAIN_LABELS above has both TCR rows inverted against this. That is a pre-existing +// labelling bug, and it is deliberately not reused here: a wrong letter in THIS table would +// pick the wrong OLGA model and produce wrong Pgen values, not just a wrong label. +RECEPTOR_CHAINS := { + "IG": { "A": "IGHeavy", "B": "IGLight" }, + "TCRAB": { "A": "TCRBeta", "B": "TCRAlpha" }, + "TCRGD": { "A": "TCRDelta", "B": "TCRGamma" } +} + +// The OLGA locus each chain maps to. IGLight is deliberately not a locus: OLGA needs IGK and +// IGL as separate models and an imported set does not record which, so it is passed through +// unchanged — the scorer skips any chain it has no model for and names it in the skipped-chains +// report, which the model already surfaces. +CHAIN_LOCUS := { + "IGHeavy": "IGH", + "IGLight": "IGLight", + "TCRAlpha": "TRA", + "TCRBeta": "TRB", + "TCRGamma": "TRG", + "TCRDelta": "TRD" +} + +// An imported receptor set carries no per-record pl7.app/vdj/chain column: the locus is a +// property of the whole unit. Resolve it from the chain the scientist mapped — on the key axis +// for a single-chain set, from the receptor plus the A/B slot for a paired one. +bareChain := func(keySpec, scChain) { + keyDomain := is_undefined(keySpec.domain) ? {} : keySpec.domain + if is_undefined(scChain) { + return keyDomain["pl7.app/vdj/chain"] + } + byLetter := RECEPTOR_CHAINS[keyDomain["pl7.app/vdj/receptor"]] + if is_undefined(byLetter) { + return undefined + } + return byLetter[scChain] +} + self.defineOutputs("pgenPf", "exportPf", "progress") self.prepare(func(args) { @@ -110,13 +151,25 @@ self.body(func(args) { columns := args.columns alphabet := args.alphabet keySpec := args.keySpec - isSingleCell := keySpec.name == "pl7.app/vdj/scClonotypeKey" // CDR3 lives under one of two namespaces (pl7.app/vdj/sequence for clonotype keys, // pl7.app/sequence for variantKey); merge both — only one is present, the other returns []. cdr3Cols := columns.getColumns("cdr3") + columns.getColumns("cdr3Plain") chainCols := columns.getColumns("chain") + // A paired imported set carries both chains in one frame under the scClonotypeChain column + // domain, on a variantKey axis — so the axis name alone would call it bulk, and the bulk slot + // takes the first CDR3 column it finds and drops the other chain. + isSingleCell := keySpec.name == "pl7.app/vdj/scClonotypeKey" + if !isSingleCell && keySpec.name == "pl7.app/variantKey" { + for col in cdr3Cols { + colDomain := is_undefined(col.spec.domain) ? {} : col.spec.domain + if !is_undefined(colDomain["pl7.app/vdj/scClonotypeChain/index"]) { + isSingleCell = true + } + } + } + units := [] addUnit := func(slot) { domain := { "pl7.app/alphabet": alphabet } @@ -124,16 +177,34 @@ self.body(func(args) { domain[k] = v } cdr3Col := firstWithDomain(cdr3Cols, domain) - chainCol := firstWithDomain(chainCols, slot.chainDomain) - if is_undefined(cdr3Col) || is_undefined(chainCol) { + if is_undefined(cdr3Col) { return } + + // Preferred: the per-row locus column. An imported set has none, so fall back to the + // unit's single mapped chain, passed to the scorer as "=LOCUS". Without a locus from + // either source there is nothing to score against. + chainCol := firstWithDomain(chainCols, slot.chainDomain) + chainArg := undefined + label := chainLabel(keySpec, cdr3Col, slot.scChain) + if is_undefined(chainCol) { + chain := bareChain(keySpec, slot.scChain) + if is_undefined(chain) || is_undefined(CHAIN_LOCUS[chain]) { + return + } + chainArg = "=" + CHAIN_LOCUS[chain] + // Name the unit from the resolved chain rather than SC_CHAIN_LABELS, whose TCR rows + // are inverted; this keeps an imported set's labels agreeing with its loci. + label = CHAIN_LABELS[chain] + } + units += [{ suffix: is_undefined(slot.scChain) ? "bulk" : slot.scChain, domain: domain, - label: chainLabel(keySpec, cdr3Col, slot.scChain), + label: label, cdr3Col: cdr3Col, chainCol: chainCol, + chainArg: chainArg, seqCols: allWithDomain(cdr3Cols, slot.chainDomain) }] } @@ -159,11 +230,14 @@ self.body(func(args) { unitParams := [] for unit in units { tableBuilder.add(unit.cdr3Col, { header: "cdr3_" + unit.suffix }) - tableBuilder.add(unit.chainCol, { header: "chain_" + unit.suffix }) + if !is_undefined(unit.chainCol) { + tableBuilder.add(unit.chainCol, { header: "chain_" + unit.suffix }) + } unitParams += [{ suffix: unit.suffix, domain: unit.domain, - label: unit.label + label: unit.label, + chainArg: unit.chainArg }] } tableBuilder.cpu(1) From 9d3255fa300ac8914edf0459f4703bfd9dd80c38 Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 13:32:34 +0200 Subject: [PATCH 02/17] MILAB-6720: move CI to node 22 build.yaml and mark-stable.yaml, matching import-vdj-data (the producer this block now reads) and the other consumers adapted under this ticket. prepublishOnly checked and left alone: it is byte-identical to import-vdj-data, antibody-tcr-lead-selection, clonotype-space, rarefaction, repertoire-score and immune-assay-data, and carries no redundant pack step. No changeset: a build-time concern that changes nothing in the published packages. --- .github/workflows/build.yaml | 2 +- .github/workflows/mark-stable.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7a6c4c9..83c9fe3 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -24,7 +24,7 @@ jobs: with: app-name: 'Block: Generation Probability' app-name-slug: 'block-generation-probability' - node-version: '20.x' + node-version: '22.x' gha-runner-label: hz-ubuntu-dind build-script-name: 'build:dev-local' build-before-publish-script-name: 'build:release' diff --git a/.github/workflows/mark-stable.yaml b/.github/workflows/mark-stable.yaml index abb4f76..39e07c3 100644 --- a/.github/workflows/mark-stable.yaml +++ b/.github/workflows/mark-stable.yaml @@ -15,7 +15,7 @@ jobs: uses: milaboratory/github-ci/.github/workflows/block-mark-stable.yaml@v4 with: app-name: 'Block: Generation Probability - Mark Stable' - node-version: '20.x' + node-version: '22.x' npmrc-config: | { "registries": { From d37c4f36094feb3807ae7ee38dad2006d59c2bac Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 13:38:30 +0200 Subject: [PATCH 03/17] MILAB-6720: bump block-tools to 2.14.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared v4 CI gates publication on block-tools being at latest (node-simple-pnpm.yaml, "Check infrastructure requirements for publication"), and it passes only block-tools to require-latest — model and ui-vue are free to lag. Version bump only, no structure refresh. Both build paths verified green at the new version, software included, so the 2.13 build script rename and the local-software regression do not bite here. Note: 'block-tools structure check' now hard-fails this block for having no sibling kind/ package. That gate is not wired into any script or into CI (build.yaml runs build:dev-local and test), so it does not block publication — but the block does now formally owe a kind, whose BlockParams contract is an author decision. No changeset: a build-time concern that changes nothing in the published packages. --- pnpm-lock.yaml | 246 +++++++++++++++++++++++++++++++------------- pnpm-workspace.yaml | 2 +- 2 files changed, 174 insertions(+), 74 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 91790f8..ff2648b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -25,8 +25,8 @@ catalogs: specifier: 1.8.2 version: 1.8.2 '@platforma-sdk/block-tools': - specifier: 2.12.9 - version: 2.12.9 + specifier: 2.14.3 + version: 2.14.3 '@platforma-sdk/model': specifier: 1.80.10 version: 1.80.10 @@ -74,10 +74,10 @@ importers: version: 2.29.8(@types/node@26.1.1) '@milaboratories/ts-builder': specifier: 'catalog:' - version: 1.6.1(@types/node@26.1.1)(esbuild@0.28.1)(rollup@4.62.2)(vue@3.5.24(typescript@5.9.3))(yaml@2.9.0) + version: 1.6.1(@types/node@26.1.1)(esbuild@0.28.1)(rollup@4.62.2)(vue@3.5.24(typescript@7.0.2))(yaml@2.9.0) '@platforma-sdk/block-tools': specifier: 'catalog:' - version: 2.12.9(@types/node@26.1.1) + version: 2.14.3(@types/node@26.1.1) shx: specifier: 'catalog:' version: 0.4.0 @@ -104,7 +104,7 @@ importers: version: link:../workflow '@platforma-sdk/block-tools': specifier: 'catalog:' - version: 2.12.9(@types/node@26.1.1) + version: 2.14.3(@types/node@26.1.1) '@platforma-sdk/model': specifier: 'catalog:' version: 1.80.10 @@ -119,7 +119,7 @@ importers: dependencies: '@milaboratories/graph-maker': specifier: 'catalog:' - version: 1.6.1(@milaboratories/pl-model-common@1.47.3)(@platforma-sdk/model@1.80.10)(@platforma-sdk/ui-vue@1.80.10(@bytecodealliance/preview2-shim@0.17.9)(@vue/compiler-dom@3.5.40)(@vue/server-renderer@3.5.24(vue@3.5.24(typescript@7.0.2)))(typescript@7.0.2))(d3-dispatch@3.0.1)(d3-path@3.1.0)(d3-scale-chromatic@3.1.0)(typescript@7.0.2) + version: 1.6.1(@milaboratories/pl-model-common@1.48.0)(@platforma-sdk/model@1.80.10)(@platforma-sdk/ui-vue@1.80.10(@bytecodealliance/preview2-shim@0.17.9)(@vue/compiler-dom@3.5.40)(@vue/server-renderer@3.5.24(vue@3.5.24(typescript@7.0.2)))(typescript@7.0.2))(d3-dispatch@3.0.1)(d3-path@3.1.0)(d3-scale-chromatic@3.1.0)(typescript@7.0.2) '@milaboratories/helpers': specifier: 'catalog:' version: 1.14.5 @@ -141,7 +141,7 @@ importers: version: 1.3.1 '@platforma-sdk/block-tools': specifier: 'catalog:' - version: 2.12.9(@types/node@26.1.1) + version: 2.14.3(@types/node@26.1.1) software: devDependencies: @@ -150,7 +150,7 @@ importers: version: 1.8.2 '@platforma-sdk/block-tools': specifier: 'catalog:' - version: 2.12.9(@types/node@26.1.1) + version: 2.14.3(@types/node@26.1.1) test: dependencies: @@ -833,11 +833,20 @@ packages: resolution: {integrity: sha512-ILtqflBY9tzd79bKa87eS98vZFCso5/uYxuArfcKrmRDatRg7KWmI/Vr9+xOEf2Y2E/44HXMCx3JKZe/Y4yF6g==} engines: {node: '>=12.10.0'} + '@grpc/grpc-js@1.14.4': + resolution: {integrity: sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==} + engines: {node: '>=12.10.0'} + '@grpc/proto-loader@0.7.15': resolution: {integrity: sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==} engines: {node: '>=6'} hasBin: true + '@grpc/proto-loader@0.8.1': + resolution: {integrity: sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==} + engines: {node: '>=6'} + hasBin: true + '@inquirer/ansi@1.0.2': resolution: {integrity: sha512-S8qNSZiYzFd0wAcyG5AXCvUHC5Sr7xpZ9wZ2py9XR88jUz8wooStVx5M6dRzczbBWjic9NP7+rY0Xi7qqK/aMQ==} engines: {node: '>=18'} @@ -1100,6 +1109,10 @@ packages: resolution: {integrity: sha512-LljKMbKa8zl2lNFkg3VthMx9E6308B5vSH7dtWnlVz20ZU1GF7steToGv5gWs1lwUtm4GRU7kmmsMzU9a7l2yg==} engines: {node: '>=22.19.0'} + '@milaboratories/pl-client@3.14.7': + resolution: {integrity: sha512-HIjPfGAYRRD0hbq5YSTkWt5XgBiv+yYtw73EjWLxui8eUEeB2ffdgsvH7IhVx2oXjUOL4p7M4i8PBnmdJ3830w==} + engines: {node: '>=22.19.0'} + '@milaboratories/pl-config@1.8.5': resolution: {integrity: sha512-XnfYXSSkRxeImQ21k6I8y5apisvagcSgMGfEeyRxNdSGwUVJbbI8TwJk+XBEDQ4lErW8oqtLxKjFQuHJMzRUoQ==} @@ -1131,18 +1144,27 @@ packages: '@milaboratories/pl-model-backend@1.4.18': resolution: {integrity: sha512-cDSreI5DV25v9JmiRDMdp8c2+ZxJgeIjnybsuToVbBwDHoANFqB+2Nsxos5jzU2nxCx5kt2KYJw9/tweuSC4xQ==} + '@milaboratories/pl-model-backend@1.4.21': + resolution: {integrity: sha512-Z2z5J8bglslgXXoi1aySi1ho+/d+ylJiuEaRw9XTNuE5iBM3EQPe0rw095pXGOinTlMpmvUcCI2wkVCraUQ1dA==} + '@milaboratories/pl-model-common@1.46.2': resolution: {integrity: sha512-VEeauisApYScvCS8lnK3zpFJ520xuTAodKJmjR8ulHcMrWMyWMfHEdGb7j5OMD0mM/OwTgmQrrJ5eB7Xd+xoOQ==} '@milaboratories/pl-model-common@1.47.3': resolution: {integrity: sha512-tXmKujm+6ru/Fh9hr9H3iRBWbS+JE0Q7FNualtzJpijaB3SNtScR4erLTamdANv5RYNn7kbYpDWr6wAjAOajrg==} + '@milaboratories/pl-model-common@1.48.0': + resolution: {integrity: sha512-oCVrjFNmjQolb7YWnbSGHnp6GGVm1PhG1lnNABp9lqYjvA2ISjIPQLIo/vT2ca0mqwLrEvbuRqiqSFOg7sQ5tQ==} + '@milaboratories/pl-model-middle-layer@1.30.15': resolution: {integrity: sha512-oBkVlKR+qgsQR74N8G3aW3wBaJw78+9HJjnaNbH7QrTjKq4pno7wTXdrz7Ptk6Tu4SWfkcecXY6+xCnkDKr4eQ==} '@milaboratories/pl-model-middle-layer@1.30.7': resolution: {integrity: sha512-rs9x3Ron4ujR/UOdEgB8WUB1SvZ8ZAScT1Av/e4or+iiQ/CzhmK9nqtYamHVwKV+JgwIFbXQwxGvIHDVz955dQ==} + '@milaboratories/pl-model-middle-layer@1.32.0': + resolution: {integrity: sha512-X1iLGgOwzkw8mQ/GfTxfOTJakBJ26np85h5HqUziMbVkFL0SR7wpd7xpgUs6TKVkYNM8viAv3iA2k63Yc5SahQ==} + '@milaboratories/pl-tree@1.13.3': resolution: {integrity: sha512-39QT6opCX4ejjT0UI7dq4HMEs/NBXEI2nQyyaHbJBLKGKr6DKhUAUd0mLqeIjsEdH0B4eo3bKMlgry5PaPSlpQ==} engines: {node: '>=22.19.0'} @@ -1762,6 +1784,10 @@ packages: resolution: {integrity: sha512-wtqkPsjMpan+XjP0GHa8pcJbj3CX0Jmzxjxez0sz+jGMiIqpsNV3sSTQGYj1zkgYA9Wi0lusKiNoVOYB7Jtmbw==} hasBin: true + '@platforma-sdk/block-tools@2.14.3': + resolution: {integrity: sha512-CD0NPfUoXiJhl6JArC057oCXbqKkJf5HJsmrlZShnh6lKRaIQlibF8VUqbpBi9+PopsGCQ4/s5HnLR7wQoWDzg==} + hasBin: true + '@platforma-sdk/blocks-deps-updater@2.2.0': resolution: {integrity: sha512-p9lBxhFXM9WoRsrJO7dfkiXSK+1m63yIn1sKhBO71eMbhrLMyVYHEOeNf3w5OCdbRF5QsNhXzWuiTmFK3zHFsA==} hasBin: true @@ -1772,6 +1798,9 @@ packages: '@platforma-sdk/package-builder-lib@1.2.1': resolution: {integrity: sha512-H6weitj7JxbiJSlteEFLafTJ+tfty6iv/imf3ysy8oCS8AZIRJk2VMW3M/aAc+xVkQeX7oVIwMwFMYrJIoFsAg==} + '@platforma-sdk/package-builder-lib@1.3.0': + resolution: {integrity: sha512-CdBjmNo6E1fBxKYWaXa49L/L2WLURxs2f1TAqxLIZlHRE4DZ6E1TEj3jNNKESWp+/9rwtLkTAzmTzNPrDgz+2Q==} + '@platforma-sdk/tengo-builder@4.0.20': resolution: {integrity: sha512-4zf38sLzctOgbwym39+YdrE07/W61zzNaeZebZd0p2QDYNacAf83hRlD5KlDvUZtrDbhYNMgoZtghw61XAchog==} engines: {node: '>=22'} @@ -7754,6 +7783,11 @@ snapshots: '@grpc/proto-loader': 0.7.15 '@js-sdsl/ordered-map': 4.4.2 + '@grpc/grpc-js@1.14.4': + dependencies: + '@grpc/proto-loader': 0.8.1 + '@js-sdsl/ordered-map': 4.4.2 + '@grpc/proto-loader@0.7.15': dependencies: lodash.camelcase: 4.3.0 @@ -7761,6 +7795,13 @@ snapshots: protobufjs: 7.6.5 yargs: 17.7.3 + '@grpc/proto-loader@0.8.1': + dependencies: + lodash.camelcase: 4.3.0 + long: 5.3.2 + protobufjs: 7.6.5 + yargs: 17.7.3 + '@inquirer/ansi@1.0.2': {} '@inquirer/checkbox@4.3.2(@types/node@26.1.1)': @@ -8040,6 +8081,30 @@ snapshots: - supports-color - typescript + '@milaboratories/graph-maker@1.6.1(@milaboratories/pl-model-common@1.48.0)(@platforma-sdk/model@1.80.10)(@platforma-sdk/ui-vue@1.80.10(@bytecodealliance/preview2-shim@0.17.9)(@vue/compiler-dom@3.5.40)(@vue/server-renderer@3.5.24(vue@3.5.24(typescript@7.0.2)))(typescript@7.0.2))(d3-dispatch@3.0.1)(d3-path@3.1.0)(d3-scale-chromatic@3.1.0)(typescript@7.0.2)': + dependencies: + '@ag-grid-community/core': 32.3.9 + '@milaboratories/helpers': 1.14.5 + '@milaboratories/miplots4': 1.4.0(d3-dispatch@3.0.1)(d3-path@3.1.0)(d3-scale-chromatic@3.1.0) + '@milaboratories/pf-plots': 1.5.0(@milaboratories/pl-model-common@1.48.0)(@platforma-sdk/model@1.80.10) + '@platforma-sdk/model': 1.80.10 + '@platforma-sdk/ui-vue': 1.80.10(@bytecodealliance/preview2-shim@0.17.9)(@vue/compiler-dom@3.5.40)(@vue/server-renderer@3.5.24(vue@3.5.24(typescript@7.0.2)))(typescript@7.0.2) + '@types/d3-hierarchy': 3.1.7 + '@types/d3-scale': 4.0.9 + '@vueuse/core': 13.9.0(vue@3.5.24(typescript@7.0.2)) + ag-grid-vue3: 34.1.2(vue@3.5.24(typescript@7.0.2)) + canonicalize: 2.1.0 + d3-hierarchy: 3.1.2 + d3-scale: 4.0.2 + vue: 3.5.24(typescript@7.0.2) + transitivePeerDependencies: + - '@milaboratories/pl-model-common' + - d3-dispatch + - d3-path + - d3-scale-chromatic + - supports-color + - typescript + '@milaboratories/helpers@1.14.2': {} '@milaboratories/helpers@1.14.5': {} @@ -8105,6 +8170,14 @@ snapshots: canonicalize: 2.1.0 lodash: 4.18.1 + '@milaboratories/pf-plots@1.5.0(@milaboratories/pl-model-common@1.48.0)(@platforma-sdk/model@1.80.10)': + dependencies: + '@milaboratories/helpers': 1.14.5 + '@milaboratories/pl-model-common': 1.48.0 + '@platforma-sdk/model': 1.80.10 + canonicalize: 2.1.0 + lodash: 4.18.1 + '@milaboratories/pf-spec-driver@1.4.24(@bytecodealliance/preview2-shim@0.17.9)': dependencies: '@milaboratories/helpers': 1.14.5 @@ -8162,6 +8235,24 @@ snapshots: utility-types: 3.11.0 yaml: 2.9.0 + '@milaboratories/pl-client@3.14.7': + dependencies: + '@grpc/grpc-js': 1.14.4 + '@milaboratories/pl-http': 1.2.4 + '@milaboratories/pl-model-common': 1.48.0 + '@milaboratories/ts-helpers': 1.8.6 + '@protobuf-ts/grpc-transport': 2.11.1(@grpc/grpc-js@1.14.4) + '@protobuf-ts/runtime': 2.11.1 + '@protobuf-ts/runtime-rpc': 2.11.1 + canonicalize: 2.1.0 + denque: 2.1.0 + long: 5.3.2 + lru-cache: 11.5.2 + openapi-fetch: 0.15.2 + undici: 7.16.0 + utility-types: 3.11.0 + yaml: 2.9.0 + '@milaboratories/pl-config@1.8.5': dependencies: '@milaboratories/ts-helpers': 1.8.6 @@ -8281,6 +8372,12 @@ snapshots: canonicalize: 2.1.0 zod: 3.25.76 + '@milaboratories/pl-model-backend@1.4.21': + dependencies: + '@milaboratories/pl-client': 3.14.7 + canonicalize: 2.1.0 + zod: 3.25.76 + '@milaboratories/pl-model-common@1.46.2': dependencies: '@milaboratories/helpers': 1.14.2 @@ -8296,6 +8393,14 @@ snapshots: es-toolkit: 1.50.0 zod: 3.25.76 + '@milaboratories/pl-model-common@1.48.0': + dependencies: + '@milaboratories/helpers': 1.14.5 + '@milaboratories/pl-error-like': 1.12.10 + canonicalize: 2.1.0 + es-toolkit: 1.50.0 + zod: 3.25.76 + '@milaboratories/pl-model-middle-layer@1.30.15': dependencies: '@milaboratories/helpers': 1.14.5 @@ -8312,6 +8417,14 @@ snapshots: utility-types: 3.11.0 zod: 3.25.76 + '@milaboratories/pl-model-middle-layer@1.32.0': + dependencies: + '@milaboratories/helpers': 1.14.5 + '@milaboratories/pl-model-common': 1.48.0 + es-toolkit: 1.50.0 + utility-types: 3.11.0 + zod: 3.25.76 + '@milaboratories/pl-tree@1.13.3': dependencies: '@milaboratories/computable': 2.9.8 @@ -8332,47 +8445,6 @@ snapshots: '@milaboratories/tengo-tester@1.6.4': {} - '@milaboratories/ts-builder@1.6.1(@types/node@26.1.1)(esbuild@0.28.1)(rollup@4.62.2)(vue@3.5.24(typescript@5.9.3))(yaml@2.9.0)': - dependencies: - '@milaboratories/ts-configs': 1.3.1 - '@vitejs/plugin-vue': 6.0.8(vite@8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0))(vue@3.5.24(typescript@5.9.3)) - commander: 15.0.0 - jsonc-parser: 3.3.1 - oxfmt: 0.35.0 - oxlint: 1.63.0 - oxlint-plugin-eslint: 1.63.0 - rolldown: 1.2.0 - rolldown-plugin-dts: 0.26.0(rolldown@1.2.0)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@5.9.3)) - rollup-plugin-copy: 3.5.0 - rollup-plugin-sourcemaps2: 0.5.8(@types/node@26.1.1)(rollup@4.62.2) - typescript: 5.9.3 - vite: 8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0) - vite-plugin-commonjs: 0.10.4 - vite-plugin-dts: 4.5.4(@types/node@26.1.1)(rollup@4.62.2)(typescript@5.9.3)(vite@8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0)) - vite-plugin-externalize-deps: 0.10.0(vite@8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0)) - vite-plugin-lib-inject-css: 2.2.2(vite@8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0)) - vue-tsc: 3.3.5(typescript@5.9.3) - transitivePeerDependencies: - - '@ts-macro/tsc' - - '@types/node' - - '@typescript/native-preview' - - '@vitejs/devtools' - - esbuild - - jiti - - less - - oxc-resolver - - oxlint-tsgolint - - rollup - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - - tsx - - vue - - yaml - '@milaboratories/ts-builder@1.6.1(@types/node@26.1.1)(esbuild@0.28.1)(rollup@4.62.2)(vue@3.5.24(typescript@7.0.2))(yaml@2.9.0)': dependencies: '@milaboratories/ts-configs': 1.3.1 @@ -8383,7 +8455,7 @@ snapshots: oxlint: 1.63.0 oxlint-plugin-eslint: 1.63.0 rolldown: 1.2.0 - rolldown-plugin-dts: 0.26.0(rolldown@1.2.0)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@5.9.3)) + rolldown-plugin-dts: 0.26.0(rolldown@1.2.0)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@7.0.2)) rollup-plugin-copy: 3.5.0 rollup-plugin-sourcemaps2: 0.5.8(@types/node@26.1.1)(rollup@4.62.2) typescript: 5.9.3 @@ -8897,6 +8969,34 @@ snapshots: - bare-buffer - react-native-b4a + '@platforma-sdk/block-tools@2.14.3(@types/node@26.1.1)': + dependencies: + '@aws-sdk/client-ecr-public': 3.859.0 + '@aws-sdk/client-s3': 3.859.0 + '@inquirer/prompts': 7.10.1(@types/node@26.1.1) + '@milaboratories/pl-http': 1.2.4 + '@milaboratories/pl-model-backend': 1.4.21 + '@milaboratories/pl-model-common': 1.48.0 + '@milaboratories/pl-model-middle-layer': 1.32.0 + '@milaboratories/resolve-helper': 1.1.3 + '@milaboratories/ts-helpers': 1.8.6 + '@platforma-sdk/blocks-deps-updater': 2.2.0 + '@platforma-sdk/package-builder-lib': 1.3.0 + canonicalize: 2.1.0 + commander: 15.0.0 + lru-cache: 11.5.2 + mime-types: 2.1.35 + tar: 7.5.21 + undici: 7.16.0 + yaml: 2.9.0 + zod: 3.25.76 + transitivePeerDependencies: + - '@types/node' + - aws-crt + - bare-abort-controller + - bare-buffer + - react-native-b4a + '@platforma-sdk/blocks-deps-updater@2.2.0': dependencies: yaml: 2.9.0 @@ -8931,6 +9031,22 @@ snapshots: - bare-buffer - react-native-b4a + '@platforma-sdk/package-builder-lib@1.3.0': + dependencies: + '@aws-sdk/client-s3': 3.859.0 + '@aws-sdk/lib-storage': 3.859.0(@aws-sdk/client-s3@3.859.0) + '@milaboratories/resolve-helper': 1.1.3 + archiver: 7.0.1 + undici: 7.16.0 + winston: 3.19.0 + yaml: 2.9.0 + zod: 3.25.76 + transitivePeerDependencies: + - aws-crt + - bare-abort-controller + - bare-buffer + - react-native-b4a + '@platforma-sdk/tengo-builder@4.0.20': dependencies: '@milaboratories/pl-model-backend': 1.4.18 @@ -9053,6 +9169,12 @@ snapshots: '@protobuf-ts/runtime': 2.11.1 '@protobuf-ts/runtime-rpc': 2.11.1 + '@protobuf-ts/grpc-transport@2.11.1(@grpc/grpc-js@1.14.4)': + dependencies: + '@grpc/grpc-js': 1.14.4 + '@protobuf-ts/runtime': 2.11.1 + '@protobuf-ts/runtime-rpc': 2.11.1 + '@protobuf-ts/plugin@2.11.1': dependencies: '@bufbuild/protobuf': 2.13.0 @@ -12099,12 +12221,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@vitejs/plugin-vue@6.0.8(vite@8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0))(vue@3.5.24(typescript@5.9.3))': - dependencies: - '@rolldown/pluginutils': 1.0.1 - vite: 8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0) - vue: 3.5.24(typescript@5.9.3) - '@vitejs/plugin-vue@6.0.8(vite@8.1.5(@types/node@26.1.1)(esbuild@0.28.1)(yaml@2.9.0))(vue@3.5.24(typescript@7.0.2))': dependencies: '@rolldown/pluginutils': 1.0.1 @@ -12330,12 +12446,6 @@ snapshots: '@vue/shared': 3.5.24 csstype: 3.2.3 - '@vue/server-renderer@3.5.24(vue@3.5.24(typescript@5.9.3))': - dependencies: - '@vue/compiler-ssr': 3.5.24 - '@vue/shared': 3.5.24 - vue: 3.5.24(typescript@5.9.3) - '@vue/server-renderer@3.5.24(vue@3.5.24(typescript@7.0.2))': dependencies: '@vue/compiler-ssr': 3.5.24 @@ -13943,7 +14053,7 @@ snapshots: reusify@1.1.0: {} - rolldown-plugin-dts@0.26.0(rolldown@1.2.0)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@5.9.3)): + rolldown-plugin-dts@0.26.0(rolldown@1.2.0)(typescript@5.9.3)(vue-tsc@3.3.5(typescript@7.0.2)): dependencies: '@babel/generator': 8.0.0 '@babel/helper-validator-identifier': 8.0.4 @@ -14591,16 +14701,6 @@ snapshots: '@vue/language-core': 3.3.5 typescript: 5.9.3 - vue@3.5.24(typescript@5.9.3): - dependencies: - '@vue/compiler-dom': 3.5.24 - '@vue/compiler-sfc': 3.5.24 - '@vue/runtime-dom': 3.5.24 - '@vue/server-renderer': 3.5.24(vue@3.5.24(typescript@5.9.3)) - '@vue/shared': 3.5.24 - optionalDependencies: - typescript: 5.9.3 - vue@3.5.24(typescript@7.0.2): dependencies: '@vue/compiler-dom': 3.5.24 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 6d53f82..5b2809c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -12,7 +12,7 @@ catalog: "@milaboratories/ts-builder": 1.6.1 "@milaboratories/ts-configs": 1.3.1 "@platforma-open/milaboratories.runenv-python-3": 1.8.2 - "@platforma-sdk/block-tools": 2.12.9 + "@platforma-sdk/block-tools": 2.14.3 "@platforma-sdk/model": 1.80.10 "@platforma-sdk/package-builder": 3.14.2 "@platforma-sdk/tengo-builder": 4.0.20 From 6021a5fcc56ad0916fb4ca08a48f8a5a47197f7f Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 15:51:46 +0200 Subject: [PATCH 04/17] MILAB-6720: add the constant chain column with pt, not in the scorer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review point: the locus for a unit with no per-row chain column can be materialised inside the block, so the scoring script does not need to know about constants at all. Replaces the '=LOCUS' marker and its resolve_chain_literals() handler in main.py with a pt step that adds chain_ as a literal over the built parquet. software/src/main.py is now byte-identical to main again, which drops the software package from the changeset: no version bump, and no image rebuild for a change that is only about where the locus comes from. Given how much friction rebuilding and republishing software just cost, that is the larger win here. The step is gated on a literal actually being needed, so bulk and single-cell inputs keep exactly the graph they had — no extra pt run, no extra parquet pass. Only an imported set pays the extra read/write, and it is the case that could not run at all before. The locus derivation itself is unchanged: bareChain() plus CHAIN_LOCUS still resolve IGHeavy/TCRAlpha/TCRBeta to IGH/TRA/TRB, and IGLight still passes through unresolved so the scorer skips it and names it in the skipped-chains report. My earlier claim that no Tengo-only route existed was wrong — I checked the xsv/parquet file builder, which takes PColumns only, and never checked pt. --- .changeset/variantkey-imported-sets.md | 6 ++-- software/src/main.py | 33 +++------------------- workflow/src/pgen-process.tpl.tengo | 8 +----- workflow/src/score.tpl.tengo | 38 +++++++++++++++++++++----- 4 files changed, 39 insertions(+), 46 deletions(-) diff --git a/.changeset/variantkey-imported-sets.md b/.changeset/variantkey-imported-sets.md index f268dcf..7ab997d 100644 --- a/.changeset/variantkey-imported-sets.md +++ b/.changeset/variantkey-imported-sets.md @@ -1,5 +1,4 @@ --- -'@platforma-open/milaboratories.generation-probability.software': minor '@platforma-open/milaboratories.generation-probability.workflow': minor '@platforma-open/milaboratories.generation-probability.model': minor '@platforma-open/milaboratories.generation-probability.block': minor @@ -20,8 +19,9 @@ vocabulary its axis does not use — so the run died on "Cannot determine CDR3 a set is amino acid throughout, which is now the answer. Scoring needed a per-row locus column and quietly produced an empty result without one. A unit -whose locus is constant now passes it to the scorer as `=LOCUS`, which materialises it as a -column, leaving the grouping, joining and unsupported-chain reporting unchanged. +whose locus is constant now gets that column added by a `pt` step over the built table, so the +scoring script is unchanged and still simply reads a locus per row. The step runs only when a +constant is actually needed, so bulk and single-cell inputs keep the graph they had. Whether the two chains are scored separately followed the key axis being `pl7.app/vdj/scClonotypeKey`. A paired imported set carries both chains in one frame under the diff --git a/software/src/main.py b/software/src/main.py index 7e0328d..420ff61 100644 --- a/software/src/main.py +++ b/software/src/main.py @@ -114,7 +114,6 @@ def parse_args() -> Args: required=True, action="append", dest="chain_columns", - help="column holding each row's locus, or =LOCUS for a unit with a constant locus", ) parser.add_argument( "--pgen-column", @@ -132,39 +131,15 @@ def parse_args() -> Args: return Args(**vars(parser.parse_args())) -def resolve_chain_literals(df: pl.LazyFrame, chain_args: list[str]) -> tuple[pl.LazyFrame, list[str]]: - """Materialise `=LOCUS` chain arguments as constant columns. - - A dataset whose locus is a property of the whole unit rather than of each row -- an - imported receptor set, which carries no per-record `pl7.app/vdj/chain` column -- passes - `=IGH` instead of a column name. Adding it as a real column here keeps every step - downstream (group_by, join, the unsupported-chain report) working unchanged. - """ - literals: dict[str, str] = {} - resolved: list[str] = [] - for index, argument in enumerate(chain_args): - if not argument.startswith("="): - resolved.append(argument) - continue - name = f"_chain_literal_{index}" - literals[name] = argument[1:] - resolved.append(name) - if literals: - df = df.with_columns([pl.lit(v).alias(k) for k, v in literals.items()]) - return df, resolved - - def main() -> None: args = parse_args() df = pl.scan_parquet(args.input) - # Args is frozen, so the resolved names travel as a local. - df, chain_columns = resolve_chain_literals(df, args.chain_columns) columns = set(df.collect_schema().names()) missing_columns = { args.key_column, *args.sequence_columns, - *chain_columns, + *args.chain_columns, } - columns if missing_columns: msg = f"columns not found in {args.input}: {missing_columns}" @@ -172,10 +147,10 @@ def main() -> None: if args.workers < 1: msg = f"--workers must be >= 1, got {args.workers}" raise SystemExit(msg) - if len(args.sequence_columns) != len(chain_columns): + if len(args.sequence_columns) != len(args.chain_columns): msg = ( "number of sequence columns and chain columns do not match " - f"({len(args.sequence_columns)} ≠ {len(chain_columns)})" + f"({len(args.sequence_columns)} ≠ {len(args.chain_columns)})" ) raise SystemExit(msg) @@ -219,7 +194,7 @@ def process_batch(batch: pl.Series) -> pl.Series: for sequence, chain, pgen, neg_log_pgen in zip( args.sequence_columns, - chain_columns, + args.chain_columns, args.pgen_columns, args.neg_log_pgen_columns, strict=True, diff --git a/workflow/src/pgen-process.tpl.tengo b/workflow/src/pgen-process.tpl.tengo index 482237a..7af8828 100644 --- a/workflow/src/pgen-process.tpl.tengo +++ b/workflow/src/pgen-process.tpl.tengo @@ -24,13 +24,7 @@ self.body(func(args) { arg("--key-column").arg("clonotypeKey") for unit in units { cmd = cmd.arg("--sequence-column").arg("cdr3_" + unit.suffix) - // A unit with a constant locus (an imported set, which has no per-row chain column) - // carries it as "=LOCUS"; the scorer materialises that as a column. - if is_undefined(unit.chainArg) { - cmd = cmd.arg("--chain-column").arg("chain_" + unit.suffix) - } else { - cmd = cmd.arg("--chain-column").arg(unit.chainArg) - } + cmd = cmd.arg("--chain-column").arg("chain_" + unit.suffix) cmd = cmd.arg("--pgen-column").arg("pgen_" + unit.suffix) cmd = cmd.arg("--neg-log-pgen-column").arg("neglog_" + unit.suffix) } diff --git a/workflow/src/score.tpl.tengo b/workflow/src/score.tpl.tengo index 2cdfb4a..7444c2c 100644 --- a/workflow/src/score.tpl.tengo +++ b/workflow/src/score.tpl.tengo @@ -4,6 +4,7 @@ smart := import("@platforma-sdk/workflow-tengo:smart") assets := import("@platforma-sdk/workflow-tengo:assets") pframes := import("@platforma-sdk/workflow-tengo:pframes") pBundle := import("@platforma-sdk/workflow-tengo:pframes.bundle") +pt := import("@platforma-sdk/workflow-tengo:pt") pgenTpl := assets.importTemplate(":pgen-process") @@ -182,17 +183,17 @@ self.body(func(args) { } // Preferred: the per-row locus column. An imported set has none, so fall back to the - // unit's single mapped chain, passed to the scorer as "=LOCUS". Without a locus from - // either source there is nothing to score against. + // unit's single mapped chain and materialise it as a constant column below. Without a + // locus from either source there is nothing to score against. chainCol := firstWithDomain(chainCols, slot.chainDomain) - chainArg := undefined + chainLiteral := undefined label := chainLabel(keySpec, cdr3Col, slot.scChain) if is_undefined(chainCol) { chain := bareChain(keySpec, slot.scChain) if is_undefined(chain) || is_undefined(CHAIN_LOCUS[chain]) { return } - chainArg = "=" + CHAIN_LOCUS[chain] + chainLiteral = CHAIN_LOCUS[chain] // Name the unit from the resolved chain rather than SC_CHAIN_LABELS, whose TCR rows // are inverted; this keeps an imported set's labels agreeing with its loci. label = CHAIN_LABELS[chain] @@ -204,7 +205,7 @@ self.body(func(args) { label: label, cdr3Col: cdr3Col, chainCol: chainCol, - chainArg: chainArg, + chainLiteral: chainLiteral, seqCols: allWithDomain(cdr3Cols, slot.chainDomain) }] } @@ -236,14 +237,37 @@ self.body(func(args) { unitParams += [{ suffix: unit.suffix, domain: unit.domain, - label: unit.label, - chainArg: unit.chainArg + label: unit.label }] } tableBuilder.cpu(1) tableBuilder.mem("4GiB") inputTable := tableBuilder.build() + // Units with a constant locus carry no chain column, so add one here rather than teaching + // the scorer about constants: main.py stays a pure per-row scorer and the software package + // stays unchanged — no version bump, no image rebuild for a change that is purely about + // where the locus comes from. + // + // Only runs when a literal is actually needed, so bulk and single-cell inputs keep exactly + // the graph they had: no extra step, no extra parquet pass. + literalUnits := [] + for unit in units { + if !is_undefined(unit.chainLiteral) { + literalUnits += [unit] + } + } + if len(literalUnits) > 0 { + ptWf := pt.workflow().mem("4GiB").cpu(1) + df := ptWf.frame(inputTable, { xsvType: "parquet" }) + exprs := [] + for unit in literalUnits { + exprs += [pt.lit(unit.chainLiteral).alias("chain_" + unit.suffix)] + } + df.withColumns(exprs...).save("with-chain.parquet") + inputTable = ptWf.run().getFile("with-chain.parquet") + } + // Specs in a keyed map so ordering can't perturb the params hash; data passed separately. renderInputs := { inputTable: inputTable } seqSpecs := {} From c59b0b8c45bf4b984989fdf4fb6781abb585c79e Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 16:34:56 +0200 Subject: [PATCH 05/17] fix: swapped chain labels on single-cell TCR datasets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MiXCR fixes "A" as the more diverse chain, so TCRAB orders TCRBeta/TCRAlpha and TCRGD orders TCRDelta/TCRGamma (mixcr-clonotyping/workflow/src/process.tpl.tengo:43,695). SC_CHAIN_LABELS read A as Alpha and B as Beta, and gamma/delta the same way, so a single-cell TCR alpha/beta dataset labelled its alpha column "(Beta)" and its beta column "(Alpha)". The values were never wrong — the locus is read from the per-record pl7.app/vdj/chain column, not from the A/B letter — but a mislabelled column reads as the other chain's result, which is worse than a missing label. Fixed by deleting SC_CHAIN_LABELS rather than correcting it. The same A/B-to-chain knowledge was written twice, in that table and in RECEPTOR_CHAINS, and having two copies is what let them drift apart; one of them was bound to be wrong. chainLabel now resolves the slot through RECEPTOR_CHAINS and names the result with CHAIN_LABELS, so the label and the OLGA model come from one source and cannot disagree. That also removes the reason the imported-set branch sets its own label, but it is kept: it takes the label from the very chain value the locus was derived from, which is a stronger guarantee than deriving both from the same table separately. --- .changeset/fix-sc-tcr-chain-labels.md | 23 ++++++++++ workflow/src/score.tpl.tengo | 60 ++++++++++++--------------- 2 files changed, 49 insertions(+), 34 deletions(-) create mode 100644 .changeset/fix-sc-tcr-chain-labels.md diff --git a/.changeset/fix-sc-tcr-chain-labels.md b/.changeset/fix-sc-tcr-chain-labels.md new file mode 100644 index 0000000..ede31ed --- /dev/null +++ b/.changeset/fix-sc-tcr-chain-labels.md @@ -0,0 +1,23 @@ +--- +'@platforma-open/milaboratories.generation-probability.workflow': patch +'@platforma-open/milaboratories.generation-probability': patch +--- + +Fix swapped chain labels on single-cell TCR datasets + +The A/B slot was mapped to a chain name twice, and the two disagreed. MiXCR fixes "A" as the +more diverse chain — the one that recombines a D segment — so a receptor's chain order is +TCRBeta/TCRAlpha and TCRDelta/TCRGamma, not alphabetical. The label table read A as Alpha and +B as Beta, so on a single-cell TCR alpha/beta dataset the alpha column was labelled +"Generation probability (Beta)" and the beta column "(Alpha)". Gamma/delta was swapped the +same way. + +The Pgen values themselves were always right: the locus came from the per-record +`pl7.app/vdj/chain` column, never from the A/B letter. Only the label was wrong — but a +mislabelled column is read as the wrong chain's result. + +Labels are now derived from the chain that the A/B slot resolves to, through the single table +that also decides the locus, so a label cannot disagree with the model that produced the value +beside it. The duplicate label table is gone, which is what allowed the two to drift apart. + +Single-cell IG datasets, bulk and imported sets are unaffected. diff --git a/workflow/src/score.tpl.tengo b/workflow/src/score.tpl.tengo index 7444c2c..e4f9491 100644 --- a/workflow/src/score.tpl.tengo +++ b/workflow/src/score.tpl.tengo @@ -17,23 +17,42 @@ CHAIN_LABELS := { "TCRDelta": "Delta" } -SC_CHAIN_LABELS := { - "IG": { "A": "Heavy", "B": "Light" }, - "TCRAB": { "A": "Alpha", "B": "Beta" }, - "TCRGD": { "A": "Gamma", "B": "Delta" } +// Which chain each A/B slot holds. MiXCR fixes "A" as the more diverse chain — the one that +// recombines a D segment — so a receptor's chain order is TCRBeta/TCRAlpha and +// TCRDelta/TCRGamma, not alphabetical (mixcr-clonotyping/workflow/src/process.tpl.tengo:43,695). +RECEPTOR_CHAINS := { + "IG": { "A": "IGHeavy", "B": "IGLight" }, + "TCRAB": { "A": "TCRBeta", "B": "TCRAlpha" }, + "TCRGD": { "A": "TCRDelta", "B": "TCRGamma" } +} + +// The OLGA locus each chain maps to. IGLight is deliberately not a locus: OLGA needs IGK and +// IGL as separate models and an imported set does not record which, so it is passed through +// unchanged — the scorer skips any chain it has no model for and names it in the skipped-chains +// report, which the model already surfaces. +CHAIN_LOCUS := { + "IGHeavy": "IGH", + "IGLight": "IGLight", + "TCRAlpha": "TRA", + "TCRBeta": "TRB", + "TCRGamma": "TRG", + "TCRDelta": "TRD" } +// Labels come from the chain, and the A/B slot resolves to a chain through RECEPTOR_CHAINS — +// the same table the locus is derived from, so a label can never disagree with the model that +// produced the value beside it. chainLabel := func(keySpec, cdr3Col, scChain) { if is_undefined(scChain) { return CHAIN_LABELS[keySpec.domain["pl7.app/vdj/chain"]] } axis := cdr3Col.spec.axesSpec[0] axisDomain := is_undefined(axis.domain) ? {} : axis.domain - receptor := axisDomain["pl7.app/vdj/receptor"] - if is_undefined(receptor) || is_undefined(SC_CHAIN_LABELS[receptor]) { + byLetter := RECEPTOR_CHAINS[axisDomain["pl7.app/vdj/receptor"]] + if is_undefined(byLetter) { return undefined } - return SC_CHAIN_LABELS[receptor][scChain] + return CHAIN_LABELS[byLetter[scChain]] } matchDomain := func(col, required) { @@ -74,31 +93,6 @@ scSlot := func(scChain) { } BULK_SLOT := { scChain: undefined, chainDomain: {} } -// MiXCR's A/B convention: "A" is always the more diverse chain, so a receptor's chain order is -// TCRBeta/TCRAlpha and TCRDelta/TCRGamma rather than alphabetical -// (mixcr-clonotyping/workflow/src/process.tpl.tengo:43,695). -// -// SC_CHAIN_LABELS above has both TCR rows inverted against this. That is a pre-existing -// labelling bug, and it is deliberately not reused here: a wrong letter in THIS table would -// pick the wrong OLGA model and produce wrong Pgen values, not just a wrong label. -RECEPTOR_CHAINS := { - "IG": { "A": "IGHeavy", "B": "IGLight" }, - "TCRAB": { "A": "TCRBeta", "B": "TCRAlpha" }, - "TCRGD": { "A": "TCRDelta", "B": "TCRGamma" } -} - -// The OLGA locus each chain maps to. IGLight is deliberately not a locus: OLGA needs IGK and -// IGL as separate models and an imported set does not record which, so it is passed through -// unchanged — the scorer skips any chain it has no model for and names it in the skipped-chains -// report, which the model already surfaces. -CHAIN_LOCUS := { - "IGHeavy": "IGH", - "IGLight": "IGLight", - "TCRAlpha": "TRA", - "TCRBeta": "TRB", - "TCRGamma": "TRG", - "TCRDelta": "TRD" -} // An imported receptor set carries no per-record pl7.app/vdj/chain column: the locus is a // property of the whole unit. Resolve it from the chain the scientist mapped — on the key axis @@ -194,8 +188,6 @@ self.body(func(args) { return } chainLiteral = CHAIN_LOCUS[chain] - // Name the unit from the resolved chain rather than SC_CHAIN_LABELS, whose TCR rows - // are inverted; this keeps an imported set's labels agreeing with its loci. label = CHAIN_LABELS[chain] } From 5a790220c4c32e279c8a60e5198314d369fef7cc Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 17:24:44 +0200 Subject: [PATCH 06/17] MILAB-6720: keep CHAIN_LOCUS to real loci; unmapped chains travel by name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CHAIN_LOCUS claimed 'IGLight' was a locus so that the scorer would skip it. It is not a locus, and a table of loci that contains a non-locus invites a reader to trust it. The map now holds only names OLGA could recognise, and a chain with no entry sends its own name instead. The scorer has no model under that name either, so the outcome is unchanged — null score, and the name in the 'skipped: ' line that MainPage.vue renders as a banner ('no recombination model for: IGLight'). The behaviour is identical; only the lie is gone. Falling through rather than dropping the unit is deliberate: the banner exists only because the scorer sees the chain. Returning early in tengo would leave the light chain absent with nothing on screen to say it was there, which is worse than an empty column. Only IGLight is affected. Every other imported chain has a real locus, including TCRGamma and TCRDelta: TRG and TRD are genuine loci that OLGA ships no model for, so they belong in the map and are skipped one level further down. Also drops a temp map in the arrangement scan in favour of the short-circuit the SDK uses itself (pframes/xsv-builder.lib.tengo:123). Tengo has no optional chaining and maps.lib exports no safe-get, so this is as short as it goes. --- workflow/src/score.tpl.tengo | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/workflow/src/score.tpl.tengo b/workflow/src/score.tpl.tengo index e4f9491..0206f8d 100644 --- a/workflow/src/score.tpl.tengo +++ b/workflow/src/score.tpl.tengo @@ -26,13 +26,11 @@ RECEPTOR_CHAINS := { "TCRGD": { "A": "TCRDelta", "B": "TCRGamma" } } -// The OLGA locus each chain maps to. IGLight is deliberately not a locus: OLGA needs IGK and -// IGL as separate models and an imported set does not record which, so it is passed through -// unchanged — the scorer skips any chain it has no model for and names it in the skipped-chains -// report, which the model already surfaces. +// The OLGA locus each chain maps to. A chain is absent here only when no locus can be named +// for it at all: an imported light chain is "IG Light", and OLGA needs IGK and IGL as separate +// models, so which one it is simply is not recorded. CHAIN_LOCUS := { "IGHeavy": "IGH", - "IGLight": "IGLight", "TCRAlpha": "TRA", "TCRBeta": "TRB", "TCRGamma": "TRG", @@ -158,8 +156,8 @@ self.body(func(args) { isSingleCell := keySpec.name == "pl7.app/vdj/scClonotypeKey" if !isSingleCell && keySpec.name == "pl7.app/variantKey" { for col in cdr3Cols { - colDomain := is_undefined(col.spec.domain) ? {} : col.spec.domain - if !is_undefined(colDomain["pl7.app/vdj/scClonotypeChain/index"]) { + if !is_undefined(col.spec.domain) && + !is_undefined(col.spec.domain["pl7.app/vdj/scClonotypeChain/index"]) { isSingleCell = true } } @@ -184,10 +182,17 @@ self.body(func(args) { label := chainLabel(keySpec, cdr3Col, slot.scChain) if is_undefined(chainCol) { chain := bareChain(keySpec, slot.scChain) - if is_undefined(chain) || is_undefined(CHAIN_LOCUS[chain]) { + if is_undefined(chain) { return } + // With no locus for this chain, send its own name: the scorer has no model under + // that name either, so it leaves the score null and reports the name — which the + // block shows as a banner. Dropping the unit instead would lose the chain with + // nothing on screen to say so. chainLiteral = CHAIN_LOCUS[chain] + if is_undefined(chainLiteral) { + chainLiteral = chain + } label = CHAIN_LABELS[chain] } From 3f0fbfe01f298952551259b848faa1a36327afd8 Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 17:29:40 +0200 Subject: [PATCH 07/17] MILAB-6720: collect the chain literals in the loop that builds the table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The loop above tableBuilder already visits every unit, so gathering the pt expressions there removes a second pass over units and the literalUnits list it existed to produce. It also states the invariant in the code rather than in a comment: a unit either has a chain column, which goes into the table, or a constant locus, which is queued as a literal — the if/else makes that exclusive by construction, where two separate loops left it implicit. No behaviour change; the pt pass still runs only when a literal exists. --- workflow/src/score.tpl.tengo | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/workflow/src/score.tpl.tengo b/workflow/src/score.tpl.tengo index 0206f8d..4fd2b44 100644 --- a/workflow/src/score.tpl.tengo +++ b/workflow/src/score.tpl.tengo @@ -226,9 +226,15 @@ self.body(func(args) { tableBuilder := pframes.parquetFileBuilder() tableBuilder.setAxisHeader(keySpec, "clonotypeKey") unitParams := [] + chainExprs := [] for unit in units { tableBuilder.add(unit.cdr3Col, { header: "cdr3_" + unit.suffix }) - if !is_undefined(unit.chainCol) { + if is_undefined(unit.chainCol) { + // A unit with a constant locus has no column to add; queue it as a literal for the + // pass below instead of teaching the scorer about constants, which keeps main.py a + // pure per-row scorer and the software package unchanged. + chainExprs += [pt.lit(unit.chainLiteral).alias("chain_" + unit.suffix)] + } else { tableBuilder.add(unit.chainCol, { header: "chain_" + unit.suffix }) } unitParams += [{ @@ -241,27 +247,13 @@ self.body(func(args) { tableBuilder.mem("4GiB") inputTable := tableBuilder.build() - // Units with a constant locus carry no chain column, so add one here rather than teaching - // the scorer about constants: main.py stays a pure per-row scorer and the software package - // stays unchanged — no version bump, no image rebuild for a change that is purely about - // where the locus comes from. - // - // Only runs when a literal is actually needed, so bulk and single-cell inputs keep exactly - // the graph they had: no extra step, no extra parquet pass. - literalUnits := [] - for unit in units { - if !is_undefined(unit.chainLiteral) { - literalUnits += [unit] - } - } - if len(literalUnits) > 0 { + // Only when a literal is actually needed, so bulk and single-cell inputs keep exactly the + // graph they had: no extra step, no extra parquet pass. + if len(chainExprs) > 0 { ptWf := pt.workflow().mem("4GiB").cpu(1) - df := ptWf.frame(inputTable, { xsvType: "parquet" }) - exprs := [] - for unit in literalUnits { - exprs += [pt.lit(unit.chainLiteral).alias("chain_" + unit.suffix)] - } - df.withColumns(exprs...).save("with-chain.parquet") + ptWf.frame(inputTable, { xsvType: "parquet" }). + withColumns(chainExprs...). + save("with-chain.parquet") inputTable = ptWf.run().getFile("with-chain.parquet") } From e4110898057e5e8ce9f08f4c8da23c65e6f86bf0 Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 17:33:14 +0200 Subject: [PATCH 08/17] MILAB-6720: hoist the input-table resource budget to one constant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pt pass repeated tableBuilder's cpu(1)/mem("4GiB"), so a change to one would silently leave the other behind. Both are column shuffles over the same table, which is the reason the numbers match — now stated once, where a reader sees it, instead of twice as bare literals. --- workflow/src/score.tpl.tengo | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/workflow/src/score.tpl.tengo b/workflow/src/score.tpl.tengo index 4fd2b44..4a06bf1 100644 --- a/workflow/src/score.tpl.tengo +++ b/workflow/src/score.tpl.tengo @@ -8,6 +8,11 @@ pt := import("@platforma-sdk/workflow-tengo:pt") pgenTpl := assets.importTemplate(":pgen-process") +// Both passes over the input table are column shuffles, not computation — assembling it +// and appending a constant cost the same, so they share one budget. +TABLE_CPU := 1 +TABLE_MEM := "4GiB" + CHAIN_LABELS := { "IGHeavy": "Heavy", "IGLight": "Light", @@ -243,14 +248,14 @@ self.body(func(args) { label: unit.label }] } - tableBuilder.cpu(1) - tableBuilder.mem("4GiB") + tableBuilder.cpu(TABLE_CPU) + tableBuilder.mem(TABLE_MEM) inputTable := tableBuilder.build() // Only when a literal is actually needed, so bulk and single-cell inputs keep exactly the // graph they had: no extra step, no extra parquet pass. if len(chainExprs) > 0 { - ptWf := pt.workflow().mem("4GiB").cpu(1) + ptWf := pt.workflow().mem(TABLE_MEM).cpu(TABLE_CPU) ptWf.frame(inputTable, { xsvType: "parquet" }). withColumns(chainExprs...). save("with-chain.parquet") From d9f22a259e9de33cfa68159da7e801cb21829819 Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 17:38:07 +0200 Subject: [PATCH 09/17] MILAB-6720: assemble the scorer's input table in one pt pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit parquetFileBuilder assembled the table and a second pt pass appended the constant chain column, so the whole table was written, read and rewritten for the sake of one literal. pt can source the columns itself — frameFromColumnBundle joins bundle columns on their common axes — so the builder is gone and there is one step, one resource budget, one pass. The renames that setAxisHeader and the per-add headers used to do are now a select(): frameFromColumnBundle names the axis by pSpec.getAxisId and each column by its bundle key, so the select maps those to clonotypeKey and cdr3_/chain_. The literals are appended with withColumns afterwards, still only when a unit has no chain column. TABLE_CPU/TABLE_MEM are dropped with the second builder; a single call needs no constant. Two things this deliberately gives up, both accepted on request: The pt path is no longer gated. It previously ran only for imported sets, leaving bulk and single-cell on the long-proven parquetFileBuilder; now every input shape is assembled by pt. That is the price of one pass, and it widens what a pt-side mistake can break. frameFromColumnBundle has no other consumer in the workspace — no block uses it — so this is its first use here. pSpec.getAxisId producing exactly the name the frame carries for the axis is the part most worth checking on the first real run. --- workflow/src/score.tpl.tengo | 42 +++++++++++++++++------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/workflow/src/score.tpl.tengo b/workflow/src/score.tpl.tengo index 4a06bf1..5381fff 100644 --- a/workflow/src/score.tpl.tengo +++ b/workflow/src/score.tpl.tengo @@ -5,13 +5,10 @@ assets := import("@platforma-sdk/workflow-tengo:assets") pframes := import("@platforma-sdk/workflow-tengo:pframes") pBundle := import("@platforma-sdk/workflow-tengo:pframes.bundle") pt := import("@platforma-sdk/workflow-tengo:pt") +pSpec := import("@platforma-sdk/workflow-tengo:pframes.spec") pgenTpl := assets.importTemplate(":pgen-process") -// Both passes over the input table are column shuffles, not computation — assembling it -// and appending a constant cost the same, so they share one budget. -TABLE_CPU := 1 -TABLE_MEM := "4GiB" CHAIN_LABELS := { "IGHeavy": "Heavy", @@ -228,19 +225,26 @@ self.body(func(args) { } } - tableBuilder := pframes.parquetFileBuilder() - tableBuilder.setAxisHeader(keySpec, "clonotypeKey") - unitParams := [] + // The scorer's input table, assembled in a single pass: pt joins the bundle columns on the + // key axis, renames them to the headers the scorer expects, and appends a constant column + // for any unit whose locus is not per-row. Doing this with parquetFileBuilder and then a + // second pt pass wrote the whole table twice for the sake of one constant. + ptWf := pt.workflow().mem("4GiB").cpu(1) + + colKeys := [] + selects := [pt.col(pSpec.getAxisId(keySpec)).alias("clonotypeKey")] chainExprs := [] + unitParams := [] for unit in units { - tableBuilder.add(unit.cdr3Col, { header: "cdr3_" + unit.suffix }) + colKeys += [unit.cdr3Col.key] + selects += [pt.col(unit.cdr3Col.key).alias("cdr3_" + unit.suffix)] if is_undefined(unit.chainCol) { - // A unit with a constant locus has no column to add; queue it as a literal for the - // pass below instead of teaching the scorer about constants, which keeps main.py a - // pure per-row scorer and the software package unchanged. + // No column to join for this unit; its locus is one value for every row. Appending + // it here keeps main.py a pure per-row scorer and the software package unchanged. chainExprs += [pt.lit(unit.chainLiteral).alias("chain_" + unit.suffix)] } else { - tableBuilder.add(unit.chainCol, { header: "chain_" + unit.suffix }) + colKeys += [unit.chainCol.key] + selects += [pt.col(unit.chainCol.key).alias("chain_" + unit.suffix)] } unitParams += [{ suffix: unit.suffix, @@ -248,19 +252,13 @@ self.body(func(args) { label: unit.label }] } - tableBuilder.cpu(TABLE_CPU) - tableBuilder.mem(TABLE_MEM) - inputTable := tableBuilder.build() - // Only when a literal is actually needed, so bulk and single-cell inputs keep exactly the - // graph they had: no extra step, no extra parquet pass. + df := ptWf.frameFromColumnBundle(columns, { columns: colKeys }).select(selects...) if len(chainExprs) > 0 { - ptWf := pt.workflow().mem(TABLE_MEM).cpu(TABLE_CPU) - ptWf.frame(inputTable, { xsvType: "parquet" }). - withColumns(chainExprs...). - save("with-chain.parquet") - inputTable = ptWf.run().getFile("with-chain.parquet") + df = df.withColumns(chainExprs...) } + df.save("input.parquet") + inputTable := ptWf.run().getFile("input.parquet") // Specs in a keyed map so ordering can't perturb the params hash; data passed separately. renderInputs := { inputTable: inputTable } From f963a97bfd3a196ddc176a8a7f857202f5c3d9b1 Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 17:41:41 +0200 Subject: [PATCH 10/17] Revert "MILAB-6720: assemble the scorer's input table in one pt pass" This reverts commit d9f22a259e9de33cfa68159da7e801cb21829819. --- workflow/src/score.tpl.tengo | 42 +++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/workflow/src/score.tpl.tengo b/workflow/src/score.tpl.tengo index 5381fff..4a06bf1 100644 --- a/workflow/src/score.tpl.tengo +++ b/workflow/src/score.tpl.tengo @@ -5,10 +5,13 @@ assets := import("@platforma-sdk/workflow-tengo:assets") pframes := import("@platforma-sdk/workflow-tengo:pframes") pBundle := import("@platforma-sdk/workflow-tengo:pframes.bundle") pt := import("@platforma-sdk/workflow-tengo:pt") -pSpec := import("@platforma-sdk/workflow-tengo:pframes.spec") pgenTpl := assets.importTemplate(":pgen-process") +// Both passes over the input table are column shuffles, not computation — assembling it +// and appending a constant cost the same, so they share one budget. +TABLE_CPU := 1 +TABLE_MEM := "4GiB" CHAIN_LABELS := { "IGHeavy": "Heavy", @@ -225,26 +228,19 @@ self.body(func(args) { } } - // The scorer's input table, assembled in a single pass: pt joins the bundle columns on the - // key axis, renames them to the headers the scorer expects, and appends a constant column - // for any unit whose locus is not per-row. Doing this with parquetFileBuilder and then a - // second pt pass wrote the whole table twice for the sake of one constant. - ptWf := pt.workflow().mem("4GiB").cpu(1) - - colKeys := [] - selects := [pt.col(pSpec.getAxisId(keySpec)).alias("clonotypeKey")] - chainExprs := [] + tableBuilder := pframes.parquetFileBuilder() + tableBuilder.setAxisHeader(keySpec, "clonotypeKey") unitParams := [] + chainExprs := [] for unit in units { - colKeys += [unit.cdr3Col.key] - selects += [pt.col(unit.cdr3Col.key).alias("cdr3_" + unit.suffix)] + tableBuilder.add(unit.cdr3Col, { header: "cdr3_" + unit.suffix }) if is_undefined(unit.chainCol) { - // No column to join for this unit; its locus is one value for every row. Appending - // it here keeps main.py a pure per-row scorer and the software package unchanged. + // A unit with a constant locus has no column to add; queue it as a literal for the + // pass below instead of teaching the scorer about constants, which keeps main.py a + // pure per-row scorer and the software package unchanged. chainExprs += [pt.lit(unit.chainLiteral).alias("chain_" + unit.suffix)] } else { - colKeys += [unit.chainCol.key] - selects += [pt.col(unit.chainCol.key).alias("chain_" + unit.suffix)] + tableBuilder.add(unit.chainCol, { header: "chain_" + unit.suffix }) } unitParams += [{ suffix: unit.suffix, @@ -252,13 +248,19 @@ self.body(func(args) { label: unit.label }] } + tableBuilder.cpu(TABLE_CPU) + tableBuilder.mem(TABLE_MEM) + inputTable := tableBuilder.build() - df := ptWf.frameFromColumnBundle(columns, { columns: colKeys }).select(selects...) + // Only when a literal is actually needed, so bulk and single-cell inputs keep exactly the + // graph they had: no extra step, no extra parquet pass. if len(chainExprs) > 0 { - df = df.withColumns(chainExprs...) + ptWf := pt.workflow().mem(TABLE_MEM).cpu(TABLE_CPU) + ptWf.frame(inputTable, { xsvType: "parquet" }). + withColumns(chainExprs...). + save("with-chain.parquet") + inputTable = ptWf.run().getFile("with-chain.parquet") } - df.save("input.parquet") - inputTable := ptWf.run().getFile("input.parquet") // Specs in a keyed map so ordering can't perturb the params hash; data passed separately. renderInputs := { inputTable: inputTable } From f994049fcf34563aebf95c6827ed868a7b2f60bb Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 17:47:48 +0200 Subject: [PATCH 11/17] MILAB-6720: assemble the scorer's input table in one pt pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second attempt. The first (d9f22a2, reverted in f963a97) used frameFromColumnBundle, which json.decodes each pool key — and an addMulti result is keyed "." (pframes/bundle.lib.tengo:399), which is not JSON. The decode returned an error, indexing it raised "invalid index on error" at pt:770, and the block failed at graph construction. This names the columns itself with the layer underneath: pt.p.column(name, {spec, data}) per bundle entry, joined with pt.p.full, the key axis aliased through pt.sc.axis(keySpec). The json.decode step is never reached, because no pool key is passed to pt at all. parquetFileBuilder and the second pt pass are gone, so the table is written once rather than written, read and rewritten to append one constant. The literals still go on with withColumns, and still only when a unit has no chain column. Note this puts every input shape through pt, where the previous version left bulk and single-cell on parquetFileBuilder and used pt only for imported sets. That is inherent to one pass, and it is why the two-pass version was worth running first: the adaptation is now known good against real data, so anything that breaks here is the assembly. Also merges the two changesets into one, and fixes a real error in the one being merged: the label fix named '@platforma-open/milaboratories.generation-probability', a package that does not exist. The facade is '.block'. Left alone it would have bumped nothing. --- .changeset/fix-sc-tcr-chain-labels.md | 23 ------------- .changeset/variantkey-imported-sets.md | 36 ++++++++++++-------- workflow/src/score.tpl.tengo | 47 +++++++++++++++----------- 3 files changed, 50 insertions(+), 56 deletions(-) delete mode 100644 .changeset/fix-sc-tcr-chain-labels.md diff --git a/.changeset/fix-sc-tcr-chain-labels.md b/.changeset/fix-sc-tcr-chain-labels.md deleted file mode 100644 index ede31ed..0000000 --- a/.changeset/fix-sc-tcr-chain-labels.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -'@platforma-open/milaboratories.generation-probability.workflow': patch -'@platforma-open/milaboratories.generation-probability': patch ---- - -Fix swapped chain labels on single-cell TCR datasets - -The A/B slot was mapped to a chain name twice, and the two disagreed. MiXCR fixes "A" as the -more diverse chain — the one that recombines a D segment — so a receptor's chain order is -TCRBeta/TCRAlpha and TCRDelta/TCRGamma, not alphabetical. The label table read A as Alpha and -B as Beta, so on a single-cell TCR alpha/beta dataset the alpha column was labelled -"Generation probability (Beta)" and the beta column "(Alpha)". Gamma/delta was swapped the -same way. - -The Pgen values themselves were always right: the locus came from the per-record -`pl7.app/vdj/chain` column, never from the A/B letter. Only the label was wrong — but a -mislabelled column is read as the wrong chain's result. - -Labels are now derived from the chain that the A/B slot resolves to, through the single table -that also decides the locus, so a label cannot disagree with the model that produced the value -beside it. The duplicate label table is gone, which is what allowed the two to drift apart. - -Single-cell IG datasets, bulk and imported sets are unaffected. diff --git a/.changeset/variantkey-imported-sets.md b/.changeset/variantkey-imported-sets.md index 7ab997d..4a3aec0 100644 --- a/.changeset/variantkey-imported-sets.md +++ b/.changeset/variantkey-imported-sets.md @@ -4,10 +4,10 @@ '@platforma-open/milaboratories.generation-probability.block': minor --- -Score imported receptor sets keyed on pl7.app/variantKey +Score imported receptor sets, and fix swapped chain labels on single-cell TCR data -An imported set from Import VDJ Data was offered nowhere and scored nothing. Four separate -things stopped it, each failing differently: +**Imported receptor sets.** A set from Import VDJ Data was offered nowhere and scored nothing. +Four separate things stopped it, each failing differently. The dataset dropdown filtered on the presence of a per-record `pl7.app/vdj/chain` column, which an imported set does not have — so it never appeared as an option at all. Its locus is a property @@ -15,13 +15,13 @@ of the whole set, recorded on the key axis, and that is now accepted as an alter The CDR3 alphabet was read from `pl7.app/alphabet` on the key axis or from a `/structure` domain key. An imported set carries neither — the structure key belongs to the `scClonotypeKey` -vocabulary its axis does not use — so the run died on "Cannot determine CDR3 alphabet". Such a -set is amino acid throughout, which is now the answer. +vocabulary its axis does not use — so the run died on "Cannot determine CDR3 alphabet". Such a set +is amino acid throughout, which is now the answer. -Scoring needed a per-row locus column and quietly produced an empty result without one. A unit -whose locus is constant now gets that column added by a `pt` step over the built table, so the -scoring script is unchanged and still simply reads a locus per row. The step runs only when a -constant is actually needed, so bulk and single-cell inputs keep the graph they had. +Scoring needed a per-row locus column and quietly produced an empty result without one. The +scorer's input table is now assembled in a single pass that appends a constant locus column for a +unit whose locus is the same for every row, so the scoring script is unchanged and still simply +reads a locus per row. Whether the two chains are scored separately followed the key axis being `pl7.app/vdj/scClonotypeKey`. A paired imported set carries both chains in one frame under the @@ -31,8 +31,16 @@ follows the column domain. **A light chain from an imported set is reported as skipped, not scored.** OLGA needs IGK and IGL as separate models and an imported set records only "IG Light", so the locus is genuinely unknown. -Rather than guess, the light unit is passed through as `IGLight`, gets no Pgen, and is named in -the block's existing skipped-chains message. Heavy chains (IGH) and TCR alpha/beta (TRA, TRB) are -scored normally. - -Bulk, single-cell, peptide and amplicon inputs are unaffected. +Rather than guess, the light unit keeps its chain name, gets no Pgen, and is named in the block's +existing skipped-chains banner. Heavy chains (IGH) and TCR alpha/beta (TRA, TRB) score normally. + +**Swapped chain labels.** MiXCR fixes "A" as the more diverse chain — the one that recombines a D +segment — so a receptor's chain order is TCRBeta/TCRAlpha and TCRDelta/TCRGamma, not alphabetical. +The label table read A as Alpha and B as Beta, so on a single-cell TCR alpha/beta dataset the alpha +column was labelled "Generation probability (Beta)" and the beta column "(Alpha)". Gamma/delta was +swapped the same way. The Pgen values were always right — the locus came from the per-record chain +column, never from the A/B letter — but a mislabelled column reads as the wrong chain's result. +Labels are now derived from the chain that the A/B slot resolves to, through the single table that +also decides the locus, so a label cannot disagree with the model that produced the value beside it. + +Bulk, single-cell IG, peptide and amplicon inputs are unaffected. diff --git a/workflow/src/score.tpl.tengo b/workflow/src/score.tpl.tengo index 4a06bf1..eb47be5 100644 --- a/workflow/src/score.tpl.tengo +++ b/workflow/src/score.tpl.tengo @@ -228,39 +228,48 @@ self.body(func(args) { } } - tableBuilder := pframes.parquetFileBuilder() - tableBuilder.setAxisHeader(keySpec, "clonotypeKey") - unitParams := [] + // The scorer's input table, assembled in one pass. pt joins the bundle columns on the key + // axis under the header names the scorer expects, and appends a constant column for a unit + // whose locus is one value for every row. Assembling with parquetFileBuilder and then + // running pt to add that constant wrote the whole table twice. + // + // The columns are named here rather than via frameFromColumnBundle: that helper decodes + // each pool key as JSON, and an addMulti result is keyed "." + // (pframes/bundle.lib.tengo:399), which is not JSON — it fails inside the helper. + ptWf := pt.workflow().mem("4GiB").cpu(1) + + entries := [] + selects := [pt.sc.axis(keySpec).alias("clonotypeKey")] chainExprs := [] + unitParams := [] for unit in units { - tableBuilder.add(unit.cdr3Col, { header: "cdr3_" + unit.suffix }) + cdr3Name := "cdr3_" + unit.suffix + entries += [pt.p.column(cdr3Name, { spec: unit.cdr3Col.spec, data: unit.cdr3Col.data })] + selects += [pt.col(cdr3Name)] + + chainName := "chain_" + unit.suffix if is_undefined(unit.chainCol) { - // A unit with a constant locus has no column to add; queue it as a literal for the - // pass below instead of teaching the scorer about constants, which keeps main.py a - // pure per-row scorer and the software package unchanged. - chainExprs += [pt.lit(unit.chainLiteral).alias("chain_" + unit.suffix)] + // No column to join: this unit's locus is a property of the whole set. Appending it + // here keeps main.py a pure per-row scorer and the software package unchanged. + chainExprs += [pt.lit(unit.chainLiteral).alias(chainName)] } else { - tableBuilder.add(unit.chainCol, { header: "chain_" + unit.suffix }) + entries += [pt.p.column(chainName, { spec: unit.chainCol.spec, data: unit.chainCol.data })] + selects += [pt.col(chainName)] } + unitParams += [{ suffix: unit.suffix, domain: unit.domain, label: unit.label }] } - tableBuilder.cpu(TABLE_CPU) - tableBuilder.mem(TABLE_MEM) - inputTable := tableBuilder.build() - // Only when a literal is actually needed, so bulk and single-cell inputs keep exactly the - // graph they had: no extra step, no extra parquet pass. + df := ptWf.frame(pt.p.full(entries...)).select(selects...) if len(chainExprs) > 0 { - ptWf := pt.workflow().mem(TABLE_MEM).cpu(TABLE_CPU) - ptWf.frame(inputTable, { xsvType: "parquet" }). - withColumns(chainExprs...). - save("with-chain.parquet") - inputTable = ptWf.run().getFile("with-chain.parquet") + df = df.withColumns(chainExprs...) } + df.save("input.parquet") + inputTable := ptWf.run().getFile("input.parquet") // Specs in a keyed map so ordering can't perturb the params hash; data passed separately. renderInputs := { inputTable: inputTable } From a392c5c076a34237e0a37d447701a4801e5f8750 Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 17:50:18 +0200 Subject: [PATCH 12/17] MILAB-6720: drop the now-dead TABLE_CPU / TABLE_MEM constants They existed to share one budget between parquetFileBuilder and the pt pass. The single-pass assembly has one call site, so they were left declared and unreferenced by the splice that replaced their only users. The budget is inline on the pt workflow now; a lone call needs no constant. --- workflow/src/score.tpl.tengo | 5 ----- 1 file changed, 5 deletions(-) diff --git a/workflow/src/score.tpl.tengo b/workflow/src/score.tpl.tengo index eb47be5..309c0ed 100644 --- a/workflow/src/score.tpl.tengo +++ b/workflow/src/score.tpl.tengo @@ -8,11 +8,6 @@ pt := import("@platforma-sdk/workflow-tengo:pt") pgenTpl := assets.importTemplate(":pgen-process") -// Both passes over the input table are column shuffles, not computation — assembling it -// and appending a constant cost the same, so they share one budget. -TABLE_CPU := 1 -TABLE_MEM := "4GiB" - CHAIN_LABELS := { "IGHeavy": "Heavy", "IGLight": "Light", From 4018b1fd77f08d1001e7bcc5c6e7fc95711de956 Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 18:00:40 +0200 Subject: [PATCH 13/17] MILAB-6720: read the CDR3 alphabet from the columns, not from an assumption The imported-set branch hardcoded "aminoacid" on the grounds that such a set only carries amino-acid sequences. True today, and an assumption where a fact was available: the key axis does not state the alphabet but the sequence columns do, in pl7.app/alphabet. main.tpl keeps reading the axis and its structure key, and now passes "" rather than panicking when neither says anything. score.tpl finishes the job, because that is where the columns are in hand, taking the alphabet off the CDR3 columns and refusing on either no column carrying one or columns that disagree. The axis stays authoritative wherever it speaks, which is the invariant that matters: a MiXCR set emits both nt and aa CDR3 columns, and the axis is what says which defines the clonotype. Deriving from the columns there would be ambiguous, so it happens only on the silent path. Also adds the ll import score.tpl was missing before it could panic. --- .changeset/variantkey-imported-sets.md | 9 ++++++--- workflow/src/main.tpl.tengo | 14 +++++-------- workflow/src/score.tpl.tengo | 27 +++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/.changeset/variantkey-imported-sets.md b/.changeset/variantkey-imported-sets.md index 4a3aec0..11a197b 100644 --- a/.changeset/variantkey-imported-sets.md +++ b/.changeset/variantkey-imported-sets.md @@ -14,9 +14,12 @@ an imported set does not have — so it never appeared as an option at all. Its of the whole set, recorded on the key axis, and that is now accepted as an alternative. The CDR3 alphabet was read from `pl7.app/alphabet` on the key axis or from a `/structure` -domain key. An imported set carries neither — the structure key belongs to the `scClonotypeKey` -vocabulary its axis does not use — so the run died on "Cannot determine CDR3 alphabet". Such a set -is amino acid throughout, which is now the answer. +domain key. An imported set states it in neither — the structure key belongs to the +`scClonotypeKey` vocabulary its axis does not use — so the run died on "Cannot determine CDR3 +alphabet". Its sequence columns do carry the alphabet, so that is where it is now read from when +the axis says nothing. The axis stays authoritative when it does say something: a MiXCR set emits +both nucleotide and amino-acid CDR3 columns, and the axis is what says which of them defines the +clonotype. Scoring needed a per-row locus column and quietly produced an empty result without one. The scorer's input table is now assembled in a single pass that appends a constant locus column for a diff --git a/workflow/src/main.tpl.tengo b/workflow/src/main.tpl.tengo index c581c9b..ea6dbc8 100644 --- a/workflow/src/main.tpl.tengo +++ b/workflow/src/main.tpl.tengo @@ -62,16 +62,12 @@ wf.body(func(args) { if is_undefined(alphabet) { alphabet = alphabetFromStructure(keyDomain[keySpec.name + "/structure"]) } - // An imported receptor set carries neither source: no pl7.app/alphabet on the key axis, and - // no /structure -- that key belongs to the scClonotypeKey vocabulary this axis does not - // use, and no variantKey producer emits one. Its sequences are amino acid throughout, so - // that is the answer rather than a failure. - if is_undefined(alphabet) && keySpec.name == "pl7.app/variantKey" && - !is_undefined(keyDomain["pl7.app/vdj/clonotypingRunId"]) { - alphabet = "aminoacid" - } + // An imported receptor set states it in neither place: no pl7.app/alphabet on the key axis, + // and no /structure -- that key belongs to the scClonotypeKey vocabulary this axis does + // not use. Its sequence columns do carry it, so resolution is finished in the score template + // where the columns are in hand; "" means "the axis did not say". if is_undefined(alphabet) { - ll.panic("Cannot determine CDR3 alphabet") + alphabet = "" } // Forward only trace + label, so an unrelated datasetSpec change doesn't bust the diff --git a/workflow/src/score.tpl.tengo b/workflow/src/score.tpl.tengo index 309c0ed..2629439 100644 --- a/workflow/src/score.tpl.tengo +++ b/workflow/src/score.tpl.tengo @@ -5,6 +5,7 @@ assets := import("@platforma-sdk/workflow-tengo:assets") pframes := import("@platforma-sdk/workflow-tengo:pframes") pBundle := import("@platforma-sdk/workflow-tengo:pframes.bundle") pt := import("@platforma-sdk/workflow-tengo:pt") +ll := import("@platforma-sdk/workflow-tengo:ll") pgenTpl := assets.importTemplate(":pgen-process") @@ -150,6 +151,30 @@ self.body(func(args) { cdr3Cols := columns.getColumns("cdr3") + columns.getColumns("cdr3Plain") chainCols := columns.getColumns("chain") + // The key axis states the alphabet for a clonotype key, directly or inside its structure, and + // that is authoritative: a MiXCR set emits both nt and aa CDR3 columns and the axis is what + // says which of them defines the clonotype. Only when the axis said nothing -- an imported + // set -- read it off the columns, which do carry pl7.app/alphabet. + if alphabet == "" { + for col in cdr3Cols { + if is_undefined(col.spec.domain) { + continue + } + colAlphabet := col.spec.domain["pl7.app/alphabet"] + if is_undefined(colAlphabet) { + continue + } + if alphabet == "" { + alphabet = colAlphabet + } else if alphabet != colAlphabet { + ll.panic("Cannot determine CDR3 alphabet: the key axis states none and the CDR3 columns disagree (%v vs %v)", alphabet, colAlphabet) + } + } + if alphabet == "" { + ll.panic("Cannot determine CDR3 alphabet: the key axis states none and no CDR3 column carries pl7.app/alphabet") + } + } + // A paired imported set carries both chains in one frame under the scClonotypeChain column // domain, on a variantKey axis — so the axis name alone would call it bulk, and the bulk slot // takes the first CDR3 column it finds and drops the other chain. @@ -175,7 +200,7 @@ self.body(func(args) { } // Preferred: the per-row locus column. An imported set has none, so fall back to the - // unit's single mapped chain and materialise it as a constant column below. Without a + // unit's single mapped chain and materialize it as a constant column below. Without a // locus from either source there is nothing to score against. chainCol := firstWithDomain(chainCols, slot.chainDomain) chainLiteral := undefined From 28fcff43577f7ee1aa001451bbcab32f0173a121 Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 18:07:28 +0200 Subject: [PATCH 14/17] MILAB-6720: resolve the CDR3 alphabet in one place, dropping the "" sentinel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The alphabet was resolved in main.tpl from the key axis and passed to score.tpl, so once the columns became a third source the value had to cross a template boundary half-resolved — and "" stood for "the axis did not say", a sentinel that meant nothing to either side alone. score.tpl already receives keySpec, so it can do the whole thing: axis domain, then the structure key, then the columns, then refuse. main.tpl no longer mentions the alphabet at all and alphabetFromStructure moves with the logic that uses it. is_undefined carries the "not yet known" state the way tengo already does it, so there is no sentinel value. Behaviour is unchanged for every input: the axis still wins wherever it speaks, which is what keeps a MiXCR set scoring the CDR3 its clonotype key was built from rather than whichever alphabet a column happened to carry. The refusal is now one message naming both sources instead of two naming one each, since at that point neither has produced anything. --- workflow/src/main.tpl.tengo | 33 --------------------------- workflow/src/score.tpl.tengo | 44 ++++++++++++++++++++++++++++-------- 2 files changed, 34 insertions(+), 43 deletions(-) diff --git a/workflow/src/main.tpl.tengo b/workflow/src/main.tpl.tengo index ea6dbc8..22653f2 100644 --- a/workflow/src/main.tpl.tengo +++ b/workflow/src/main.tpl.tengo @@ -12,25 +12,6 @@ ENTITY_KEY_NAMES := { "pl7.app/variantKey": true } -// The alphabet may live directly on the key axis domain, or nested in the structured -// key's `/structure` domain under the CDR3 sequence entry. -alphabetFromStructure := func(structureStr) { - if is_undefined(structureStr) { - return undefined - } - for entry in json.decode(structureStr) { - if len(entry) == 0 || entry[0] != "pl7.app/vdj/sequence" { - continue - } - for idx, item in entry { - if idx > 0 && item[0] == "pl7.app/alphabet" { - return item[1] - } - } - } - return undefined -} - // Resolve only the anchor spec here; CDR3/chain columns are resolved in the child (score), // which anchors them by the entity-key axis name — unknown until the anchor spec resolves. wf.prepare(func(args) { @@ -57,19 +38,6 @@ wf.body(func(args) { ll.panic("Selected dataset has no clonotype or variant key axis") } - keyDomain := is_undefined(keySpec.domain) ? {} : keySpec.domain - alphabet := keyDomain["pl7.app/alphabet"] - if is_undefined(alphabet) { - alphabet = alphabetFromStructure(keyDomain[keySpec.name + "/structure"]) - } - // An imported receptor set states it in neither place: no pl7.app/alphabet on the key axis, - // and no /structure -- that key belongs to the scClonotypeKey vocabulary this axis does - // not use. Its sequence columns do carry it, so resolution is finished in the score template - // where the columns are in hand; "" means "the axis did not say". - if is_undefined(alphabet) { - alphabet = "" - } - // Forward only trace + label, so an unrelated datasetSpec change doesn't bust the // downstream cache. datasetLabel := "" @@ -88,7 +56,6 @@ wf.body(func(args) { anchorRef: anchorRef, keyName: keySpec.name, keySpec: keySpec, - alphabet: alphabet, species: args.species, datasetLabel: datasetLabel, traceAnnotations: traceAnnotations diff --git a/workflow/src/score.tpl.tengo b/workflow/src/score.tpl.tengo index 2629439..001d0e5 100644 --- a/workflow/src/score.tpl.tengo +++ b/workflow/src/score.tpl.tengo @@ -6,9 +6,29 @@ pframes := import("@platforma-sdk/workflow-tengo:pframes") pBundle := import("@platforma-sdk/workflow-tengo:pframes.bundle") pt := import("@platforma-sdk/workflow-tengo:pt") ll := import("@platforma-sdk/workflow-tengo:ll") +json := import("json") pgenTpl := assets.importTemplate(":pgen-process") +// The alphabet may sit directly on the key axis domain, or nested in the structured key's +// `/structure` domain under the CDR3 sequence entry. +alphabetFromStructure := func(structureStr) { + if is_undefined(structureStr) { + return undefined + } + for entry in json.decode(structureStr) { + if len(entry) == 0 || entry[0] != "pl7.app/vdj/sequence" { + continue + } + for idx, item in entry { + if idx > 0 && item[0] == "pl7.app/alphabet" { + return item[1] + } + } + } + return undefined +} + CHAIN_LABELS := { "IGHeavy": "Heavy", "IGLight": "Light", @@ -143,7 +163,6 @@ self.prepare(func(args) { self.body(func(args) { columns := args.columns - alphabet := args.alphabet keySpec := args.keySpec // CDR3 lives under one of two namespaces (pl7.app/vdj/sequence for clonotype keys, @@ -151,11 +170,16 @@ self.body(func(args) { cdr3Cols := columns.getColumns("cdr3") + columns.getColumns("cdr3Plain") chainCols := columns.getColumns("chain") - // The key axis states the alphabet for a clonotype key, directly or inside its structure, and - // that is authoritative: a MiXCR set emits both nt and aa CDR3 columns and the axis is what - // says which of them defines the clonotype. Only when the axis said nothing -- an imported - // set -- read it off the columns, which do carry pl7.app/alphabet. - if alphabet == "" { + // Which alphabet the CDR3 is scored in. The key axis is authoritative wherever it speaks, + // directly or inside its structure key: a MiXCR set emits both nt and aa CDR3 columns, and + // the axis is what says which of them defines the clonotype. An imported set's axis states + // neither, so fall back to the columns, which do carry pl7.app/alphabet. + keyDomain := is_undefined(keySpec.domain) ? {} : keySpec.domain + alphabet := keyDomain["pl7.app/alphabet"] + if is_undefined(alphabet) { + alphabet = alphabetFromStructure(keyDomain[keySpec.name + "/structure"]) + } + if is_undefined(alphabet) { for col in cdr3Cols { if is_undefined(col.spec.domain) { continue @@ -164,15 +188,15 @@ self.body(func(args) { if is_undefined(colAlphabet) { continue } - if alphabet == "" { + if is_undefined(alphabet) { alphabet = colAlphabet } else if alphabet != colAlphabet { ll.panic("Cannot determine CDR3 alphabet: the key axis states none and the CDR3 columns disagree (%v vs %v)", alphabet, colAlphabet) } } - if alphabet == "" { - ll.panic("Cannot determine CDR3 alphabet: the key axis states none and no CDR3 column carries pl7.app/alphabet") - } + } + if is_undefined(alphabet) { + ll.panic("Cannot determine CDR3 alphabet: neither the key axis nor any CDR3 column states one") } // A paired imported set carries both chains in one frame under the scClonotypeChain column From d7970a4fbf090557ed3599df6b45e9b1f848da89 Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 18:22:07 +0200 Subject: [PATCH 15/17] MILAB-6720: find the key axis by name in the model, not by position MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scorability check read axesSpec[1]. Which index the key axis sits at is a property of the producer, not something a consumer can assume — main.tpl.tengo searches its ENTITY_KEY_NAMES for it rather than indexing, and the model now does the same through keyAxisOf. The selector list is derived from that same array, so the three axis names are written once. They were previously spelled out in the selectors and again in the check, which is how the two would have drifted the first time a fourth key axis appeared. --- model/src/index.ts | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/model/src/index.ts b/model/src/index.ts index d8e105c..a901588 100644 --- a/model/src/index.ts +++ b/model/src/index.ts @@ -32,20 +32,23 @@ export type BlockData = { distributionGraphState: GraphMakerState; }; -const inputSelectors = [ - { - axes: [{ name: "pl7.app/vdj/clonotypeKey" }], - annotations: { "pl7.app/isAnchor": "true" }, - }, - { - axes: [{ name: "pl7.app/vdj/scClonotypeKey" }], - annotations: { "pl7.app/isAnchor": "true" }, - }, - { - axes: [{ name: "pl7.app/variantKey" }], - annotations: { "pl7.app/isAnchor": "true" }, - }, -]; +// The axes a dataset can be keyed on, mirroring ENTITY_KEY_NAMES in main.tpl.tengo. Both the +// selectors and the scorability check below are derived from this one list so they cannot drift. +const ENTITY_KEY_NAMES = [ + "pl7.app/vdj/clonotypeKey", + "pl7.app/vdj/scClonotypeKey", + "pl7.app/variantKey", +] as const; + +const inputSelectors = ENTITY_KEY_NAMES.map((name) => ({ + axes: [{ name }], + annotations: { "pl7.app/isAnchor": "true" }, +})); + +// The key axis is found by name, never by position: which index it sits at is a property of the +// producer, and main.tpl.tengo searches for it the same way rather than assuming one. +const keyAxisOf = (spec: { axesSpec: { name: string; domain?: Record }[] }) => + spec.axesSpec.find((axis) => (ENTITY_KEY_NAMES as readonly string[]).includes(axis.name)); const dataModel = new DataModelBuilder().from("v1").init(() => ({ datasetLabel: "", @@ -83,7 +86,7 @@ export const platforma = BlockModelV3.create(dataModel) collection .getColumns() .filter((anchor) => { - const keyAxis = anchor.getSpec().axesSpec[1]; + const keyAxis = keyAxisOf(anchor.getSpec()); const keyDomain = keyAxis?.domain ?? {}; if ( keyAxis?.name === "pl7.app/variantKey" && From b821558a4fd4908c875ac06e8929bd2fe3c08751 Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 18:28:45 +0200 Subject: [PATCH 16/17] MILAB-6720: accept the lint autofix on ENTITY_KEY_NAMES oxlint dropped 'as const' and the readonly cast it forced. Without the literal tuple the array is string[], which is all .includes() and the selector map need, so the cast was work the type was creating for itself. Applied by the build's check step, not by hand. --- model/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/model/src/index.ts b/model/src/index.ts index a901588..968f9c5 100644 --- a/model/src/index.ts +++ b/model/src/index.ts @@ -38,7 +38,7 @@ const ENTITY_KEY_NAMES = [ "pl7.app/vdj/clonotypeKey", "pl7.app/vdj/scClonotypeKey", "pl7.app/variantKey", -] as const; +]; const inputSelectors = ENTITY_KEY_NAMES.map((name) => ({ axes: [{ name }], @@ -48,7 +48,7 @@ const inputSelectors = ENTITY_KEY_NAMES.map((name) => ({ // The key axis is found by name, never by position: which index it sits at is a property of the // producer, and main.tpl.tengo searches for it the same way rather than assuming one. const keyAxisOf = (spec: { axesSpec: { name: string; domain?: Record }[] }) => - spec.axesSpec.find((axis) => (ENTITY_KEY_NAMES as readonly string[]).includes(axis.name)); + spec.axesSpec.find((axis) => ENTITY_KEY_NAMES.includes(axis.name)); const dataModel = new DataModelBuilder().from("v1").init(() => ({ datasetLabel: "", From 0220b260a68d42dac8d52f86162b2fc17ce3f293 Mon Sep 17 00:00:00 2001 From: Elena Erokhina Date: Thu, 27 Aug 2026 18:36:15 +0200 Subject: [PATCH 17/17] remove extra check --- workflow/src/score.tpl.tengo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/src/score.tpl.tengo b/workflow/src/score.tpl.tengo index 001d0e5..5ef2404 100644 --- a/workflow/src/score.tpl.tengo +++ b/workflow/src/score.tpl.tengo @@ -203,7 +203,7 @@ self.body(func(args) { // domain, on a variantKey axis — so the axis name alone would call it bulk, and the bulk slot // takes the first CDR3 column it finds and drops the other chain. isSingleCell := keySpec.name == "pl7.app/vdj/scClonotypeKey" - if !isSingleCell && keySpec.name == "pl7.app/variantKey" { + if keySpec.name == "pl7.app/variantKey" { for col in cdr3Cols { if !is_undefined(col.spec.domain) && !is_undefined(col.spec.domain["pl7.app/vdj/scClonotypeChain/index"]) {