From 5b749bc978bcc1543e1920df16702be5c5a93cca Mon Sep 17 00:00:00 2001 From: Stanislav Poslavsky Date: Tue, 28 Jul 2026 01:13:06 +0200 Subject: [PATCH] Read sample identity from the folder, not just the file name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patterns were matched against the bare file name, so a one-folder-per-sample tree — what BaseSpace, bcl2fastq and CellRanger all produce — could not be described at all. Where the folder was the only place a sample name appeared, every file resolved to the same sample and getOrCreateSample collapsed them, keeping only the last. - Match against each file's path relative to the longest common directory of the selection. Recomputed over the whole set on every render rather than threaded through the file dialog, so it stays well defined when the selection spans folders or grows over several "add more files" rounds. With every file in one folder the remainder is the bare name, so flat imports are byte-identical to before. - Bound {{Sample}}, {{*}} and tag matchers to a single path segment; add {{**}} for crossing them. Required, not cosmetic: a lazy `.+?` swallows the separator and pulls the folder into the sample name. - Infer over file-name, folder-ignored and folder-carries-identity forms, taking the first that gives every file its own identity. Per-sample-folder FASTQ and CellRanger MTX now infer. - Recognise _S_L__001, so the Illumina sample number stops being absorbed into the sample name. Three inference tests changed expectation; all three inputs are real 10x names where the new answer is the right one. - Report files that collapse onto one identity and hold the import, rather than overwriting silently. --- .changeset/path-aware-file-patterns.md | 13 ++ ui/src/dialogs/ImportDatasetDialog.vue | 34 ++++- ui/src/dialogs/datasets.test.ts | 87 +++++++++++++ ui/src/dialogs/datasets.ts | 86 +++++++++++- ui/src/dialogs/file_name_parser.test.ts | 99 +++++++++++++- ui/src/dialogs/file_name_parser.ts | 166 ++++++++++++++++++------ 6 files changed, 432 insertions(+), 53 deletions(-) create mode 100644 .changeset/path-aware-file-patterns.md create mode 100644 ui/src/dialogs/datasets.test.ts diff --git a/.changeset/path-aware-file-patterns.md b/.changeset/path-aware-file-patterns.md new file mode 100644 index 0000000..914cbb6 --- /dev/null +++ b/.changeset/path-aware-file-patterns.md @@ -0,0 +1,13 @@ +--- +"@platforma-open/milaboratories.samples-and-data.ui": minor +--- + +Read sample identity from the folder, not just the file name. + +Patterns were matched against the bare file name, so a one-folder-per-sample tree — what BaseSpace, bcl2fastq and CellRanger all produce — could not be described. Where the folder was the only place a sample name appeared, every file resolved to the same sample and all but the last were dropped without a word. + +- Patterns are now matched against each file's path relative to the longest common directory of the selection. With every file in one folder that is the bare file name, so flat imports are unchanged. +- `{{Sample}}`, `{{*}}` and tag matchers are bounded to one path segment; the new `{{**}}` crosses segments. `{{**}}/{{Sample}}_S{{n}}_L{{n}}_{{RR}}_{{n}}.fastq.gz` and `{{Sample}}/{{R}}.fastq.gz` both work. +- Inference tries the file name, folder-ignored and folder-carries-identity forms, and takes the first that gives every file its own identity. Per-sample-folder FASTQ and per-sample-folder CellRanger MTX (`Sample_A/matrix.mtx.gz`) now infer. +- The canonical Illumina naming `_S_L__001` is recognised, so `A_S7_L001_R1_001.fastq.gz` gives sample `A` rather than `A_S7`. This changes inferred sample names for bcl2fastq/BaseSpace output — `_S` is the sample number, not part of the name. +- A pattern that maps two files onto one identity is reported in the import dialog and blocks the import, instead of silently overwriting. diff --git a/ui/src/dialogs/ImportDatasetDialog.vue b/ui/src/dialogs/ImportDatasetDialog.vue index 93f8527..1008df7 100644 --- a/ui/src/dialogs/ImportDatasetDialog.vue +++ b/ui/src/dialogs/ImportDatasetDialog.vue @@ -19,9 +19,10 @@ import type { ReadIndices, } from "@platforma-open/milaboratories.samples-and-data.model"; import type { ImportFileHandle } from "@platforma-sdk/model"; -import { getFileNameFromHandle, getFilePathFromHandle, uniquePlId } from "@platforma-sdk/model"; +import { uniquePlId } from "@platforma-sdk/model"; import type { ListOption } from "@platforma-sdk/ui-vue"; import { + PlAlert, PlBtnGhost, PlBtnGroup, PlBtnPrimary, @@ -40,9 +41,10 @@ import { useApp } from "../app"; import type { ImportMode } from "./datasets"; import { datasetTypes, - extractFileName, + findDuplicateKeys, getOrCreateSample, modesOptions, + relativeFilePaths, useParsedFiles, usePatternCompilation, } from "./datasets"; @@ -215,6 +217,27 @@ const parsedFiles = useParsedFiles(data, compiledPattern); // Whether any of the files matched the pattern const hasMatchedFiles = computed(() => parsedFiles.value.filter((f) => f.match).length > 0); +/** + * Files that the current pattern collapses onto one identity. The dataset + * content builders index by that identity, so importing would keep only the + * last file of each group — silently. Report instead, and hold the import. + */ +const duplicateKeys = computed(() => findDuplicateKeys(parsedFiles.value)); + +const duplicateKeysMessage = computed(() => { + const groups = duplicateKeys.value; + if (groups.length === 0) return undefined; + const shown = groups + .slice(0, 3) + .map((g) => `"${g.sample}" ← ${g.fileNames.join(", ")}`) + .join("; "); + const rest = groups.length > 3 ? ` (and ${groups.length - 3} more)` : ""; + return ( + `${groups.length} sample(s) would be overwritten because several files resolve ` + + `to the same one: ${shown}${rest}. Adjust the pattern so each file gets its own sample.` + ); +}); + const dsTypeOptions = computed(() => { if (!data.fileType) { return []; @@ -243,7 +266,9 @@ function updateDatasetType(datasetType: DSType | undefined) { // Add more files to the data function addFiles(files: ImportFileHandle[]) { - const fileNames = files.map((h) => extractFileName(getFilePathFromHandle(h))); + // Inference has to see the same strings matching will see later — paths + // relative to the folder the files came from, not bare file names. + const fileNames = relativeFilePaths(files); if (data.files.length === 0) { const inferredPattern = inferFileNamePattern(fileNames); if (inferredPattern) { @@ -929,6 +954,7 @@ watch(availableColumnsOptions, (options) => { const canCreateOrAdd = computed(() => { const basicConditions = hasMatchedFiles.value && + duplicateKeys.value.length === 0 && (data.mode === "create-new-dataset" || data.targetAddDataset !== undefined) && data.datasetType !== undefined && !data.loadingColumns && @@ -1003,6 +1029,8 @@ const canCreateOrAdd = computed(() => { + {{ duplicateKeysMessage }} + diff --git a/ui/src/dialogs/datasets.test.ts b/ui/src/dialogs/datasets.test.ts new file mode 100644 index 0000000..faa4995 --- /dev/null +++ b/ui/src/dialogs/datasets.test.ts @@ -0,0 +1,87 @@ +import { test } from "vitest"; +import type { ImportFileHandle } from "@platforma-sdk/model"; +import type { ParsedFile } from "./datasets"; +import { commonDirPrefix, findDuplicateKeys, toPosixPath } from "./datasets"; +import { FileNamePattern } from "./file_name_parser"; + +test.for([ + { + name: "files in one folder leave bare file names", + paths: ["/data/run/A_R1.fastq.gz", "/data/run/A_R2.fastq.gz"], + expected: "/data/run/", + }, + { + name: "a single file leaves its bare name", + paths: ["/data/run/A_R1.fastq.gz"], + expected: "/data/run/", + }, + { + name: "per-sample folders keep the folder", + paths: ["/data/run/A/R1.fastq.gz", "/data/run/B/R1.fastq.gz"], + expected: "/data/run/", + }, + { + name: "a file name is never mistaken for a shared directory", + paths: ["/data/run/A/R1.fastq.gz", "/data/run/A/R2.fastq.gz"], + expected: "/data/run/A/", + }, + { + name: "unrelated roots share nothing", + paths: ["/data/one/A_R1.fastq.gz", "runs/two/B_R1.fastq.gz"], + expected: "", + }, + { + name: "a partial segment match is not a shared directory", + paths: ["/data/run1/A_R1.fastq.gz", "/data/run2/A_R1.fastq.gz"], + expected: "/data/", + }, + { + name: "no paths", + paths: [], + expected: "", + }, +])("commonDirPrefix: $name", ({ paths, expected }, { expect }) => { + expect(commonDirPrefix(paths)).to.equal(expected); +}); + +test("toPosixPath normalizes Windows separators", ({ expect }) => { + expect(toPosixPath("C:\\data\\run\\A_R1.fastq.gz")).to.equal("C:/data/run/A_R1.fastq.gz"); +}); + +/** ParsedFile carrying only what findDuplicateKeys reads. */ +function parsed(pattern: FileNamePattern, fileName: string): ParsedFile { + return { + handle: `upload://upload/${fileName}` as ImportFileHandle, + fileName, + match: pattern.match(fileName), + }; +} + +test("findDuplicateKeys is empty when every file resolves to its own identity", ({ expect }) => { + const pattern = FileNamePattern.parse("{{Sample}}/{{R}}.fastq.gz"); + const files = ["A/R1.fastq.gz", "A/R2.fastq.gz", "B/R1.fastq.gz"].map((f) => parsed(pattern, f)); + expect(findDuplicateKeys(files)).to.toMatchObject([]); +}); + +test("findDuplicateKeys catches files that would overwrite each other", ({ expect }) => { + // The folder carries the identity, but the pattern reads only the file name — + // so both samples collapse onto "R1" and one file would be dropped silently. + const pattern = FileNamePattern.parse("{{**}}/{{Sample}}.fastq.gz"); + const files = ["A/R1.fastq.gz", "B/R1.fastq.gz"].map((f) => parsed(pattern, f)); + const duplicates = findDuplicateKeys(files); + expect(duplicates.length).to.equal(1); + expect(duplicates[0].sample).to.equal("R1"); + expect(duplicates[0].fileNames).to.toMatchObject(["A/R1.fastq.gz", "B/R1.fastq.gz"]); +}); + +test("findDuplicateKeys treats read index and lane as part of the identity", ({ expect }) => { + const pattern = FileNamePattern.parse("{{Sample}}_L{{L}}_{{RR}}.fastq.gz"); + const distinct = ["A_L001_R1.fastq.gz", "A_L001_R2.fastq.gz", "A_L002_R1.fastq.gz"].map((f) => + parsed(pattern, f), + ); + expect(findDuplicateKeys(distinct)).to.toMatchObject([]); + + // Unmatched files carry no identity and must not be reported as duplicates. + const unmatched = ["nope.txt", "also-nope.txt"].map((f) => parsed(pattern, f)); + expect(findDuplicateKeys(unmatched)).to.toMatchObject([]); +}); diff --git a/ui/src/dialogs/datasets.ts b/ui/src/dialogs/datasets.ts index af900ad..76a8728 100644 --- a/ui/src/dialogs/datasets.ts +++ b/ui/src/dialogs/datasets.ts @@ -1,6 +1,6 @@ import type { BlockData, DSType } from "@platforma-open/milaboratories.samples-and-data.model"; import type { ImportFileHandle, PlId } from "@platforma-sdk/model"; -import { getFileNameFromHandle, uniquePlId } from "@platforma-sdk/model"; +import { getFilePathFromHandle, uniquePlId } from "@platforma-sdk/model"; import type { AppV3, SimpleOption } from "@platforma-sdk/ui-vue"; import type { ComputedRef, Reactive, ShallowRef } from "vue"; import { computed, ref, shallowRef, watch } from "vue"; @@ -126,6 +126,44 @@ export function extractFileName(filePath: string) { return filePath.replace(/^.*[\\/]/, ""); } +/** Windows handles carry `\`; patterns and prefix arithmetic assume `/`. */ +export function toPosixPath(filePath: string) { + return filePath.replace(/\\/g, "/"); +} + +/** + * Longest directory prefix shared by every path, with a trailing `/`, or `""` + * when they share no directory at all. + * + * The last segment of a path is its file name and is never part of the prefix, + * so a single file — or a set of files all sitting in one folder — yields that + * folder and leaves bare file names behind. + */ +export function commonDirPrefix(paths: string[]): string { + if (paths.length === 0) return ""; + const segments = paths.map((p) => p.split("/")); + const maxDirs = Math.min(...segments.map((s) => s.length - 1)); + const first = segments[0]; + let shared = 0; + while (shared < maxDirs && segments.every((s) => s[shared] === first[shared])) shared++; + return shared === 0 ? "" : first.slice(0, shared).join("/") + "/"; +} + +/** + * Paths of the given files relative to the folder they were imported from. + * + * Patterns are matched against these rather than against bare file names, so + * `{{Sample}}` can be read out of a folder name in one-folder-per-sample + * layouts. Recomputing the prefix over the whole set on every call — instead of + * threading a root through the file dialog — keeps this well defined when the + * selection spans folders or grows over several "add more files" rounds. + */ +export function relativeFilePaths(handles: ImportFileHandle[]): string[] { + const paths = handles.map((h) => toPosixPath(getFilePathFromHandle(h))); + const prefix = commonDirPrefix(paths); + return paths.map((p) => p.slice(prefix.length)); +} + // Pattern compilation and file name matching export function usePatternCompilation(data: Reactive<{ pattern: string }>) { const patternError = ref(undefined); @@ -164,17 +202,53 @@ export function useParsedFiles( data: Reactive<{ files: ImportFileHandle[] }>, compiledPattern: ShallowRef, ): ComputedRef { - return computed(() => - data.files.map((handle) => { - const fileName = extractFileName(getFileNameFromHandle(handle)); + return computed(() => { + const paths = relativeFilePaths(data.files); + return data.files.map((handle, i) => { + const fileName = paths[i]; const match = compiledPattern.value?.match(fileName); return { handle, fileName, match, }; - }), - ); + }); + }); +} + +/** + * The identity a matched file resolves to — everything the dataset content + * builders key on when they place a handle. Two files sharing a key would + * overwrite one another, keeping only whichever was processed last. + */ +export function sampleKeyOf(match: FileNamePatternMatch): string { + const parts = [match.sample.value]; + if (match.lane) parts.push("lane=" + match.lane.value); + if (match.readIndex) parts.push("read=" + match.readIndex.value); + if (match.cellRangerFileRole) parts.push("role=" + match.cellRangerFileRole.value); + for (const tag of Object.keys(match.tags ?? {}).sort()) + parts.push(`${tag}=${match.tags![tag].value}`); + return parts.join("\u0000"); +} + +export type DuplicateKey = { sample: string; fileNames: string[] }; + +/** Groups of matched files that collapse onto one identity. Empty when fine. */ +export function findDuplicateKeys(files: ParsedFile[]): DuplicateKey[] { + const byKey = new Map(); + for (const f of files) { + if (!f.match) continue; + const key = sampleKeyOf(f.match); + const group = byKey.get(key); + if (group) group.push(f); + else byKey.set(key, [f]); + } + return [...byKey.values()] + .filter((group) => group.length > 1) + .map((group) => ({ + sample: group[0].match!.sample.value, + fileNames: group.map((f) => f.fileName), + })); } export function getOrCreateSample(appUt: unknown, sampleName: string): PlId { diff --git a/ui/src/dialogs/file_name_parser.test.ts b/ui/src/dialogs/file_name_parser.test.ts index 7f69801..3ad1332 100644 --- a/ui/src/dialogs/file_name_parser.test.ts +++ b/ui/src/dialogs/file_name_parser.test.ts @@ -169,9 +169,13 @@ test("infer file name pattern 1", ({ expect }) => { "10k_PBMC_5pv2_nextgem_Chromium_Controller_gex_2_S5_L004_R2_001.fastq.gz", ]; const result = inferFileNamePattern(fileNames); - expect(result?.pattern.rawPattern).to.equal("{{Sample}}_L{{L}}_{{RR}}_{{n}}.fastq.gz"); + expect(result?.pattern.rawPattern).to.equal("{{Sample}}_S{{n}}_L{{L}}_{{RR}}_{{n}}.fastq.gz"); expect(result?.extension).to.equal("fastq.gz"); expect(result?.readIndices).to.toMatchObject(["R1", "R2"]); + // The Illumina sample number (_S7) is read as such, not folded into the name. + expect(result?.pattern.match(fileNames[2])?.sample.value).to.equal( + "10k_PBMC_5pv2_nextgem_Chromium_Controller_gex_1", + ); }); test("infer file name pattern 2", ({ expect }) => { @@ -198,7 +202,7 @@ test("infer file name pattern 2", ({ expect }) => { "10k_PBMC_5pv2_nextgem_Chromium_Controller_gex_2_S5_L004_R2.fastq.gz", ]; const result = inferFileNamePattern(fileNames); - expect(result?.pattern.rawPattern).to.equal("{{Sample}}_L{{L}}_{{RR}}.fastq.gz"); + expect(result?.pattern.rawPattern).to.equal("{{Sample}}_S{{n}}_L{{L}}_{{RR}}.fastq.gz"); expect(result?.extension).to.equal("fastq.gz"); expect(result?.readIndices).to.toMatchObject(["R1", "R2"]); }); @@ -215,7 +219,7 @@ test("infer file name pattern 3", ({ expect }) => { "10k_PBMC_5pv2_nextgem_Chromium_Controller_gex_2_S5_L004_R2.fastq.gz", ]; const result = inferFileNamePattern(fileNames); - expect(result?.pattern.rawPattern).to.equal("{{Sample}}_L{{n}}_{{RR}}.fastq.gz"); + expect(result?.pattern.rawPattern).to.equal("{{Sample}}_S{{n}}_L{{n}}_{{RR}}.fastq.gz"); expect(result?.extension).to.equal("fastq.gz"); expect(result?.readIndices).to.toMatchObject(["R1", "R2"]); }); @@ -462,3 +466,92 @@ test("CellRanger MTX pattern properties", ({ expect }) => { expect(pattern.gzipped).to.equal(true); expect(pattern.fileContentType).to.equal("CellRangerMTX"); }); + +// --------------------------------------------------------------------------- +// Path-aware matching. Patterns are matched against a path relative to the +// import root, so one-folder-per-sample layouts can be described. +// --------------------------------------------------------------------------- + +test("matchers do not cross path separators", ({ expect }) => { + const pattern = FileNamePattern.parse("{{Sample}}_{{R}}.fastq.gz"); + // Would match with an unbounded `.+?`, pulling the folder into the sample. + expect(pattern.match("Sample_A/Sample_R1.fastq.gz")).to.equal(undefined); + expect(pattern.match("Sample_R1.fastq.gz")?.sample.value).to.equal("Sample"); +}); + +test("{{**}} crosses path separators", ({ expect }) => { + const pattern = FileNamePattern.parse("{{**}}/{{Sample}}_{{R}}.fastq.gz"); + expect(pattern.match("a/b/c/Sample_R1.fastq.gz")?.sample.value).to.equal("Sample"); + expect(pattern.match("Sample_R1.fastq.gz")).to.equal(undefined); +}); + +test("{{Sample}} can be taken from a folder name", ({ expect }) => { + const pattern = FileNamePattern.parse("{{Sample}}/{{R}}.fastq.gz"); + const match = pattern.match("Sample_A/R1.fastq.gz"); + expect(match?.sample.value).to.equal("Sample_A"); + expect(match?.readIndex?.value).to.equal("R1"); +}); + +test("infer BaseSpace one-folder-per-sample tree", ({ expect }) => { + // Relative paths of a real BaseSpace download (Synolo): a folder per sample, + // each holding one R1/R2 pair, the sample name repeated in the file name. + const fileNames = [ + "Lib85_Uns_L1-ds.ddabcbdb/Lib85_Uns_S1_L001_R1_001.fastq.gz", + "Lib85_Uns_L1-ds.ddabcbdb/Lib85_Uns_S1_L001_R2_001.fastq.gz", + "Sel78_R1_01_L1-ds.8350c0fa/Sel78_R1_01_S2_L001_R1_001.fastq.gz", + "Sel78_R1_01_L1-ds.8350c0fa/Sel78_R1_01_S2_L001_R2_001.fastq.gz", + "Sel78_R1_02_L1-ds.bf24f8fb/Sel78_R1_02_S3_L001_R1_001.fastq.gz", + "Sel78_R1_02_L1-ds.bf24f8fb/Sel78_R1_02_S3_L001_R2_001.fastq.gz", + "Sel78_R6_02_L1-ds.7acb3f2c/Sel78_R6_02_S9_L001_R1_001.fastq.gz", + "Sel78_R6_02_L1-ds.7acb3f2c/Sel78_R6_02_S9_L001_R2_001.fastq.gz", + ]; + const result = inferFileNamePattern(fileNames); + expect(result?.pattern.rawPattern).to.equal( + "{{**}}/{{Sample}}_S{{n}}_L{{n}}_{{RR}}_{{n}}.fastq.gz", + ); + expect(result?.readIndices).to.toMatchObject(["R1", "R2"]); + // Neither the enclosing folder nor the Illumina sample number leaks in. + const samples = new Set(fileNames.map((f) => result?.pattern.match(f)?.sample.value)); + expect([...samples].sort()).to.toMatchObject([ + "Lib85_Uns", + "Sel78_R1_01", + "Sel78_R1_02", + "Sel78_R6_02", + ]); +}); + +test("infer per-sample folders whose file names carry no sample", ({ expect }) => { + const fileNames = [ + "Sample_A/R1.fastq.gz", + "Sample_A/R2.fastq.gz", + "Sample_B/R1.fastq.gz", + "Sample_B/R2.fastq.gz", + ]; + const result = inferFileNamePattern(fileNames); + expect(result?.pattern.rawPattern).to.equal("{{Sample}}/{{R}}.fastq.gz"); + expect(result?.readIndices).to.toMatchObject(["R1", "R2"]); + const samples = new Set(fileNames.map((f) => result?.pattern.match(f)?.sample.value)); + expect([...samples].sort()).to.toMatchObject(["Sample_A", "Sample_B"]); +}); + +test("infer CellRanger MTX laid out one folder per sample", ({ expect }) => { + const fileNames = [ + "Sample_A/barcodes.tsv.gz", + "Sample_A/features.tsv.gz", + "Sample_A/matrix.mtx.gz", + "Sample_B/barcodes.tsv.gz", + "Sample_B/features.tsv.gz", + "Sample_B/matrix.mtx.gz", + ]; + const result = inferFileNamePattern(fileNames); + expect(result?.pattern.rawPattern).to.equal("{{Sample}}/{{CellRangerFileRole}}.gz"); + expect(result?.pattern.datasetType).to.equal("CellRangerMTX"); + const samples = new Set(fileNames.map((f) => result?.pattern.match(f)?.sample.value)); + expect([...samples].sort()).to.toMatchObject(["Sample_A", "Sample_B"]); +}); + +test("a flat folder still infers a bare file-name pattern", ({ expect }) => { + const fileNames = ["A_S1_L001_R1_001.fastq.gz", "A_S1_L001_R2_001.fastq.gz"]; + const result = inferFileNamePattern(fileNames); + expect(result?.pattern.rawPattern).to.equal("{{Sample}}_S{{n}}_L{{n}}_{{RR}}_{{n}}.fastq.gz"); +}); diff --git a/ui/src/dialogs/file_name_parser.ts b/ui/src/dialogs/file_name_parser.ts index d737477..f27b67d 100644 --- a/ui/src/dialogs/file_name_parser.ts +++ b/ui/src/dialogs/file_name_parser.ts @@ -147,8 +147,22 @@ export class FileNamePattern { return result; } + /** + * Patterns are matched against a path relative to the import root, not a bare + * file name, so `/` can appear in the target. Matchers are therefore bounded + * to a single path segment — an unbounded `.` would happily eat a separator + * and pull a folder name into `{{Sample}}`. `{{**}}` opts back in to crossing + * segments, for the "ignore however many folders are above" case. + */ + private static readonly segmentGroup = "([^/]+?)"; + private static readonly deepGroup = "(.+?)"; + private static readonly numberGroup = "([0-9]+)"; + + // `anydeep` must precede `any` in the alternation, otherwise `{{**}}` matches + // `any` and then fails on the second `*`, and the whole element is left as a + // literal. private static patternElement = - /\{\{ *(:?(?l|lane)|(?r)|(?rr)|(?s|sample)|(?CellRangerFileRole)|\*?:(?[a-zA-Z0-9_]+)|n:(?[a-zA-Z0-9_]+)|(?\*)|(?n)) *\}\}/dgi; + /\{\{ *(:?(?l|lane)|(?r)|(?rr)|(?s|sample)|(?CellRangerFileRole)|\*?:(?[a-zA-Z0-9_]+)|n:(?[a-zA-Z0-9_]+)|(?\*\*)|(?\*)|(?n)) *\}\}/dgi; static parse(fileNamePattern: string, ops?: FileNamePatternParsingOps): FileNamePattern { let regexp = "^"; @@ -188,33 +202,37 @@ export class FileNamePattern { } else if (match.groups!["lane"]) { if (groups.lane !== undefined) throw new Error(`Repeated {{L}} / {{Lane}} matcher`); groups.lane = groupCounter++; - regexp += "([0-9]+)"; + regexp += FileNamePattern.numberGroup; rawElements.lane = range; } else if (match.groups!["sample"]) { if (groups.sample !== undefined) throw new Error(`Repeated {{S}} / {{Sample}} sample name matcher`); groups.sample = groupCounter++; - regexp += "(.+?)"; + regexp += FileNamePattern.segmentGroup; rawElements.sample = range; + } else if (match.groups!["anydeep"]) { + groups.anyMatchers!.push(groupCounter++); + regexp += FileNamePattern.deepGroup; + rawElements.anyMatchers!.push(range); } else if (match.groups!["any"]) { groups.anyMatchers!.push(groupCounter++); - regexp += "(.+?)"; + regexp += FileNamePattern.segmentGroup; rawElements.anyMatchers!.push(range); } else if (match.groups?.["anytag"]) { if (groups.tags === undefined) groups.tags = {}; groups.tags[match.groups["anytag"]] = groupCounter++; - regexp += "(.+?)"; + regexp += FileNamePattern.segmentGroup; if (rawElements.tags === undefined) rawElements.tags = {}; rawElements.tags[match.groups["anytag"]] = range; } else if (match.groups?.["anynumbertag"]) { if (groups.tags === undefined) groups.tags = {}; groups.tags[match.groups["anynumbertag"]] = groupCounter++; - regexp += "([0-9]+)"; + regexp += FileNamePattern.numberGroup; if (rawElements.tags === undefined) rawElements.tags = {}; rawElements.tags[match.groups["anynumbertag"]] = range; } else if (match.groups!["anynumber"]) { groups.anyNumberMatchers!.push(groupCounter++); - regexp += "([0-9]+)"; + regexp += FileNamePattern.numberGroup; rawElements.anyNumberMatchers!.push(range); } else if (match.groups!["cellRangerFileRole"]) { if (groups.cellRangerFileRole !== undefined) @@ -368,6 +386,34 @@ type WellKnownPattern = { }; const wellKnownPattern: WellKnownPattern[] = [ + // Canonical Illumina / BaseSpace naming: + // _S_L__001.fastq.gz. Tried ahead of + // the plain `_L…` bodies so the sample number is recognised as such instead + // of being swallowed into the sample name. + { + patternWithoutExtension: "{{Sample}}_S{{n}}_L{{n}}_{{RR}}_{{n}}", + defaultReadIndices: ["R1"], + extensions: ["fastq", "fastq.gz", "fq", "fq.gz"], + minimalPercent: 0.49, + }, + { + patternWithoutExtension: "{{Sample}}_S{{n}}_L{{n}}_{{RR}}", + defaultReadIndices: ["R1"], + extensions: ["fastq", "fastq.gz", "fq", "fq.gz"], + minimalPercent: 0.49, + }, + { + patternWithoutExtension: "{{Sample}}_S{{n}}_L{{L}}_{{RR}}_{{n}}", + defaultReadIndices: ["R1"], + extensions: ["fastq", "fastq.gz", "fq", "fq.gz"], + minimalPercent: 0.49, + }, + { + patternWithoutExtension: "{{Sample}}_S{{n}}_L{{L}}_{{RR}}", + defaultReadIndices: ["R1"], + extensions: ["fastq", "fastq.gz", "fq", "fq.gz"], + minimalPercent: 0.49, + }, { patternWithoutExtension: "{{Sample}}_L{{n}}_{{RR}}_{{n}}", defaultReadIndices: ["R1"], @@ -487,11 +533,35 @@ function setEquals(a: Set, b: Set): boolean { return a.size === b.size && [...a].every((x) => b.has(x)); } +/** + * How a well-known body is projected onto a path relative to the import root. + * + * Inputs are relative paths, so a one-folder-per-sample tree arrives here as + * `Sample_A/…` rather than a bare file name. `direct` is tried first, and is + * the only variant a single-folder import can match — matchers are + * segment-bounded, so a body without `/` can never match a nested path. That + * keeps flat imports inferring exactly what they inferred before. + */ +const pathVariants: ((body: string) => string)[] = [ + // Everything in one folder. + (body) => body, + // Sample identity is in the file name; however many folders sit above it are + // irrelevant (BaseSpace-style downloads). + (body) => `{{**}}/${body}`, + // Sample identity is in the folder name and the file name carries some other + // prefix before the read index. + (body) => `{{Sample}}/${body.replace(/\{\{Sample\}\}/g, "{{*}}")}`, + // Sample identity is in the folder name and the file name is nothing but the + // read index or the role — `SampleA/R1.fastq.gz`, `SampleA/matrix.mtx.gz`. + // The separator that followed {{Sample}} in the body goes with it. + (body) => `{{Sample}}/${body.replace(/\{\{Sample\}\}[-_]?/g, "")}`, +]; + export function inferFileNamePattern( fileNames: string[], ops?: InferFileNamePatternOps, ): InferFileNamePatternResult | undefined { - outer: for (const wkPattern of wellKnownPattern) { + for (const wkPattern of wellKnownPattern) { if (ops?.expectedReadIndices?.length === 0 && wkPattern.defaultReadIndices.length !== 0) // don't consider fasta pattern if non-zero set of read indices is expected continue; @@ -499,43 +569,57 @@ export function inferFileNamePattern( for (const extension of wkPattern.extensions) { if (ops?.isGzipped !== undefined && extension.endsWith(".gz") !== ops.isGzipped) continue; - const patternStr = extension + const body = extension ? wkPattern.patternWithoutExtension + "." + extension : wkPattern.patternWithoutExtension; - const pattern = FileNamePattern.parse(patternStr); - - let matchedFiles = 0; - const readIndices = pattern.hasReadIndexMatcher ? new Set() : undefined; - const samples = new Set(); - for (const file of fileNames) { - const match = pattern.match(file); - if (match !== undefined) { - let sample = match.sample.value; - if (match.lane) sample += "___" + match.lane.value; - if (match.readIndex) sample += "___" + match.readIndex.value; - if (match.cellRangerFileRole) sample += "___" + match.cellRangerFileRole.value; - if (samples.has(sample)) continue outer; - samples.add(sample); - matchedFiles++; - if (readIndices !== undefined) readIndices.add(getWellFormattedReadIndex(match)); + + variants: for (const asPath of pathVariants) { + const patternStr = asPath(body); + + let pattern: FileNamePattern; + try { + pattern = FileNamePattern.parse(patternStr); + } catch { + // A variant can be malformed for a given body — dropping {{Sample}} + // from a body that has no other matcher leaves nothing to capture. + continue variants; + } + + let matchedFiles = 0; + const readIndices = pattern.hasReadIndexMatcher ? new Set() : undefined; + const samples = new Set(); + for (const file of fileNames) { + const match = pattern.match(file); + if (match !== undefined) { + let sample = match.sample.value; + if (match.lane) sample += "___" + match.lane.value; + if (match.readIndex) sample += "___" + match.readIndex.value; + if (match.cellRangerFileRole) sample += "___" + match.cellRangerFileRole.value; + // Two files landing on one key means this variant cannot describe + // the set: reject it and try the next one. + if (samples.has(sample)) continue variants; + samples.add(sample); + matchedFiles++; + if (readIndices !== undefined) readIndices.add(getWellFormattedReadIndex(match)); + } } - } - const resultReadIndices = - readIndices === undefined ? wkPattern.defaultReadIndices : [...readIndices].sort(); - - if ( - ops?.expectedReadIndices !== undefined && - !setEquals(new Set(resultReadIndices), new Set(ops.expectedReadIndices)) - ) - continue; - - if (matchedFiles / fileNames.length > wkPattern.minimalPercent) - return { - pattern, - extension, - readIndices: resultReadIndices, - }; + const resultReadIndices = + readIndices === undefined ? wkPattern.defaultReadIndices : [...readIndices].sort(); + + if ( + ops?.expectedReadIndices !== undefined && + !setEquals(new Set(resultReadIndices), new Set(ops.expectedReadIndices)) + ) + continue variants; + + if (matchedFiles / fileNames.length > wkPattern.minimalPercent) + return { + pattern, + extension, + readIndices: resultReadIndices, + }; + } } } return undefined;