Read sample identity from the folder, not just the file name

.changeset/path-aware-file-patterns.md

@@ -0,0 +1,13 @@
+---
+"@platforma-open/milaboratories.samples-and-data.ui": minor
+---
+
+Read sample identity from the folder, not just the file name.
+
+Patterns were matched against the bare file name, so a one-folder-per-sample tree — what BaseSpace, bcl2fastq and CellRanger all produce — could not be described. Where the folder was the only place a sample name appeared, every file resolved to the same sample and all but the last were dropped without a word.
+
+- Patterns are now matched against each file's path relative to the longest common directory of the selection. With every file in one folder that is the bare file name, so flat imports are unchanged.
+- `{{Sample}}`, `{{*}}` and tag matchers are bounded to one path segment; the new `{{**}}` crosses segments. `{{**}}/{{Sample}}_S{{n}}_L{{n}}_{{RR}}_{{n}}.fastq.gz` and `{{Sample}}/{{R}}.fastq.gz` both work.
+- Inference tries the file name, folder-ignored and folder-carries-identity forms, and takes the first that gives every file its own identity. Per-sample-folder FASTQ and per-sample-folder CellRanger MTX (`Sample_A/matrix.mtx.gz`) now infer.
+- The canonical Illumina naming `<Sample>_S<n>_L<lane>_<read>_001` is recognised, so `A_S7_L001_R1_001.fastq.gz` gives sample `A` rather than `A_S7`. This changes inferred sample names for bcl2fastq/BaseSpace output — `_S<n>` is the sample number, not part of the name.
+- A pattern that maps two files onto one identity is reported in the import dialog and blocks the import, instead of silently overwriting.

ui/src/dialogs/ImportDatasetDialog.vue

@@ -19,9 +19,10 @@ import type {
   ReadIndices,
 } from "@platforma-open/milaboratories.samples-and-data.model";
 import type { ImportFileHandle } from "@platforma-sdk/model";
-import { getFileNameFromHandle, getFilePathFromHandle, uniquePlId } from "@platforma-sdk/model";
+import { uniquePlId } from "@platforma-sdk/model";
 import type { ListOption } from "@platforma-sdk/ui-vue";
 import {
+  PlAlert,
   PlBtnGhost,
   PlBtnGroup,
   PlBtnPrimary,
@@ -40,9 +41,10 @@ import { useApp } from "../app";
 import type { ImportMode } from "./datasets";
 import {
   datasetTypes,
-  extractFileName,
+  findDuplicateKeys,
   getOrCreateSample,
   modesOptions,
+  relativeFilePaths,
   useParsedFiles,
   usePatternCompilation,
 } from "./datasets";
@@ -215,6 +217,27 @@ const parsedFiles = useParsedFiles(data, compiledPattern);
 // Whether any of the files matched the pattern
 const hasMatchedFiles = computed(() => parsedFiles.value.filter((f) => f.match).length > 0);
 
+/**
+ * Files that the current pattern collapses onto one identity. The dataset
+ * content builders index by that identity, so importing would keep only the
+ * last file of each group — silently. Report instead, and hold the import.
+ */
+const duplicateKeys = computed(() => findDuplicateKeys(parsedFiles.value));
+
+const duplicateKeysMessage = computed(() => {
+  const groups = duplicateKeys.value;
+  if (groups.length === 0) return undefined;
+  const shown = groups
+    .slice(0, 3)
+    .map((g) => `"${g.sample}" ← ${g.fileNames.join(", ")}`)
+    .join("; ");
+  const rest = groups.length > 3 ? ` (and ${groups.length - 3} more)` : "";
+  return (
+    `${groups.length} sample(s) would be overwritten because several files resolve ` +
+    `to the same one: ${shown}${rest}. Adjust the pattern so each file gets its own sample.`
+  );
+});
+
 const dsTypeOptions = computed(() => {
   if (!data.fileType) {
     return [];
@@ -243,7 +266,9 @@ function updateDatasetType(datasetType: DSType | undefined) {
 
 // Add more files to the data
 function addFiles(files: ImportFileHandle[]) {
-  const fileNames = files.map((h) => extractFileName(getFilePathFromHandle(h)));
+  // Inference has to see the same strings matching will see later — paths
+  // relative to the folder the files came from, not bare file names.
+  const fileNames = relativeFilePaths(files);
   if (data.files.length === 0) {
     const inferredPattern = inferFileNamePattern(fileNames);
     if (inferredPattern) {
@@ -929,6 +954,7 @@ watch(availableColumnsOptions, (options) => {
 const canCreateOrAdd = computed(() => {
   const basicConditions =
     hasMatchedFiles.value &&
+    duplicateKeys.value.length === 0 &&
     (data.mode === "create-new-dataset" || data.targetAddDataset !== undefined) &&
     data.datasetType !== undefined &&
     !data.loadingColumns &&
@@ -1003,6 +1029,8 @@ const canCreateOrAdd = computed(() => {
       </PlRow>
     </div>
 
+    <PlAlert v-if="duplicateKeysMessage" type="error">{{ duplicateKeysMessage }}</PlAlert>
+
     <ParsedFilesList :items="parsedFiles" />
 
     <PlBtnSecondary @click="() => (data.fileDialogOpened = true)">

ui/src/dialogs/datasets.test.ts

@@ -0,0 +1,87 @@
+import { test } from "vitest";
+import type { ImportFileHandle } from "@platforma-sdk/model";
+import type { ParsedFile } from "./datasets";
+import { commonDirPrefix, findDuplicateKeys, toPosixPath } from "./datasets";
+import { FileNamePattern } from "./file_name_parser";
+
+test.for([
+  {
+    name: "files in one folder leave bare file names",
+    paths: ["/data/run/A_R1.fastq.gz", "/data/run/A_R2.fastq.gz"],
+    expected: "/data/run/",
+  },
+  {
+    name: "a single file leaves its bare name",
+    paths: ["/data/run/A_R1.fastq.gz"],
+    expected: "/data/run/",
+  },
+  {
+    name: "per-sample folders keep the folder",
+    paths: ["/data/run/A/R1.fastq.gz", "/data/run/B/R1.fastq.gz"],
+    expected: "/data/run/",
+  },
+  {
+    name: "a file name is never mistaken for a shared directory",
+    paths: ["/data/run/A/R1.fastq.gz", "/data/run/A/R2.fastq.gz"],
+    expected: "/data/run/A/",
+  },
+  {
+    name: "unrelated roots share nothing",
+    paths: ["/data/one/A_R1.fastq.gz", "runs/two/B_R1.fastq.gz"],
+    expected: "",
+  },
+  {
+    name: "a partial segment match is not a shared directory",
+    paths: ["/data/run1/A_R1.fastq.gz", "/data/run2/A_R1.fastq.gz"],
+    expected: "/data/",
+  },
+  {
+    name: "no paths",
+    paths: [],
+    expected: "",
+  },
+])("commonDirPrefix: $name", ({ paths, expected }, { expect }) => {
+  expect(commonDirPrefix(paths)).to.equal(expected);
+});
+
+test("toPosixPath normalizes Windows separators", ({ expect }) => {
+  expect(toPosixPath("C:\\data\\run\\A_R1.fastq.gz")).to.equal("C:/data/run/A_R1.fastq.gz");
+});
+
+/** ParsedFile carrying only what findDuplicateKeys reads. */
+function parsed(pattern: FileNamePattern, fileName: string): ParsedFile {
+  return {
+    handle: `upload://upload/${fileName}` as ImportFileHandle,
+    fileName,
+    match: pattern.match(fileName),
+  };
+}
+
+test("findDuplicateKeys is empty when every file resolves to its own identity", ({ expect }) => {
+  const pattern = FileNamePattern.parse("{{Sample}}/{{R}}.fastq.gz");
+  const files = ["A/R1.fastq.gz", "A/R2.fastq.gz", "B/R1.fastq.gz"].map((f) => parsed(pattern, f));
+  expect(findDuplicateKeys(files)).to.toMatchObject([]);
+});
+
+test("findDuplicateKeys catches files that would overwrite each other", ({ expect }) => {
+  // The folder carries the identity, but the pattern reads only the file name —
+  // so both samples collapse onto "R1" and one file would be dropped silently.
+  const pattern = FileNamePattern.parse("{{**}}/{{Sample}}.fastq.gz");
+  const files = ["A/R1.fastq.gz", "B/R1.fastq.gz"].map((f) => parsed(pattern, f));
+  const duplicates = findDuplicateKeys(files);
+  expect(duplicates.length).to.equal(1);
+  expect(duplicates[0].sample).to.equal("R1");
+  expect(duplicates[0].fileNames).to.toMatchObject(["A/R1.fastq.gz", "B/R1.fastq.gz"]);
+});
+
+test("findDuplicateKeys treats read index and lane as part of the identity", ({ expect }) => {
+  const pattern = FileNamePattern.parse("{{Sample}}_L{{L}}_{{RR}}.fastq.gz");
+  const distinct = ["A_L001_R1.fastq.gz", "A_L001_R2.fastq.gz", "A_L002_R1.fastq.gz"].map((f) =>
+    parsed(pattern, f),
+  );
+  expect(findDuplicateKeys(distinct)).to.toMatchObject([]);
+
+  // Unmatched files carry no identity and must not be reported as duplicates.
+  const unmatched = ["nope.txt", "also-nope.txt"].map((f) => parsed(pattern, f));
+  expect(findDuplicateKeys(unmatched)).to.toMatchObject([]);
+});

ui/src/dialogs/datasets.ts

@@ -1,6 +1,6 @@
 import type { BlockData, DSType } from "@platforma-open/milaboratories.samples-and-data.model";
 import type { ImportFileHandle, PlId } from "@platforma-sdk/model";
-import { getFileNameFromHandle, uniquePlId } from "@platforma-sdk/model";
+import { getFilePathFromHandle, uniquePlId } from "@platforma-sdk/model";
 import type { AppV3, SimpleOption } from "@platforma-sdk/ui-vue";
 import type { ComputedRef, Reactive, ShallowRef } from "vue";
 import { computed, ref, shallowRef, watch } from "vue";
@@ -126,6 +126,44 @@ export function extractFileName(filePath: string) {
   return filePath.replace(/^.*[\\/]/, "");
 }
 
+/** Windows handles carry `\`; patterns and prefix arithmetic assume `/`. */
+export function toPosixPath(filePath: string) {
+  return filePath.replace(/\\/g, "/");
+}
+
+/**
+ * Longest directory prefix shared by every path, with a trailing `/`, or `""`
+ * when they share no directory at all.
+ *
+ * The last segment of a path is its file name and is never part of the prefix,
+ * so a single file — or a set of files all sitting in one folder — yields that
+ * folder and leaves bare file names behind.
+ */
+export function commonDirPrefix(paths: string[]): string {
+  if (paths.length === 0) return "";
+  const segments = paths.map((p) => p.split("/"));
+  const maxDirs = Math.min(...segments.map((s) => s.length - 1));
+  const first = segments[0];
+  let shared = 0;
+  while (shared < maxDirs && segments.every((s) => s[shared] === first[shared])) shared++;
+  return shared === 0 ? "" : first.slice(0, shared).join("/") + "/";
+}
+
+/**
+ * Paths of the given files relative to the folder they were imported from.
+ *
+ * Patterns are matched against these rather than against bare file names, so
+ * `{{Sample}}` can be read out of a folder name in one-folder-per-sample
+ * layouts. Recomputing the prefix over the whole set on every call — instead of
+ * threading a root through the file dialog — keeps this well defined when the
+ * selection spans folders or grows over several "add more files" rounds.
+ */
+export function relativeFilePaths(handles: ImportFileHandle[]): string[] {
+  const paths = handles.map((h) => toPosixPath(getFilePathFromHandle(h)));
+  const prefix = commonDirPrefix(paths);
+  return paths.map((p) => p.slice(prefix.length));
+}
+
 // Pattern compilation and file name matching
 export function usePatternCompilation(data: Reactive<{ pattern: string }>) {
   const patternError = ref<string | undefined>(undefined);
@@ -164,17 +202,53 @@ export function useParsedFiles(
   data: Reactive<{ files: ImportFileHandle[] }>,
   compiledPattern: ShallowRef<FileNamePattern | undefined>,
 ): ComputedRef<ParsedFile[]> {
-  return computed<ParsedFile[]>(() =>
-    data.files.map((handle) => {
-      const fileName = extractFileName(getFileNameFromHandle(handle));
+  return computed<ParsedFile[]>(() => {
+    const paths = relativeFilePaths(data.files);
+    return data.files.map((handle, i) => {
+      const fileName = paths[i];
       const match = compiledPattern.value?.match(fileName);
       return {
         handle,
         fileName,
         match,
       };
-    }),
-  );
+    });
+  });
+}
+
+/**
+ * The identity a matched file resolves to — everything the dataset content
+ * builders key on when they place a handle. Two files sharing a key would
+ * overwrite one another, keeping only whichever was processed last.
+ */
+export function sampleKeyOf(match: FileNamePatternMatch): string {
+  const parts = [match.sample.value];
+  if (match.lane) parts.push("lane=" + match.lane.value);
+  if (match.readIndex) parts.push("read=" + match.readIndex.value);
+  if (match.cellRangerFileRole) parts.push("role=" + match.cellRangerFileRole.value);
+  for (const tag of Object.keys(match.tags ?? {}).sort())
+    parts.push(`${tag}=${match.tags![tag].value}`);
+  return parts.join("\u0000");
+}
+
+export type DuplicateKey = { sample: string; fileNames: string[] };
+
+/** Groups of matched files that collapse onto one identity. Empty when fine. */
+export function findDuplicateKeys(files: ParsedFile[]): DuplicateKey[] {
+  const byKey = new Map<string, ParsedFile[]>();
+  for (const f of files) {
+    if (!f.match) continue;
+    const key = sampleKeyOf(f.match);
+    const group = byKey.get(key);
+    if (group) group.push(f);
+    else byKey.set(key, [f]);
+  }
+  return [...byKey.values()]
+    .filter((group) => group.length > 1)
+    .map((group) => ({
+      sample: group[0].match!.sample.value,
+      fileNames: group.map((f) => f.fileName),
+    }));
 }
 
 export function getOrCreateSample(appUt: unknown, sampleName: string): PlId {