From 509e2a25422fd282bf47c4c2c16a286ba98b1750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Tue, 4 Aug 2026 17:22:39 +0200 Subject: [PATCH 01/40] Add Batch and DatasetBatch schema for experiment organization --- dj_pipeline/vr4mice/schema/decision.py | 16 +++++- dj_pipeline/vr4mice/schema/vr4mice.py | 71 ++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/dj_pipeline/vr4mice/schema/decision.py b/dj_pipeline/vr4mice/schema/decision.py index 3c0578fd..4123898b 100644 --- a/dj_pipeline/vr4mice/schema/decision.py +++ b/dj_pipeline/vr4mice/schema/decision.py @@ -123,7 +123,7 @@ def sync_lookup_contents(): On deployed databases, new rows added to ``contents`` in code must be back-filled explicitly (parent rows before ``ExperimentMember.populate``). """ - for table in (ExperimentSet, ExperimentStage, SessionLabel): + for table in (ExperimentSet, ExperimentStage, SessionLabel, vr4mice.Batch): if table.contents: table.insert(table.contents, skip_duplicates=True) @@ -132,7 +132,8 @@ def sync_lookup_contents(): class ExperimentMember(dj.Imported): """ ExperimentMember definition table: - links each dataset to an experiment set, stage, and session label + links each dataset to an experiment set, stage, session label, and batch + (batch is resolved from the dataset's experiment date, see vr4mice.Batch) """ definition = """ @@ -141,6 +142,7 @@ class ExperimentMember(dj.Imported): -> ExperimentSet -> ExperimentStage -> SessionLabel + -> vr4mice.Batch """ def make(self, key): @@ -164,12 +166,20 @@ def make(self, key): label_info = label_info[0] + batch_info = (vr4mice.DatasetBatch & key).fetch(as_dict=True) + if not batch_info: + raise ValueError( + f"DatasetBatch missing for '{key['dataset']}'; " + "run vr4mice.DatasetBatch.populate() first" + ) + self.insert1( { "dataset": key["dataset"], "set_name": label_info["set_name"], "stage_name": label_info["stage_name"], "session_label": session_label, + "batch_name": batch_info[0]["batch_name"], } ) except Exception as err: @@ -404,6 +414,7 @@ class PredictionModel(dj.Computed): -> ModelParams -> ExperimentSet -> ExperimentStage + -> vr4mice.Batch --- coefficients : # coefficients per session (per_mouse=True) n_sessions : int # number of sessions included @@ -611,6 +622,7 @@ class PredictionModel10Windows(dj.Computed): -> ModelParams -> ExperimentSet -> ExperimentStage + -> vr4mice.Batch --- coefficients_by_window : # dict mapping window_id (0-9) -> coefficients scalers_by_window : # dict mapping window_id (0-9) -> list of scaler params per fold diff --git a/dj_pipeline/vr4mice/schema/vr4mice.py b/dj_pipeline/vr4mice/schema/vr4mice.py index 27e03066..076b9568 100644 --- a/dj_pipeline/vr4mice/schema/vr4mice.py +++ b/dj_pipeline/vr4mice/schema/vr4mice.py @@ -1,5 +1,6 @@ """Core VR4Mice schema tables for datasets, metadata, and raw signals.""" +import datetime import os import datajoint as dj @@ -93,6 +94,76 @@ def make(self, key): logger.warning(err) +@schema +class Batch(dj.Lookup): + """ + Batch definition table: + experiment batches (e.g. mouse cohorts run under a different setup), + each starting on a given date. A dataset belongs to whichever batch has + the latest start_date that is still <= its own experiment date, so new + batches are added over time by inserting a new row here, not by editing + code. + """ + + definition = """ + batch_name : varchar(32) + --- + start_date : date # datasets on/after this date fall in this batch + description : varchar(255) + has_neural_data : bool + """ + + contents = [ + ("batch1", "2000-01-01", "behavioral_cohort", False), + ("batch2", "2026-06-01", "ephys_cohort", True), + ] + + @classmethod + def resolve(cls, doe): + """Return the batch_name for an experiment date (latest start_date <= doe).""" + rows = (cls & f"start_date <= '{doe}'").fetch( + "batch_name", order_by="start_date DESC", limit=1 + ) + if len(rows) == 0: + raise ValueError(f"No {cls.__name__} defined with start_date <= {doe}") + return rows[0] + + +@schema +class DatasetBatch(dj.Computed): + """ + DatasetBatch definition table: + resolves each dataset to a Batch, based on the experiment date (doe) + encoded in its name (see Dataset docstring: mouse_name_doe_attempt) + """ + + definition = """ + -> Dataset + --- + -> Batch + doe : date # experiment date parsed from the dataset name + """ + + def make(self, key): + parts = key["dataset"].split("_") + if len(parts) < 2: + logger.warning(f"Could not parse date from dataset '{key['dataset']}'") + return + try: + doe = datetime.date.fromisoformat(parts[1]) + except ValueError: + logger.warning(f"Could not parse date from dataset '{key['dataset']}'") + return + + try: + batch_name = Batch.resolve(doe) + except ValueError as err: + logger.warning(f"{err} (dataset '{key['dataset']}')") + return + + self.insert1({**key, "batch_name": batch_name, "doe": doe}) + + @schema class FailedSession(dj.Manual): """Tracks dataset/table pairs that failed during populate/compute.""" From 4c18078d8750d7dc4548400b156fcd0f3f0a1410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Tue, 4 Aug 2026 19:00:17 +0200 Subject: [PATCH 02/40] Add ExcludedDataset schema and update decision analysis logic --- dj_pipeline/cron_scenario.py | 85 ++++++++++++++------------ dj_pipeline/run.py | 5 +- dj_pipeline/vr4mice/schema/barcodes.py | 6 +- dj_pipeline/vr4mice/schema/decision.py | 57 ++++++++++------- dj_pipeline/vr4mice/schema/vr4mice.py | 59 ++++++++++++++++++ 5 files changed, 145 insertions(+), 67 deletions(-) diff --git a/dj_pipeline/cron_scenario.py b/dj_pipeline/cron_scenario.py index 48834094..0bf76fd7 100755 --- a/dj_pipeline/cron_scenario.py +++ b/dj_pipeline/cron_scenario.py @@ -84,10 +84,10 @@ def import_actions(): def import_core_schemas(): from vr4mice.schema import ( + vr4mice, barcodes, base_analysis, dlc, - vr4mice, interpolated_trajectories, session_metrics, latency_tests, @@ -95,10 +95,10 @@ def import_core_schemas(): ) return ( + vr4mice, barcodes, base_analysis, dlc, - vr4mice, interpolated_trajectories, session_metrics, latency_tests, @@ -113,10 +113,10 @@ def import_core_schemas(): ) ( + vr4mice, barcodes, base_analysis, dlc, - vr4mice, interpolated_trajectories, session_metrics, latency_tests, @@ -128,6 +128,14 @@ def import_core_schemas(): lambda: create_folder_if_not_exist("/data/summary_plots"), ) + run_step( + "vr4mice.Batch.sync_contents", + lambda: vr4mice.Batch.insert(vr4mice.Batch.contents, skip_duplicates=True), + ) + run_step( + "vr4mice.DatasetBatch.populate", + lambda: vr4mice.DatasetBatch().populate(), + ) run_step( "vr4mice.Collab.populate", lambda: populate_pending(vr4mice.Collab, vr4mice.Dataset, logger=logger), @@ -270,45 +278,44 @@ def import_core_schemas(): lambda: summary_emails.send_pending_summary_emails(logger=logger), ) - if args.aws: + def import_decision_schema(): + from vr4mice.schema import decision - def import_decision_schema(): - from vr4mice.schema import decision + return decision - return decision + decision = run_import("import decision schema", import_decision_schema) + if decision: + run_step( + "decision.sync_lookup_contents", + decision.sync_lookup_contents, + ) + run_step( + "decision.ExperimentMember.populate", + lambda: decision.ExperimentMember().populate(), + ) + run_step( + "decision.InclusionStatus.populate", + lambda: decision.InclusionStatus().populate(), + ) + run_step("decision.LabelSet.fill", lambda: decision.LabelSet.fill()) + run_step( + "decision.PredictionModel.populate", + lambda: decision.PredictionModel().populate(), + ) + run_step( + "decision.DecisionPoints.populate", + lambda: decision.DecisionPoints().populate(), + ) + run_step( + "decision.PredictionModel10Windows.populate", + lambda: decision.PredictionModel10Windows().populate(), + ) + run_step( + "decision.DecisionPoints10Windows.populate", + lambda: decision.DecisionPoints10Windows().populate(), + ) - decision = run_import("import decision schema", import_decision_schema) - if decision: - run_step( - "decision.sync_lookup_contents", - decision.sync_lookup_contents, - ) - run_step( - "decision.ExperimentMember.populate", - lambda: decision.ExperimentMember().populate(), - ) - run_step( - "decision.InclusionStatus.populate", - lambda: decision.InclusionStatus().populate(), - ) - run_step("decision.LabelSet.fill", lambda: decision.LabelSet.fill()) - run_step( - "decision.PredictionModel.populate", - lambda: decision.PredictionModel().populate(), - ) - run_step( - "decision.DecisionPoints.populate", - lambda: decision.DecisionPoints().populate(), - ) - run_step( - "decision.PredictionModel10Windows.populate", - lambda: decision.PredictionModel10Windows().populate(), - ) - run_step( - "decision.DecisionPoints10Windows.populate", - lambda: decision.DecisionPoints10Windows().populate(), - ) - else: + if not args.aws: def import_inputs_videos_schema(): from vr4mice.schema import inputs_videos diff --git a/dj_pipeline/run.py b/dj_pipeline/run.py index 8d94df2b..06c613cf 100644 --- a/dj_pipeline/run.py +++ b/dj_pipeline/run.py @@ -88,7 +88,7 @@ def check_folder_existence(folder_path): connect(tag="") if args.mode == "connect": - from vr4mice.schema import barcodes, base, base_analysis, dlc, vr4mice + from vr4mice.schema import base, base_analysis, dlc, vr4mice, barcodes pass @@ -144,7 +144,7 @@ def check_folder_existence(folder_path): elif args.mode == "dlc": # NOTE: populate and analysis have to be run before - from vr4mice.schema import barcodes, dlc, vr4mice + from vr4mice.schema import dlc, vr4mice, barcodes from vr4mice.utils.populate_helpers import populate_pending create_folder_if_not_exist("/data/summary_plots") @@ -261,6 +261,7 @@ def check_folder_existence(folder_path): from vr4mice.schema import decision decision.sync_lookup_contents() + vr4mice.DatasetBatch().populate() decision.ExperimentMember().populate() decision.InclusionStatus().populate() decision.LabelSet().fill() diff --git a/dj_pipeline/vr4mice/schema/barcodes.py b/dj_pipeline/vr4mice/schema/barcodes.py index c1045e60..6b264f97 100644 --- a/dj_pipeline/vr4mice/schema/barcodes.py +++ b/dj_pipeline/vr4mice/schema/barcodes.py @@ -18,8 +18,6 @@ logger = logger.Logger.get_logger() -TEENSY_TTL_START_DATE = "2026-06-01" - @schema class TeensyTTL(dj.Imported): @@ -33,7 +31,9 @@ class TeensyTTL(dj.Imported): has_ttl=0: bool # True when aligned, non-empty Teensy TTL arrays are available """ - key_source = vr4mice.DLC & f"doe > '{TEENSY_TTL_START_DATE}'" + # Teensy barcodes were only wired up starting with the batch2 (Neuropixels) cohort + _batch2_start_date = (vr4mice.Batch & {"batch_name": "batch2"}).fetch1("start_date") + key_source = vr4mice.DLC & f"doe > '{_batch2_start_date}'" def make(self, key): """Load the raw Teensy TTL arrays from one DLC PROC file.""" diff --git a/dj_pipeline/vr4mice/schema/decision.py b/dj_pipeline/vr4mice/schema/decision.py index 4123898b..b39ea8d0 100644 --- a/dj_pipeline/vr4mice/schema/decision.py +++ b/dj_pipeline/vr4mice/schema/decision.py @@ -13,6 +13,7 @@ from vr4mice.schema.interpolated_trajectories import InterpolatedTrials from vr4mice.utils.logger import Logger +from vr4mice.utils.populate_helpers import BEHAVIOR_DATASET_RESTRICTION from vr4mice.utils.schema_config import get_schema @@ -123,7 +124,13 @@ def sync_lookup_contents(): On deployed databases, new rows added to ``contents`` in code must be back-filled explicitly (parent rows before ``ExperimentMember.populate``). """ - for table in (ExperimentSet, ExperimentStage, SessionLabel, vr4mice.Batch): + for table in ( + ExperimentSet, + ExperimentStage, + SessionLabel, + vr4mice.Batch, + vr4mice.ExcludedDataset, + ): if table.contents: table.insert(table.contents, skip_duplicates=True) @@ -133,7 +140,10 @@ class ExperimentMember(dj.Imported): """ ExperimentMember definition table: links each dataset to an experiment set, stage, session label, and batch - (batch is resolved from the dataset's experiment date, see vr4mice.Batch) + (batch_name resolved from vr4mice.DatasetBatch; run its populate() first). + + Latencytest* sessions and datasets matched by vr4mice.ExcludedDataset + are skipped and never get a row here. """ definition = """ @@ -147,6 +157,19 @@ class ExperimentMember(dj.Imported): def make(self, key): try: + if not (Dataset & key & BEHAVIOR_DATASET_RESTRICTION): + logger.debug( + f"Skipping latency test session for {self.__class__.__name__}: {key['dataset']}" + ) + return + + excluded_reason = vr4mice.ExcludedDataset.matches(key["dataset"]) + if excluded_reason: + logger.debug( + f"Skipping excluded dataset for {self.__class__.__name__}: {key['dataset']} ({excluded_reason})" + ) + return + session_label = (Dataset & key).fetch1("session_label") if not session_label: @@ -218,31 +241,19 @@ def make(self, key): self.insert1({**key, "included": False}) return - # NOTE(celia): this is a fix to exclude datasets that were not manually added by tom - # in the Groups() table, but this table was not consistently populated for all datasets - # so we exclude these datasets in this hardcoded way for now - # We could also drop the Groups table entirely if it's not used elsewhere - # NOTE(celia): (update) for now kept, but the Groups table was dropped in the DJ 2.0 migration so the - # code should get the correct set of datasets without it now. - # if ( - # session_label == "ar_discrim_occluders" - # or session_label == "ar_discrim_5_occluders" - # ): - # tables = TrialMetrics() * vr4mice.Groups() * (Dataset() & key) - # else: + excluded_reason = vr4mice.ExcludedDataset.matches(key["dataset"]) + if excluded_reason: + logger.debug( + f"Skipping excluded dataset for {self.__class__.__name__}: {key['dataset']} ({excluded_reason})" + ) + self.insert1({**key, "included": False}) + return + tables = TrialMetrics() * (Dataset() & key) trial_df = tables.fetch(as_dict=True) - # NOTE(celia): - # "Lemming_2024-08-09_1" and "Lemming_2024-08-09_2" were dropped through the Groups table for now, - # but kept if we decide to drop it - # "Hamster_2026-02-02_1" is missing the dlc data - if not trial_df or key in [ - {"dataset": "Hamster_2026-02-02_1"}, - {"dataset": "Lemming_2024-08-09_1"}, - {"dataset": "Lemming_2024-08-09_2"}, - ]: + if not trial_df: self.insert1({**key, "included": False}) return diff --git a/dj_pipeline/vr4mice/schema/vr4mice.py b/dj_pipeline/vr4mice/schema/vr4mice.py index 076b9568..dfb1e539 100644 --- a/dj_pipeline/vr4mice/schema/vr4mice.py +++ b/dj_pipeline/vr4mice/schema/vr4mice.py @@ -164,6 +164,65 @@ def make(self, key): self.insert1({**key, "batch_name": batch_name, "doe": doe}) +@schema +class ExcludedDataset(dj.Manual): + """ + ExcludedDataset definition table: + datasets excluded from decision analyses - curation calls, known + data-quality issues, or whole test/debug mice - even though their raw + data may still be ingested normally through the rest of the pipeline. + + dataset_pattern is either an exact dataset name (excludes that one + session) or a SQL LIKE pattern such as "Testmouse%" (excludes every + matching dataset, past and future, without needing a row per session). + Not a -> Dataset FK, since a "%" pattern doesn't name a real row. + """ + + definition = """ + dataset_pattern : varchar(512) # exact dataset name, or a SQL LIKE pattern + --- + reason : varchar(255) + """ + + contents = [ + ("Testmouse%", "Test/debug mouse used to exercise the rig"), + # NOTE(celia): inherited from the dropped Groups table (DJ 2.0 + # migration); the original curation reason was not recorded. + ( + "Lemming_2024-08-09_1", + "Excluded via legacy Groups curation; reason not recorded", + ), + ( + "Lemming_2024-08-09_2", + "Excluded via legacy Groups curation; reason not recorded", + ), + ("Hamster_2026-02-02_1", "Missing DLC data"), + ] + + @classmethod + def matches(cls, dataset): + """Return the matching row's reason, or None if dataset isn't excluded.""" + rows = (cls & f"'{dataset}' LIKE dataset_pattern").fetch(as_dict=True) + if not rows: + return None + return rows[0]["reason"] + + @classmethod + def restriction(cls): + """Return a `dataset`-column restriction string excluding every pattern here. + + Use as e.g. `Dataset() & ExcludedDataset.restriction()` in ad hoc + queries (notebooks, one-off scripts) that build a cohort directly + from Dataset/TrialMetrics instead of going through + decision.ExperimentMember/InclusionStatus, which already apply this + exclusion automatically. + """ + patterns = cls.fetch("dataset_pattern") + if len(patterns) == 0: + return "TRUE" + return " AND ".join(f'dataset NOT LIKE "{p}"' for p in patterns) + + @schema class FailedSession(dj.Manual): """Tracks dataset/table pairs that failed during populate/compute.""" From 531b8b388e2198d5646d85c3723f93aee605b271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Tue, 4 Aug 2026 19:06:18 +0200 Subject: [PATCH 03/40] Add key_source property to PredictionModel and PredictionModel10Windows to restrict session combos --- dj_pipeline/vr4mice/schema/decision.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/dj_pipeline/vr4mice/schema/decision.py b/dj_pipeline/vr4mice/schema/decision.py index b39ea8d0..22b9cdd1 100644 --- a/dj_pipeline/vr4mice/schema/decision.py +++ b/dj_pipeline/vr4mice/schema/decision.py @@ -435,6 +435,18 @@ class PredictionModel(dj.Computed): bic : float # Bayesian Information Criterion for the model """ + @property + def key_source(self): + # Restrict to (set_name, stage_name, batch_name) combos that actually + # have included sessions, instead of the full cartesian product with + # LabelSet/ModelParams/ExperimentSet/ExperimentStage/Batch - most + # combos have no data (e.g. "training" stage, or a batch that hasn't + # run a given task yet), and make() raises for those. + included_combos = dj.U("set_name", "stage_name", "batch_name") & ( + InclusionStatus * ExperimentMember & {"included": 1} + ) + return LabelSet.proj() * ModelParams.proj() * included_combos + class SessionPrediction(dj.Part): definition = """ -> master @@ -646,6 +658,15 @@ class PredictionModel10Windows(dj.Computed): cross_window_accuracy_mean : float # mean off-diagonal cross-window accuracy """ + @property + def key_source(self): + # See PredictionModel.key_source: restrict to (set_name, stage_name, + # batch_name) combos that actually have included sessions. + included_combos = dj.U("set_name", "stage_name", "batch_name") & ( + InclusionStatus * ExperimentMember & {"included": 1} + ) + return LabelSet.proj() * ModelParams.proj() * included_combos + class SessionPrediction(dj.Part): definition = """ -> master From d08e65a7b06039e44b83834b6403a1f833a456db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Tue, 4 Aug 2026 19:09:30 +0200 Subject: [PATCH 04/40] Refactor ExperimentMember to fetch batch_name directly from DatasetBatch --- dj_pipeline/vr4mice/schema/decision.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dj_pipeline/vr4mice/schema/decision.py b/dj_pipeline/vr4mice/schema/decision.py index 22b9cdd1..38afee43 100644 --- a/dj_pipeline/vr4mice/schema/decision.py +++ b/dj_pipeline/vr4mice/schema/decision.py @@ -189,12 +189,12 @@ def make(self, key): label_info = label_info[0] - batch_info = (vr4mice.DatasetBatch & key).fetch(as_dict=True) - if not batch_info: + if not (vr4mice.DatasetBatch & key): raise ValueError( f"DatasetBatch missing for '{key['dataset']}'; " "run vr4mice.DatasetBatch.populate() first" ) + batch_name = (vr4mice.DatasetBatch & key).fetch1("batch_name") self.insert1( { @@ -202,7 +202,7 @@ def make(self, key): "set_name": label_info["set_name"], "stage_name": label_info["stage_name"], "session_label": session_label, - "batch_name": batch_info[0]["batch_name"], + "batch_name": batch_name, } ) except Exception as err: From 2a47cc34c096fe711627da4ec18fedbba15c88c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Tue, 4 Aug 2026 19:16:09 +0200 Subject: [PATCH 05/40] Rename restriction method to exclusion_filter in ExcludedDataset for clarity --- dj_pipeline/vr4mice/schema/vr4mice.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dj_pipeline/vr4mice/schema/vr4mice.py b/dj_pipeline/vr4mice/schema/vr4mice.py index dfb1e539..e289b8ad 100644 --- a/dj_pipeline/vr4mice/schema/vr4mice.py +++ b/dj_pipeline/vr4mice/schema/vr4mice.py @@ -208,14 +208,13 @@ def matches(cls, dataset): return rows[0]["reason"] @classmethod - def restriction(cls): + def exclusion_filter(cls): """Return a `dataset`-column restriction string excluding every pattern here. - Use as e.g. `Dataset() & ExcludedDataset.restriction()` in ad hoc - queries (notebooks, one-off scripts) that build a cohort directly - from Dataset/TrialMetrics instead of going through - decision.ExperimentMember/InclusionStatus, which already apply this - exclusion automatically. + Use as e.g. `Dataset() & ExcludedDataset.exclusion_filter()` in ad + hoc queries that build a cohort directly from Dataset/TrialMetrics + instead of going through decision.ExperimentMember/InclusionStatus, + which already apply this exclusion automatically. """ patterns = cls.fetch("dataset_pattern") if len(patterns) == 0: From 9d79427bbd18b0890d57d57695e4195fe75968d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Tue, 4 Aug 2026 19:19:42 +0200 Subject: [PATCH 06/40] Change ExcludedDataset from Manual to Lookup for improved data handling --- dj_pipeline/vr4mice/schema/vr4mice.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dj_pipeline/vr4mice/schema/vr4mice.py b/dj_pipeline/vr4mice/schema/vr4mice.py index e289b8ad..6f3e349a 100644 --- a/dj_pipeline/vr4mice/schema/vr4mice.py +++ b/dj_pipeline/vr4mice/schema/vr4mice.py @@ -165,7 +165,7 @@ def make(self, key): @schema -class ExcludedDataset(dj.Manual): +class ExcludedDataset(dj.Lookup): """ ExcludedDataset definition table: datasets excluded from decision analyses - curation calls, known @@ -212,8 +212,8 @@ def exclusion_filter(cls): """Return a `dataset`-column restriction string excluding every pattern here. Use as e.g. `Dataset() & ExcludedDataset.exclusion_filter()` in ad - hoc queries that build a cohort directly from Dataset/TrialMetrics - instead of going through decision.ExperimentMember/InclusionStatus, + hoc queries that build a cohort directly from Dataset/TrialMetrics + instead of going through decision.ExperimentMember/InclusionStatus, which already apply this exclusion automatically. """ patterns = cls.fetch("dataset_pattern") From e3da5c4003a204b9e07e412c9c3f5bc8dc74bc19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Tue, 4 Aug 2026 19:53:36 +0200 Subject: [PATCH 07/40] Exclude Whale --- dj_pipeline/vr4mice/schema/vr4mice.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dj_pipeline/vr4mice/schema/vr4mice.py b/dj_pipeline/vr4mice/schema/vr4mice.py index 6f3e349a..095e1a44 100644 --- a/dj_pipeline/vr4mice/schema/vr4mice.py +++ b/dj_pipeline/vr4mice/schema/vr4mice.py @@ -197,6 +197,7 @@ class ExcludedDataset(dj.Lookup): "Excluded via legacy Groups curation; reason not recorded", ), ("Hamster_2026-02-02_1", "Missing DLC data"), + ("Whale%", "Ephys recording didn't work; stopped at detection stage"), ] @classmethod From 3b83c261e90632d33aa41304df9ef8229d5b1fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 15:18:56 +0200 Subject: [PATCH 08/40] Add np_sync module for VR-to-NP barcode alignment and corresponding unit tests --- dj_pipeline/.env.compose.example | 3 + dj_pipeline/docker-compose.yml | 3 +- dj_pipeline/vr4mice/analysis/np_sync.py | 73 ++++++++++++++++++++++++ tests/unit/test_np_sync.py | 74 +++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 dj_pipeline/vr4mice/analysis/np_sync.py create mode 100644 tests/unit/test_np_sync.py diff --git a/dj_pipeline/.env.compose.example b/dj_pipeline/.env.compose.example index 381ca147..2def8f90 100644 --- a/dj_pipeline/.env.compose.example +++ b/dj_pipeline/.env.compose.example @@ -11,6 +11,9 @@ DB_DATA_PATH=/mnt/database/vr4mice/vr4mice_database/database/ SHARED_PATH=/mnt/database/shared DATA_PATH=/mnt/database/vr4mice/vr4mice_database/data SCREEN_RECORDINGS_PATH=/mnt/neuropixel_data/vr4mice/raw_screen_recordings +# Path to the np_pipeline "src" checkout (must contain a np_pipeline/ package dir), +# mounted so vr4mice/schema/np_sync.py can import it. Defaults to a sibling checkout. +NP_PIPELINE_SRC_PATH=../FreelyMoving_Mouse_NP/src JUPYTER_PORT=8887 CLIENT_IMAGE=mmathislab/vr4mice_app:0.1.0 CLIENT_CONTAINER_NAME=vr4mice_${USER} diff --git a/dj_pipeline/docker-compose.yml b/dj_pipeline/docker-compose.yml index 8323340f..5c78504b 100644 --- a/dj_pipeline/docker-compose.yml +++ b/dj_pipeline/docker-compose.yml @@ -36,7 +36,8 @@ services: - "${SCREEN_RECORDINGS_PATH:-/mnt/neuropixel_data/vr4mice/raw_screen_recordings}:/vr4mice_screen_recordings" - "./base/base_min_schemas:/base_schemas" - "./base/base_actions:/base_actions" - ports: + - "${NP_PIPELINE_SRC_PATH:-../FreelyMoving_Mouse_NP/src}:/np_pipeline" + ports: - "${JUPYTER_PORT:-8887}:8888" working_dir: /app env_file: diff --git a/dj_pipeline/vr4mice/analysis/np_sync.py b/dj_pipeline/vr4mice/analysis/np_sync.py new file mode 100644 index 00000000..045160a7 --- /dev/null +++ b/dj_pipeline/vr4mice/analysis/np_sync.py @@ -0,0 +1,73 @@ +"""Fit a VR-time-to-NP-time alignment from barcode values shared by both streams.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import scipy.interpolate +import scipy.stats + +# Number of leading (chronologically earliest) VR barcode events excluded from the +# regression fit by default. DLC-live starts receiving data slightly after the +# Unity/game stream does, which is why downstream analysis already drops +# `trial == 1` as a DLC-live initialization trial (see vr4mice/analysis/analysis.py). +# Barcodes in that same early window can carry an unreliable onset_time_unity, +# biasing the fit if used as tie points, so we drop the earliest few by default. +DEFAULT_SKIP_FIRST_N_BARCODES = 10 + + +@dataclass(frozen=True) +class BarcodeAlignmentFit: + """Linear fit + interpolator mapping VR time to NP time.""" + + slope: float + intercept: float + r2: float + interpol_func: scipy.interpolate.interp1d + shared_barcodes: np.ndarray + + +def align_barcodes( + vr_times: np.ndarray, + vr_values: np.ndarray, + np_times: np.ndarray, + np_values: np.ndarray, + skip_first_n_barcodes: int = 0, +) -> BarcodeAlignmentFit: + """Fit VR time -> NP time from barcode values shared between both streams. + + Args: + vr_times: VR-side barcode onset times, ordered by event index (chronological). + vr_values: VR-side barcode integer payloads, same order as `vr_times`. + np_times: NP-side barcode onset times, ordered by event index. + np_values: NP-side barcode integer payloads, same order as `np_times`. + skip_first_n_barcodes: number of leading (earliest) VR events to exclude + before matching, to avoid the DLC-live startup-lag window. + """ + if skip_first_n_barcodes: + vr_times = vr_times[skip_first_n_barcodes:] + vr_values = vr_values[skip_first_n_barcodes:] + + shared_barcodes, vr_index, np_index = np.intersect1d( + vr_values, np_values, return_indices=True + ) + + vr_shared_times = np.asarray(vr_times)[vr_index] + np_shared_times = np.asarray(np_times)[np_index] + + linreg = scipy.stats.linregress(vr_shared_times, np_shared_times) + interpol_func = scipy.interpolate.interp1d( + vr_shared_times, + np_shared_times, + bounds_error=False, + fill_value="extrapolate", + ) + + return BarcodeAlignmentFit( + slope=linreg.slope, + intercept=linreg.intercept, + r2=linreg.rvalue**2, + interpol_func=interpol_func, + shared_barcodes=shared_barcodes, + ) diff --git a/tests/unit/test_np_sync.py b/tests/unit/test_np_sync.py new file mode 100644 index 00000000..fa6cf48e --- /dev/null +++ b/tests/unit/test_np_sync.py @@ -0,0 +1,74 @@ +"""Unit tests for the VR-to-NP barcode alignment fit.""" + +import numpy as np +import pytest + +from np_sync import align_barcodes + + +def _linear_barcode_streams(*, n=20, slope=2.0, intercept=100.0, skip_vr=0, skip_np=0): + """Two barcode streams sharing all-but-`skip_*` values, related by a known linear fit.""" + values = np.arange(n) + vr_times = np.arange(n, dtype=np.float64) + np_times = slope * vr_times + intercept + + vr_values = values[skip_vr:] if skip_vr else values + vr_times = vr_times[skip_vr:] if skip_vr else vr_times + np_values = values[skip_np:] if skip_np else values + np_times = np_times[skip_np:] if skip_np else np_times + + return vr_times, vr_values, np_times, np_values + + +def test_align_barcodes_recovers_known_linear_fit(): + vr_times, vr_values, np_times, np_values = _linear_barcode_streams( + slope=2.0, intercept=100.0 + ) + + fit = align_barcodes(vr_times, vr_values, np_times, np_values) + + assert fit.slope == pytest.approx(2.0) + assert fit.intercept == pytest.approx(100.0) + assert fit.r2 == pytest.approx(1.0) + assert len(fit.shared_barcodes) == 20 + + +def test_align_barcodes_uses_only_shared_barcode_values(): + # VR side is missing the first 3 barcode values, NP side the last 3. + vr_times, vr_values, np_times, np_values = _linear_barcode_streams( + n=20, skip_vr=3 + ) + np_values = np_values[:-3] + np_times = np_times[:-3] + + fit = align_barcodes(vr_times, vr_values, np_times, np_values) + + assert len(fit.shared_barcodes) == 14 # 20 - 3 leading - 3 trailing + assert fit.slope == pytest.approx(2.0) + + +def test_align_barcodes_skip_first_n_excludes_leading_vr_events(): + vr_times, vr_values, np_times, np_values = _linear_barcode_streams(n=20) + # Corrupt the first two VR onset times so an unskipped fit would be biased. + vr_times = vr_times.copy() + vr_times[:2] += 1000.0 + + biased_fit = align_barcodes(vr_times, vr_values, np_times, np_values) + assert biased_fit.slope != pytest.approx(2.0) + + corrected_fit = align_barcodes( + vr_times, vr_values, np_times, np_values, skip_first_n_barcodes=2 + ) + assert corrected_fit.slope == pytest.approx(2.0) + assert corrected_fit.intercept == pytest.approx(100.0) + assert len(corrected_fit.shared_barcodes) == 18 + + +def test_align_barcodes_interpol_func_maps_vr_time_to_np_time(): + vr_times, vr_values, np_times, np_values = _linear_barcode_streams( + slope=2.0, intercept=100.0 + ) + + fit = align_barcodes(vr_times, vr_values, np_times, np_values) + + assert fit.interpol_func(5.0) == pytest.approx(110.0) From 9842def1c1a35de066d1729a866ef5b101a54ffe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 15:22:04 +0200 Subject: [PATCH 09/40] Add BarcodeSync class for aligning VR time to NP time using shared barcodes --- dj_pipeline/vr4mice/schema/np_sync.py | 122 ++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 dj_pipeline/vr4mice/schema/np_sync.py diff --git a/dj_pipeline/vr4mice/schema/np_sync.py b/dj_pipeline/vr4mice/schema/np_sync.py new file mode 100644 index 00000000..c38acd44 --- /dev/null +++ b/dj_pipeline/vr4mice/schema/np_sync.py @@ -0,0 +1,122 @@ +"""Alignment of VR (behavior) time to Neuropixels (NP) native time via shared barcodes. + +Not every VR ``Dataset`` has a corresponding NP recording; ``BarcodeSync.key_source`` +intersects with the NP-side linkage tables so ``populate()`` simply never calls +``make()`` for behavior-only sessions, instead of raising. +""" + +import pickle + +import datajoint as dj +import numpy as np + +try: + from np_pipeline.schemas import acquisition, barcodes as np_barcodes, session_link +except ModuleNotFoundError: + import sys + + sys.path.insert(0, "/np_pipeline") + from np_pipeline.schemas import acquisition, barcodes as np_barcodes, session_link + +from vr4mice.analysis.np_sync import DEFAULT_SKIP_FIRST_N_BARCODES, align_barcodes +from vr4mice.schema import base +from vr4mice.schema import barcodes as vr_barcodes +from vr4mice.schema import vr4mice +from vr4mice.utils.logger import Logger +from vr4mice.utils.schema_config import get_schema + +schema_name = "np_sync" +schema = get_schema(schema_name, locals()) + +logger = Logger.get_logger() + + +@schema +class BarcodeSync(dj.Computed): + """Linear fit + interpolator mapping VR Unity/game time to NP OneBox DAQ time.""" + + definition = """ + -> base.Base + -> session_link.RecordingSessionLink + -> acquisition.OneBoxDaq + --- + skip_first_n_barcodes: smallint unsigned # leading VR barcode events excluded from the fit + slope: float # Slope of the linear fit mapping VR time to NP time + intercept: float # Intercept of the linear fit mapping VR time to NP time + r2: float # R-squared value of the linear fit + interpol_func: # pickled scipy.interpolate.interp1d, VR time -> NP time + barcode_overlap: float # Fraction of NP barcodes also found on the VR side + """ + + key_source = ( + base.Base * session_link.RecordingSessionLink * acquisition.OneBoxDaq + & (vr_barcodes.TeensyBarcodes & 'extraction_status = "success"') + & (np_barcodes.OneBoxBarcodeExtraction & 'extraction_status = "success"') + ) + + skip_first_n_barcodes = DEFAULT_SKIP_FIRST_N_BARCODES + + def make(self, key): + """Fit a VR-time-to-NP-time alignment from shared barcode events.""" + if vr4mice.FailedSession.should_skip(key, self.__class__.__name__, logger): + return + + try: + vr_values, vr_times = (vr_barcodes.TeensyBarcodes.Event & key).fetch( + "barcode_value", "onset_time_unity", order_by="barcode_index" + ) + np_values, np_times = (np_barcodes.OneBoxBarcodeExtraction.Event & key).fetch( + "barcode_value", "onset_time", order_by="barcode_index" + ) + + fit = align_barcodes( + vr_times, + vr_values, + np_times, + np_values, + skip_first_n_barcodes=self.skip_first_n_barcodes, + ) + + self.insert1( + { + **key, + "skip_first_n_barcodes": self.skip_first_n_barcodes, + "slope": fit.slope, + "intercept": fit.intercept, + "r2": fit.r2, + "interpol_func": pickle.dumps(fit.interpol_func), + "barcode_overlap": len(fit.shared_barcodes) / len(np_values), + } + ) + logger.info( + "%s aligned %d shared barcodes for %s", + self.__class__.__name__, + len(fit.shared_barcodes), + key, + ) + + except Exception as err: + dataset = key["dataset"] + vr4mice.FailedSession().add_entry( + f"{dataset}", f"{self.__class__.__name__}", str(err) + ) + logger.warning( + "Can't populate %s, key: %s. Error: %s.", + self.__class__.__name__, + key, + err, + ) + return None + + @classmethod + def align_timepoints(cls, sess_key, timepoints: list): + """Convert VR times to NP times using interpolation (accounts for clock drift).""" + interpol_func = pickle.loads((cls & sess_key).fetch1("interpol_func")) + timepoints = np.array(timepoints, dtype=np.float64) + return [float(tx) if not np.isnan(tx) else None for tx in interpol_func(timepoints)] + + @classmethod + def align_timepoints_lin(cls, sess_key, timepoints: list): + """Convert VR times to NP times using the linear fit only.""" + slope, intercept = (cls & sess_key).fetch1("slope", "intercept") + return [tx * slope + intercept if tx is not None else None for tx in timepoints] From d97930743524f3a999bffbc0bdd90877e1e5f215 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 15:18:56 +0200 Subject: [PATCH 10/40] Add np_sync module for VR-to-NP barcode alignment and corresponding unit tests --- dj_pipeline/.env.compose.example | 3 + dj_pipeline/docker-compose.yml | 3 +- dj_pipeline/vr4mice/analysis/np_sync.py | 73 ++++++++++++++++++++++++ tests/unit/test_np_sync.py | 74 +++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 dj_pipeline/vr4mice/analysis/np_sync.py create mode 100644 tests/unit/test_np_sync.py diff --git a/dj_pipeline/.env.compose.example b/dj_pipeline/.env.compose.example index 381ca147..2def8f90 100644 --- a/dj_pipeline/.env.compose.example +++ b/dj_pipeline/.env.compose.example @@ -11,6 +11,9 @@ DB_DATA_PATH=/mnt/database/vr4mice/vr4mice_database/database/ SHARED_PATH=/mnt/database/shared DATA_PATH=/mnt/database/vr4mice/vr4mice_database/data SCREEN_RECORDINGS_PATH=/mnt/neuropixel_data/vr4mice/raw_screen_recordings +# Path to the np_pipeline "src" checkout (must contain a np_pipeline/ package dir), +# mounted so vr4mice/schema/np_sync.py can import it. Defaults to a sibling checkout. +NP_PIPELINE_SRC_PATH=../FreelyMoving_Mouse_NP/src JUPYTER_PORT=8887 CLIENT_IMAGE=mmathislab/vr4mice_app:0.1.0 CLIENT_CONTAINER_NAME=vr4mice_${USER} diff --git a/dj_pipeline/docker-compose.yml b/dj_pipeline/docker-compose.yml index 8323340f..5c78504b 100644 --- a/dj_pipeline/docker-compose.yml +++ b/dj_pipeline/docker-compose.yml @@ -36,7 +36,8 @@ services: - "${SCREEN_RECORDINGS_PATH:-/mnt/neuropixel_data/vr4mice/raw_screen_recordings}:/vr4mice_screen_recordings" - "./base/base_min_schemas:/base_schemas" - "./base/base_actions:/base_actions" - ports: + - "${NP_PIPELINE_SRC_PATH:-../FreelyMoving_Mouse_NP/src}:/np_pipeline" + ports: - "${JUPYTER_PORT:-8887}:8888" working_dir: /app env_file: diff --git a/dj_pipeline/vr4mice/analysis/np_sync.py b/dj_pipeline/vr4mice/analysis/np_sync.py new file mode 100644 index 00000000..045160a7 --- /dev/null +++ b/dj_pipeline/vr4mice/analysis/np_sync.py @@ -0,0 +1,73 @@ +"""Fit a VR-time-to-NP-time alignment from barcode values shared by both streams.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import scipy.interpolate +import scipy.stats + +# Number of leading (chronologically earliest) VR barcode events excluded from the +# regression fit by default. DLC-live starts receiving data slightly after the +# Unity/game stream does, which is why downstream analysis already drops +# `trial == 1` as a DLC-live initialization trial (see vr4mice/analysis/analysis.py). +# Barcodes in that same early window can carry an unreliable onset_time_unity, +# biasing the fit if used as tie points, so we drop the earliest few by default. +DEFAULT_SKIP_FIRST_N_BARCODES = 10 + + +@dataclass(frozen=True) +class BarcodeAlignmentFit: + """Linear fit + interpolator mapping VR time to NP time.""" + + slope: float + intercept: float + r2: float + interpol_func: scipy.interpolate.interp1d + shared_barcodes: np.ndarray + + +def align_barcodes( + vr_times: np.ndarray, + vr_values: np.ndarray, + np_times: np.ndarray, + np_values: np.ndarray, + skip_first_n_barcodes: int = 0, +) -> BarcodeAlignmentFit: + """Fit VR time -> NP time from barcode values shared between both streams. + + Args: + vr_times: VR-side barcode onset times, ordered by event index (chronological). + vr_values: VR-side barcode integer payloads, same order as `vr_times`. + np_times: NP-side barcode onset times, ordered by event index. + np_values: NP-side barcode integer payloads, same order as `np_times`. + skip_first_n_barcodes: number of leading (earliest) VR events to exclude + before matching, to avoid the DLC-live startup-lag window. + """ + if skip_first_n_barcodes: + vr_times = vr_times[skip_first_n_barcodes:] + vr_values = vr_values[skip_first_n_barcodes:] + + shared_barcodes, vr_index, np_index = np.intersect1d( + vr_values, np_values, return_indices=True + ) + + vr_shared_times = np.asarray(vr_times)[vr_index] + np_shared_times = np.asarray(np_times)[np_index] + + linreg = scipy.stats.linregress(vr_shared_times, np_shared_times) + interpol_func = scipy.interpolate.interp1d( + vr_shared_times, + np_shared_times, + bounds_error=False, + fill_value="extrapolate", + ) + + return BarcodeAlignmentFit( + slope=linreg.slope, + intercept=linreg.intercept, + r2=linreg.rvalue**2, + interpol_func=interpol_func, + shared_barcodes=shared_barcodes, + ) diff --git a/tests/unit/test_np_sync.py b/tests/unit/test_np_sync.py new file mode 100644 index 00000000..fa6cf48e --- /dev/null +++ b/tests/unit/test_np_sync.py @@ -0,0 +1,74 @@ +"""Unit tests for the VR-to-NP barcode alignment fit.""" + +import numpy as np +import pytest + +from np_sync import align_barcodes + + +def _linear_barcode_streams(*, n=20, slope=2.0, intercept=100.0, skip_vr=0, skip_np=0): + """Two barcode streams sharing all-but-`skip_*` values, related by a known linear fit.""" + values = np.arange(n) + vr_times = np.arange(n, dtype=np.float64) + np_times = slope * vr_times + intercept + + vr_values = values[skip_vr:] if skip_vr else values + vr_times = vr_times[skip_vr:] if skip_vr else vr_times + np_values = values[skip_np:] if skip_np else values + np_times = np_times[skip_np:] if skip_np else np_times + + return vr_times, vr_values, np_times, np_values + + +def test_align_barcodes_recovers_known_linear_fit(): + vr_times, vr_values, np_times, np_values = _linear_barcode_streams( + slope=2.0, intercept=100.0 + ) + + fit = align_barcodes(vr_times, vr_values, np_times, np_values) + + assert fit.slope == pytest.approx(2.0) + assert fit.intercept == pytest.approx(100.0) + assert fit.r2 == pytest.approx(1.0) + assert len(fit.shared_barcodes) == 20 + + +def test_align_barcodes_uses_only_shared_barcode_values(): + # VR side is missing the first 3 barcode values, NP side the last 3. + vr_times, vr_values, np_times, np_values = _linear_barcode_streams( + n=20, skip_vr=3 + ) + np_values = np_values[:-3] + np_times = np_times[:-3] + + fit = align_barcodes(vr_times, vr_values, np_times, np_values) + + assert len(fit.shared_barcodes) == 14 # 20 - 3 leading - 3 trailing + assert fit.slope == pytest.approx(2.0) + + +def test_align_barcodes_skip_first_n_excludes_leading_vr_events(): + vr_times, vr_values, np_times, np_values = _linear_barcode_streams(n=20) + # Corrupt the first two VR onset times so an unskipped fit would be biased. + vr_times = vr_times.copy() + vr_times[:2] += 1000.0 + + biased_fit = align_barcodes(vr_times, vr_values, np_times, np_values) + assert biased_fit.slope != pytest.approx(2.0) + + corrected_fit = align_barcodes( + vr_times, vr_values, np_times, np_values, skip_first_n_barcodes=2 + ) + assert corrected_fit.slope == pytest.approx(2.0) + assert corrected_fit.intercept == pytest.approx(100.0) + assert len(corrected_fit.shared_barcodes) == 18 + + +def test_align_barcodes_interpol_func_maps_vr_time_to_np_time(): + vr_times, vr_values, np_times, np_values = _linear_barcode_streams( + slope=2.0, intercept=100.0 + ) + + fit = align_barcodes(vr_times, vr_values, np_times, np_values) + + assert fit.interpol_func(5.0) == pytest.approx(110.0) From 1ca4adc3d3e60179978123530010455588e1e511 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 15:37:04 +0200 Subject: [PATCH 11/40] Add decision restriction for local execution of batch1 outputs --- dj_pipeline/cron_scenario.py | 39 ++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/dj_pipeline/cron_scenario.py b/dj_pipeline/cron_scenario.py index 0bf76fd7..2a63b28a 100755 --- a/dj_pipeline/cron_scenario.py +++ b/dj_pipeline/cron_scenario.py @@ -289,30 +289,57 @@ def import_decision_schema(): "decision.sync_lookup_contents", decision.sync_lookup_contents, ) + + decision_restriction = None + if not args.aws: + from vr4mice.schema import vr4mice as vr4mice_schema + + # Local server: do not recompute batch1 decision outputs locally; + # batch1 decision results are produced on AWS, as it needs to be + # combined to other sessions from the other labs. + # batch2 decision outputs are produced locally, as they are only + # for local sessions. + decision_restriction = ( + vr4mice_schema.DatasetBatch & 'batch_name != "batch1"' + ) + run_step( "decision.ExperimentMember.populate", - lambda: decision.ExperimentMember().populate(), + lambda: decision.ExperimentMember().populate(decision_restriction) + if decision_restriction is not None + else decision.ExperimentMember().populate(), ) run_step( "decision.InclusionStatus.populate", - lambda: decision.InclusionStatus().populate(), + lambda: decision.InclusionStatus().populate(decision_restriction) + if decision_restriction is not None + else decision.InclusionStatus().populate(), ) run_step("decision.LabelSet.fill", lambda: decision.LabelSet.fill()) + run_step( "decision.PredictionModel.populate", - lambda: decision.PredictionModel().populate(), + lambda: decision.PredictionModel().populate(decision_restriction) + if decision_restriction is not None + else decision.PredictionModel().populate(), ) run_step( "decision.DecisionPoints.populate", - lambda: decision.DecisionPoints().populate(), + lambda: decision.DecisionPoints().populate(decision_restriction) + if decision_restriction is not None + else decision.DecisionPoints().populate(), ) run_step( "decision.PredictionModel10Windows.populate", - lambda: decision.PredictionModel10Windows().populate(), + lambda: decision.PredictionModel10Windows().populate(decision_restriction) + if decision_restriction is not None + else decision.PredictionModel10Windows().populate(), ) run_step( "decision.DecisionPoints10Windows.populate", - lambda: decision.DecisionPoints10Windows().populate(), + lambda: decision.DecisionPoints10Windows().populate(decision_restriction) + if decision_restriction is not None + else decision.DecisionPoints10Windows().populate(), ) if not args.aws: From 25590e91afdd2038205f8fd0be4023b881666787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 15:39:39 +0200 Subject: [PATCH 12/40] Run black --- dj_pipeline/cron_scenario.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dj_pipeline/cron_scenario.py b/dj_pipeline/cron_scenario.py index 2a63b28a..500eaea8 100755 --- a/dj_pipeline/cron_scenario.py +++ b/dj_pipeline/cron_scenario.py @@ -297,7 +297,7 @@ def import_decision_schema(): # Local server: do not recompute batch1 decision outputs locally; # batch1 decision results are produced on AWS, as it needs to be # combined to other sessions from the other labs. - # batch2 decision outputs are produced locally, as they are only + # batch2 decision outputs are produced locally, as they are only # for local sessions. decision_restriction = ( vr4mice_schema.DatasetBatch & 'batch_name != "batch1"' From 895e66e3536b42ea109c905596de223063152fb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 15:37:04 +0200 Subject: [PATCH 13/40] Add decision restriction for local execution of batch1 outputs --- dj_pipeline/cron_scenario.py | 39 ++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/dj_pipeline/cron_scenario.py b/dj_pipeline/cron_scenario.py index 0bf76fd7..2a63b28a 100755 --- a/dj_pipeline/cron_scenario.py +++ b/dj_pipeline/cron_scenario.py @@ -289,30 +289,57 @@ def import_decision_schema(): "decision.sync_lookup_contents", decision.sync_lookup_contents, ) + + decision_restriction = None + if not args.aws: + from vr4mice.schema import vr4mice as vr4mice_schema + + # Local server: do not recompute batch1 decision outputs locally; + # batch1 decision results are produced on AWS, as it needs to be + # combined to other sessions from the other labs. + # batch2 decision outputs are produced locally, as they are only + # for local sessions. + decision_restriction = ( + vr4mice_schema.DatasetBatch & 'batch_name != "batch1"' + ) + run_step( "decision.ExperimentMember.populate", - lambda: decision.ExperimentMember().populate(), + lambda: decision.ExperimentMember().populate(decision_restriction) + if decision_restriction is not None + else decision.ExperimentMember().populate(), ) run_step( "decision.InclusionStatus.populate", - lambda: decision.InclusionStatus().populate(), + lambda: decision.InclusionStatus().populate(decision_restriction) + if decision_restriction is not None + else decision.InclusionStatus().populate(), ) run_step("decision.LabelSet.fill", lambda: decision.LabelSet.fill()) + run_step( "decision.PredictionModel.populate", - lambda: decision.PredictionModel().populate(), + lambda: decision.PredictionModel().populate(decision_restriction) + if decision_restriction is not None + else decision.PredictionModel().populate(), ) run_step( "decision.DecisionPoints.populate", - lambda: decision.DecisionPoints().populate(), + lambda: decision.DecisionPoints().populate(decision_restriction) + if decision_restriction is not None + else decision.DecisionPoints().populate(), ) run_step( "decision.PredictionModel10Windows.populate", - lambda: decision.PredictionModel10Windows().populate(), + lambda: decision.PredictionModel10Windows().populate(decision_restriction) + if decision_restriction is not None + else decision.PredictionModel10Windows().populate(), ) run_step( "decision.DecisionPoints10Windows.populate", - lambda: decision.DecisionPoints10Windows().populate(), + lambda: decision.DecisionPoints10Windows().populate(decision_restriction) + if decision_restriction is not None + else decision.DecisionPoints10Windows().populate(), ) if not args.aws: From 25ba627c3a5abbdbaa690c269fa053142803e498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 15:39:39 +0200 Subject: [PATCH 14/40] Run black --- dj_pipeline/cron_scenario.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dj_pipeline/cron_scenario.py b/dj_pipeline/cron_scenario.py index 2a63b28a..500eaea8 100755 --- a/dj_pipeline/cron_scenario.py +++ b/dj_pipeline/cron_scenario.py @@ -297,7 +297,7 @@ def import_decision_schema(): # Local server: do not recompute batch1 decision outputs locally; # batch1 decision results are produced on AWS, as it needs to be # combined to other sessions from the other labs. - # batch2 decision outputs are produced locally, as they are only + # batch2 decision outputs are produced locally, as they are only # for local sessions. decision_restriction = ( vr4mice_schema.DatasetBatch & 'batch_name != "batch1"' From 65bba288519c8fe7ba078f60938efba050c81a29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 16:01:08 +0200 Subject: [PATCH 15/40] Enhance documentation for BarcodeSync and align_barcodes functions to clarify usage and behavior --- dj_pipeline/vr4mice/analysis/np_sync.py | 5 ++ dj_pipeline/vr4mice/schema/np_sync.py | 61 +++++++++++++++++++++---- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/dj_pipeline/vr4mice/analysis/np_sync.py b/dj_pipeline/vr4mice/analysis/np_sync.py index 045160a7..ee783a9d 100644 --- a/dj_pipeline/vr4mice/analysis/np_sync.py +++ b/dj_pipeline/vr4mice/analysis/np_sync.py @@ -37,6 +37,11 @@ def align_barcodes( ) -> BarcodeAlignmentFit: """Fit VR time -> NP time from barcode values shared between both streams. + Alignment approach (intersect1d + linregress + interp1d) ported from + cross_analysis_schemas/schemas/vr_np_sync.py::BarcodeSync.align_barcodes in + https://github.com/AdaptiveMotorControlLab/auxPipelines-DataJoint_Mathis, + adapted for this repo's VR (vr4mice) / NP (np_pipeline) schemas. + Args: vr_times: VR-side barcode onset times, ordered by event index (chronological). vr_values: VR-side barcode integer payloads, same order as `vr_times`. diff --git a/dj_pipeline/vr4mice/schema/np_sync.py b/dj_pipeline/vr4mice/schema/np_sync.py index c38acd44..3919d758 100644 --- a/dj_pipeline/vr4mice/schema/np_sync.py +++ b/dj_pipeline/vr4mice/schema/np_sync.py @@ -33,7 +33,16 @@ @schema class BarcodeSync(dj.Computed): - """Linear fit + interpolator mapping VR Unity/game time to NP OneBox DAQ time.""" + """Linear fit + interpolator mapping VR Unity/game time to NP OneBox DAQ time. + + One row per (dataset, recording, OneBox DAQ stream) triple that has a linked NP + recording and successfully decoded barcodes on both sides. Populated only for + those keys — see ``key_source`` — so ``populate()`` is a no-op, not an error, for + VR-only sessions with no matching neural recording. + + Downstream code should not fetch ``slope``/``intercept``/``interpol_func`` + directly; use ``align_timepoints``/``align_timepoints_lin`` instead. + """ definition = """ -> base.Base @@ -57,7 +66,16 @@ class BarcodeSync(dj.Computed): skip_first_n_barcodes = DEFAULT_SKIP_FIRST_N_BARCODES def make(self, key): - """Fit a VR-time-to-NP-time alignment from shared barcode events.""" + """Fit and insert one VR-time-to-NP-time alignment. + + Fetches decoded barcode events for one dataset/recording/DAQ key from both + `vr_barcodes.TeensyBarcodes.Event` (VR side) and + `np_barcodes.OneBoxBarcodeExtraction.Event` (NP side), fits the alignment via + `vr4mice.analysis.np_sync.align_barcodes`, and inserts the resulting fit + parameters and pickled interpolator. On failure, records the error in + `vr4mice.FailedSession` and logs a warning instead of raising, matching the + error-handling convention used by `barcodes.TeensyTTL`/`TeensyBarcodes`. + """ if vr4mice.FailedSession.should_skip(key, self.__class__.__name__, logger): return @@ -109,14 +127,41 @@ def make(self, key): return None @classmethod - def align_timepoints(cls, sess_key, timepoints: list): - """Convert VR times to NP times using interpolation (accounts for clock drift).""" - interpol_func = pickle.loads((cls & sess_key).fetch1("interpol_func")) + def align_timepoints(cls, key, timepoints: list): + """Convert a list of VR times to NP times via the fitted interpolator. + + Preferred over `align_timepoints_lin` for most uses: the interpolator is fit + only within the range spanned by the shared barcode events, so it captures + any small clock drift between the VR and NP streams rather than assuming a + perfectly constant offset/rate. + + Args: + key: A restriction identifying exactly one `BarcodeSync` row (e.g. a + dataset/recording/DAQ key). + timepoints: VR-side times (`onset_time_unity`-style values) to convert. + `None` entries pass through as `None`; DataJoint-`NULL`/`NaN`-producing + extrapolation misses become `None` as well. + + Returns: + A list of NP-side times (or `None`), same length and order as `timepoints`. + """ + interpol_func = pickle.loads((cls & key).fetch1("interpol_func")) timepoints = np.array(timepoints, dtype=np.float64) return [float(tx) if not np.isnan(tx) else None for tx in interpol_func(timepoints)] @classmethod - def align_timepoints_lin(cls, sess_key, timepoints: list): - """Convert VR times to NP times using the linear fit only.""" - slope, intercept = (cls & sess_key).fetch1("slope", "intercept") + def align_timepoints_lin(cls, key, timepoints: list): + """Convert a list of VR times to NP times via the fitted line only (`y = slope*x + intercept`). + + Faster than `align_timepoints` and fine for a single global rate/offset, but + ignores any local clock drift the interpolator would otherwise correct for. + + Args: + key: A restriction identifying exactly one `BarcodeSync` row. + timepoints: VR-side times to convert; `None` entries pass through as `None`. + + Returns: + A list of NP-side times (or `None`), same length and order as `timepoints`. + """ + slope, intercept = (cls & key).fetch1("slope", "intercept") return [tx * slope + intercept if tx is not None else None for tx in timepoints] From f9dcc47f07220e9aec45c88d162aa0397eb0da81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 16:04:11 +0200 Subject: [PATCH 16/40] Update datajoint docs --- docs/software/datajoint.md | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/software/datajoint.md b/docs/software/datajoint.md index 9874a45e..fe675c91 100644 --- a/docs/software/datajoint.md +++ b/docs/software/datajoint.md @@ -396,4 +396,35 @@ Extracts the binary sync trace from the sync ROI video. class AlignedVideoFrame(dj.Computed) ``` **Depends on:** `VideoSyncSignal`, `vr4mice.State` -Aligns game steps to video frames using photodiode when available; stores QA metrics. \ No newline at end of file +Aligns game steps to video frames using photodiode when available; stores QA metrics. + +### `barcodes.py` + +*Schema for barcodes recorded through the Teensy synchronization channel.* + +```python +class TeensyTTL(dj.Imported) +``` +**Depends on:** `vr4mice.DLC` +Raw Teensy barcode channel imported from a DLC PROC file. + +```python +class TeensyBarcodes(dj.Computed) +``` +**Depends on:** `TeensyTTL`, `dlc.DLCProcessor`, `base_analysis.DataFrame` +Barcode events decoded from the TTL signal sampled by the Teensy. + +### `np_sync.py` + +*Alignment of VR (behavior) time to Neuropixels (NP) native time via shared barcodes.* + +```python +class BarcodeSync(dj.Computed) +``` +**Depends on:** `base.Base`, `np_pipeline.schemas.session_link.RecordingSessionLink`, `np_pipeline.schemas.acquisition.OneBoxDaq` +Fits a linear regression + interpolator mapping VR Unity/game time to NP OneBox DAQ +time, from barcode values shared between `barcodes.TeensyBarcodes.Event` and +`np_pipeline`'s `OneBoxBarcodeExtraction.Event`. `key_source` is restricted to +datasets with a linked, successfully barcode-decoded NP recording, so `populate()` +is a no-op — not an error — for VR-only sessions with no neural data. Use +`align_timepoints`/`align_timepoints_lin` to convert VR times to NP times. \ No newline at end of file From 98b62ddb026412032d437c31a067d4fd2613f1e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 16:37:49 +0200 Subject: [PATCH 17/40] Wire np_sync into run.py/cron_scenario.py as an isolated, optional step --- dj_pipeline/.env.compose.example | 4 ++-- dj_pipeline/.env.example | 1 + dj_pipeline/cron_scenario.py | 22 ++++++++++++++++++++ dj_pipeline/run.py | 29 ++++++++++++++++++++++++++- dj_pipeline/vr4mice/schema/np_sync.py | 7 +++++++ docs/software/datajoint.md | 7 ++++++- 6 files changed, 66 insertions(+), 4 deletions(-) diff --git a/dj_pipeline/.env.compose.example b/dj_pipeline/.env.compose.example index 2def8f90..b26ac024 100644 --- a/dj_pipeline/.env.compose.example +++ b/dj_pipeline/.env.compose.example @@ -11,8 +11,8 @@ DB_DATA_PATH=/mnt/database/vr4mice/vr4mice_database/database/ SHARED_PATH=/mnt/database/shared DATA_PATH=/mnt/database/vr4mice/vr4mice_database/data SCREEN_RECORDINGS_PATH=/mnt/neuropixel_data/vr4mice/raw_screen_recordings -# Path to the np_pipeline "src" checkout (must contain a np_pipeline/ package dir), -# mounted so vr4mice/schema/np_sync.py can import it. Defaults to a sibling checkout. +# Path to the np_pipeline "src" checkout (must contain a np_pipeline/ package dir). +# Optional — the "np_sync" mode skips gracefully if unset/not importable. NP_PIPELINE_SRC_PATH=../FreelyMoving_Mouse_NP/src JUPYTER_PORT=8887 CLIENT_IMAGE=mmathislab/vr4mice_app:0.1.0 diff --git a/dj_pipeline/.env.example b/dj_pipeline/.env.example index 6d4da1a1..ef899ec0 100644 --- a/dj_pipeline/.env.example +++ b/dj_pipeline/.env.example @@ -1,5 +1,6 @@ # Pipeline / DataJoint credentials (loaded into the client container via docker-compose env_file). # DJ_HOST port must match DB_PORT in .env.compose when using the local Docker database. +# For the "np_sync" mode: DJ_HOST must match np_pipeline's DJ_HOST (same MySQL server). DJ_HOST=your-db-hostname-or-ip:3306 DJ_USER=your-db-user DJ_PWD=your-db-password diff --git a/dj_pipeline/cron_scenario.py b/dj_pipeline/cron_scenario.py index 500eaea8..734ee16f 100755 --- a/dj_pipeline/cron_scenario.py +++ b/dj_pipeline/cron_scenario.py @@ -278,6 +278,28 @@ def import_core_schemas(): lambda: summary_emails.send_pending_summary_emails(logger=logger), ) + def import_np_sync_schema(): + # np_sync depends on the separate np_pipeline package/repo, unlike the + # core_schemas above. Imported in its own run_import call (not bundled + # into import_core_schemas) so a missing/unreachable NP repo only skips + # this one step, without nulling out the rest of the behavioral pipeline. + from vr4mice.schema import np_sync + + return np_sync + + np_sync = run_import("import np_sync schema", import_np_sync_schema) + if np_sync: + from vr4mice.utils.populate_helpers import populate_pending + + run_step( + "np_sync.BarcodeSync.populate", + lambda: populate_pending( + np_sync.BarcodeSync, + np_sync.BarcodeSync.key_source, + logger=logger, + ), + ) + def import_decision_schema(): from vr4mice.schema import decision diff --git a/dj_pipeline/run.py b/dj_pipeline/run.py index 06c613cf..41f53f92 100644 --- a/dj_pipeline/run.py +++ b/dj_pipeline/run.py @@ -25,6 +25,9 @@ "latency": compute latencies based on photodiode signals "inputs_videos": process input videos and extract frames "decision": analyze decision-making metrics + "np_sync": align VR time to Neuropixels time via barcodes (requires the + np_pipeline package/repo; skips gracefully, without affecting any other + mode, if it isn't installed or no dataset has a linked NP recording yet) "maintenance": rebuild DataJoint lineage tables (one-time setup) """ @@ -77,9 +80,10 @@ def check_folder_existence(folder_path): "sync_days", "inputs_videos", "decision", + "np_sync", "maintenance", ], - help="Mode to execute: 'connect', 'populate', 'summary', 'dlc', 'fetch', 'sync_days', 'analysis', 'inputs_videos', 'decision', 'maintenance'", + help="Mode to execute: 'connect', 'populate', 'summary', 'dlc', 'fetch', 'sync_days', 'analysis', 'inputs_videos', 'decision', 'np_sync', 'maintenance'", ) args = parser.parse_args() @@ -270,6 +274,29 @@ def check_folder_existence(folder_path): decision.PredictionModel10Windows().populate() decision.DecisionPoints10Windows().populate() + elif args.mode == "np_sync": + # np_sync depends on the separate np_pipeline package/repo, which may be + # absent (ModuleNotFoundError) or unreachable (e.g. its own DB connection + # fails while its schema modules activate). Either way, isolate the + # failure to this mode so every other (purely behavioral) mode is + # unaffected. + try: + from vr4mice.schema import np_sync + except Exception as err: + logger.warning( + "Skipping np_sync: np_pipeline is not available (%s). " + "Behavioral analysis is unaffected.", + err, + ) + else: + from vr4mice.utils.populate_helpers import populate_pending + + populate_pending( + np_sync.BarcodeSync, + np_sync.BarcodeSync.key_source, + logger=logger, + ) + elif args.mode == "fetch": from vr4mice.actions.fetch_data import fetch_data diff --git a/dj_pipeline/vr4mice/schema/np_sync.py b/dj_pipeline/vr4mice/schema/np_sync.py index 3919d758..4e0f18c5 100644 --- a/dj_pipeline/vr4mice/schema/np_sync.py +++ b/dj_pipeline/vr4mice/schema/np_sync.py @@ -3,6 +3,13 @@ Not every VR ``Dataset`` has a corresponding NP recording; ``BarcodeSync.key_source`` intersects with the NP-side linkage tables so ``populate()`` simply never calls ``make()`` for behavior-only sessions, instead of raising. + +Cross-repo foreign keys: ``BarcodeSync`` references ``np_pipeline`` tables +directly via ``-> ``. This requires ``vr4mice`` and ``np_pipeline`` to share one +MySQL server (``DJ_HOST``) and one process-global ``dj.conn()`` — both already +satisfied by this repo's connection setup. `np_sync` is imported as its own +isolated step in ``run.py``/``cron_scenario.py``, so a missing ``np_pipeline`` +only skips NP sync, not the rest of the pipeline. """ import pickle diff --git a/docs/software/datajoint.md b/docs/software/datajoint.md index fe675c91..f7cab513 100644 --- a/docs/software/datajoint.md +++ b/docs/software/datajoint.md @@ -427,4 +427,9 @@ time, from barcode values shared between `barcodes.TeensyBarcodes.Event` and `np_pipeline`'s `OneBoxBarcodeExtraction.Event`. `key_source` is restricted to datasets with a linked, successfully barcode-decoded NP recording, so `populate()` is a no-op — not an error — for VR-only sessions with no neural data. Use -`align_timepoints`/`align_timepoints_lin` to convert VR times to NP times. \ No newline at end of file +`align_timepoints`/`align_timepoints_lin` to convert VR times to NP times. + +> **Cross-repo foreign keys.** `BarcodeSync` references `np_pipeline` tables +> directly via `-> `. Requires `vr4mice` and `np_pipeline` to share one MySQL +> server (`DJ_HOST`). `np_sync` is imported as its own isolated step in +> `run.py`/`cron_scenario.py`, so a missing `np_pipeline` only skips NP sync. \ No newline at end of file From 169ab01d02f87c3397e577138b18d4536e0fd1a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 16:46:49 +0200 Subject: [PATCH 18/40] Refactor np_pipeline source path handling and enhance unit tests for np_sync functionality --- dj_pipeline/.env.compose.example | 4 +-- dj_pipeline/docker-compose.yml | 2 +- tests/unit/test_np_sync.py | 60 +++++++++++++++++++++++++++++++- 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/dj_pipeline/.env.compose.example b/dj_pipeline/.env.compose.example index b26ac024..3e89950d 100644 --- a/dj_pipeline/.env.compose.example +++ b/dj_pipeline/.env.compose.example @@ -12,8 +12,8 @@ SHARED_PATH=/mnt/database/shared DATA_PATH=/mnt/database/vr4mice/vr4mice_database/data SCREEN_RECORDINGS_PATH=/mnt/neuropixel_data/vr4mice/raw_screen_recordings # Path to the np_pipeline "src" checkout (must contain a np_pipeline/ package dir). -# Optional — the "np_sync" mode skips gracefully if unset/not importable. -NP_PIPELINE_SRC_PATH=../FreelyMoving_Mouse_NP/src +# Optional — leave unset to mount an empty stub; the "np_sync" mode skips gracefully. +# NP_PIPELINE_SRC_PATH=../FreelyMoving_Mouse_NP/src JUPYTER_PORT=8887 CLIENT_IMAGE=mmathislab/vr4mice_app:0.1.0 CLIENT_CONTAINER_NAME=vr4mice_${USER} diff --git a/dj_pipeline/docker-compose.yml b/dj_pipeline/docker-compose.yml index 5c78504b..432fd68d 100644 --- a/dj_pipeline/docker-compose.yml +++ b/dj_pipeline/docker-compose.yml @@ -36,7 +36,7 @@ services: - "${SCREEN_RECORDINGS_PATH:-/mnt/neuropixel_data/vr4mice/raw_screen_recordings}:/vr4mice_screen_recordings" - "./base/base_min_schemas:/base_schemas" - "./base/base_actions:/base_actions" - - "${NP_PIPELINE_SRC_PATH:-../FreelyMoving_Mouse_NP/src}:/np_pipeline" + - "${NP_PIPELINE_SRC_PATH:-./np_pipeline_stub}:/np_pipeline" ports: - "${JUPYTER_PORT:-8887}:8888" working_dir: /app diff --git a/tests/unit/test_np_sync.py b/tests/unit/test_np_sync.py index fa6cf48e..a5cb6bf8 100644 --- a/tests/unit/test_np_sync.py +++ b/tests/unit/test_np_sync.py @@ -1,10 +1,20 @@ -"""Unit tests for the VR-to-NP barcode alignment fit.""" +"""Unit tests for the VR-to-NP barcode alignment fit, plus guards that +behavioral analysis stays unaffected when np_pipeline is unavailable. +""" + +import importlib +import sys +from pathlib import Path import numpy as np import pytest from np_sync import align_barcodes +REPO_ROOT = Path(__file__).parent.parent.parent +RUN_PY = REPO_ROOT / "dj_pipeline" / "run.py" +CRON_SCENARIO_PY = REPO_ROOT / "dj_pipeline" / "cron_scenario.py" + def _linear_barcode_streams(*, n=20, slope=2.0, intercept=100.0, skip_vr=0, skip_np=0): """Two barcode streams sharing all-but-`skip_*` values, related by a known linear fit.""" @@ -72,3 +82,51 @@ def test_align_barcodes_interpol_func_maps_vr_time_to_np_time(): fit = align_barcodes(vr_times, vr_values, np_times, np_values) assert fit.interpol_func(5.0) == pytest.approx(110.0) + + +def test_analysis_np_sync_has_no_datajoint_or_np_pipeline_dependency(monkeypatch): + """The pure alignment math must import and run with np_pipeline/datajoint absent.""" + monkeypatch.setitem(sys.modules, "datajoint", None) + monkeypatch.setitem(sys.modules, "np_pipeline", None) + + import np_sync as np_sync_analysis + + importlib.reload(np_sync_analysis) + + vr_times, vr_values, np_times, np_values = _linear_barcode_streams( + slope=2.0, intercept=100.0 + ) + fit = np_sync_analysis.align_barcodes(vr_times, vr_values, np_times, np_values) + assert fit.slope == pytest.approx(2.0) + + +def _function_source(file_text: str, def_line: str) -> str: + """Return the source of a top-level-indented function, up to the next `def `.""" + start = file_text.index(def_line) + rest = file_text[start + len(def_line) :] + end = rest.find("\n def ") + return def_line + (rest if end == -1 else rest[:end]) + + +def test_cron_scenario_core_schemas_import_excludes_np_sync(): + """import_core_schemas() must not import np_sync, so a missing np_pipeline + can't null out the whole behavioral core_schemas tuple (see import_np_sync_schema, + which is deliberately a separate run_import() call).""" + text = CRON_SCENARIO_PY.read_text() + core_schemas_src = _function_source(text, " def import_core_schemas():") + + assert "np_sync" not in core_schemas_src + assert "def import_np_sync_schema():" in text + + +def test_run_py_np_sync_mode_catches_broad_exception(): + """The np_sync CLI mode must not let an import/connection failure crash the + process; it should catch broadly and warn, not just ModuleNotFoundError.""" + text = RUN_PY.read_text() + start = text.index('elif args.mode == "np_sync":') + end = text.index('elif args.mode == "fetch":') + block = text[start:end] + + assert "from vr4mice.schema import np_sync" in block + assert "except Exception as err:" in block + assert "except ModuleNotFoundError" not in block From 80433b6a7f1ce6ed42b54c3e44fe182913340537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 17:26:20 +0200 Subject: [PATCH 19/40] Update cron_scenario --- .github/workflows/cron-scenario.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/cron-scenario.yml b/.github/workflows/cron-scenario.yml index ef29d3af..0cd3727b 100644 --- a/.github/workflows/cron-scenario.yml +++ b/.github/workflows/cron-scenario.yml @@ -237,6 +237,9 @@ jobs: done < "$RUNNER_TEMP/mysql-dump/schemas.txt" dataset_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM vr4mice.dataset;')" + batch_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM vr4mice.`#batch`;')" + dataset_batch_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM vr4mice.__dataset_batch;')" + collab_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM vr4mice.__collab;')" state_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM vr4mice.state;')" dataframe_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM base_analysis.__data_frame;')" summary_plot_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM base_analysis.__summary_plots;')" @@ -249,12 +252,18 @@ jobs: fi echo "vr4mice.dataset rows: ${dataset_count}" + echo "vr4mice.#batch rows: ${batch_count}" + echo "vr4mice.__dataset_batch rows: ${dataset_batch_count}" + echo "vr4mice.__collab rows: ${collab_count}" echo "vr4mice.state rows: ${state_count}" echo "base_analysis.__data_frame rows: ${dataframe_count}" echo "base_analysis.__summary_plots rows: ${summary_plot_count}" echo "summary_emails.summary_plot_email successful rows: ${summary_email_count}" test "${dataset_count}" -ge 1 + test "${batch_count}" -ge 2 + test "${dataset_batch_count}" -ge 1 + test "${collab_count}" -ge 1 test "${state_count}" -ge 1 test "${dataframe_count}" -ge 1 test "${summary_plot_count}" -ge 1 @@ -262,7 +271,11 @@ jobs: - name: Verify decision tables (--aws populated) run: | experiment_member_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision._experiment_member;')" + experiment_member_batch1_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = "batch1";')" + experiment_member_batch2_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = "batch2";')" inclusion_status_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__inclusion_status;')" + inclusion_status_batch1_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = "batch1";')" + inclusion_status_batch2_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = "batch2";')" label_set_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.`#label_set`;')" prediction_model_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__prediction_model;')" decision_points_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__decision_points;')" @@ -270,7 +283,11 @@ jobs: decision_points10_windows_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__decision_points10_windows;')" echo "decision._experiment_member rows: ${experiment_member_count}" + echo "decision._experiment_member batch1 rows: ${experiment_member_batch1_count}" + echo "decision._experiment_member batch2 rows: ${experiment_member_batch2_count}" echo "decision.__inclusion_status rows: ${inclusion_status_count}" + echo "decision.__inclusion_status batch1 rows: ${inclusion_status_batch1_count}" + echo "decision.__inclusion_status batch2 rows: ${inclusion_status_batch2_count}" echo "decision.#label_set rows: ${label_set_count}" echo "decision.__prediction_model rows: ${prediction_model_count}" echo "decision.__decision_points rows: ${decision_points_count}" @@ -278,6 +295,9 @@ jobs: echo "decision.__decision_points10_windows rows: ${decision_points10_windows_count}" test "${experiment_member_count}" -ge 1 + test "${experiment_member_batch1_count}" -ge 1 test "${inclusion_status_count}" -ge 1 + test "${inclusion_status_batch1_count}" -ge 1 test "${label_set_count}" -ge 1 + # batch2 counts are logged for visibility and depend on available batch2 input datasets. # PredictionModel/DecisionPoints and PredictionModel10Windows/DecisionPoints10Windows need multi-session data to train so no test; logged for visibility only From 5100f9cdc9b6f9f4cc9a2df42b54778497aeb381 Mon Sep 17 00:00:00 2001 From: CeliaBenquet Date: Wed, 5 Aug 2026 18:07:20 +0200 Subject: [PATCH 20/40] Add w_ephys test session data via LFS --- .../data/w_ephys/TS_vr4mice_Xestia_2026-07-30_1_CAMERA3.npy | 3 +++ dj_pipeline/tests/data/w_ephys/Xestia_2026-07-30_1.json | 3 +++ dj_pipeline/tests/data/w_ephys/Xestia_2026-07-30_1.npy | 3 +++ dj_pipeline/tests/data/w_ephys/Xestia_2026-07-30_1.pickle | 3 +++ .../tests/data/w_ephys/vr4mice_Xestia_2026-07-30_1_DLC.hdf5 | 3 +++ .../tests/data/w_ephys/vr4mice_Xestia_2026-07-30_1_PROC | 3 +++ 6 files changed, 18 insertions(+) create mode 100644 dj_pipeline/tests/data/w_ephys/TS_vr4mice_Xestia_2026-07-30_1_CAMERA3.npy create mode 100644 dj_pipeline/tests/data/w_ephys/Xestia_2026-07-30_1.json create mode 100644 dj_pipeline/tests/data/w_ephys/Xestia_2026-07-30_1.npy create mode 100644 dj_pipeline/tests/data/w_ephys/Xestia_2026-07-30_1.pickle create mode 100644 dj_pipeline/tests/data/w_ephys/vr4mice_Xestia_2026-07-30_1_DLC.hdf5 create mode 100644 dj_pipeline/tests/data/w_ephys/vr4mice_Xestia_2026-07-30_1_PROC diff --git a/dj_pipeline/tests/data/w_ephys/TS_vr4mice_Xestia_2026-07-30_1_CAMERA3.npy b/dj_pipeline/tests/data/w_ephys/TS_vr4mice_Xestia_2026-07-30_1_CAMERA3.npy new file mode 100644 index 00000000..09928c8a --- /dev/null +++ b/dj_pipeline/tests/data/w_ephys/TS_vr4mice_Xestia_2026-07-30_1_CAMERA3.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:829744b4779114569e69b4a69410b101dab22f1b9760d173d374383a0afe874b +size 2344360 diff --git a/dj_pipeline/tests/data/w_ephys/Xestia_2026-07-30_1.json b/dj_pipeline/tests/data/w_ephys/Xestia_2026-07-30_1.json new file mode 100644 index 00000000..0301fa00 --- /dev/null +++ b/dj_pipeline/tests/data/w_ephys/Xestia_2026-07-30_1.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f40138d27d5f7c3c31605c167e5d1a035b02097f64bb01abcf79ce0593858a8 +size 2316 diff --git a/dj_pipeline/tests/data/w_ephys/Xestia_2026-07-30_1.npy b/dj_pipeline/tests/data/w_ephys/Xestia_2026-07-30_1.npy new file mode 100644 index 00000000..4cd70389 --- /dev/null +++ b/dj_pipeline/tests/data/w_ephys/Xestia_2026-07-30_1.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e107c104ae81f418a92d4c4029624c5a5e30eae3d1e99ee3eefdf447f448cb8 +size 2454 diff --git a/dj_pipeline/tests/data/w_ephys/Xestia_2026-07-30_1.pickle b/dj_pipeline/tests/data/w_ephys/Xestia_2026-07-30_1.pickle new file mode 100644 index 00000000..947a3dd4 --- /dev/null +++ b/dj_pipeline/tests/data/w_ephys/Xestia_2026-07-30_1.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0efb7d1a5d529732cb27b1acebdcc4f84a895664c3389634b52fb40f2f80f812 +size 28783944 diff --git a/dj_pipeline/tests/data/w_ephys/vr4mice_Xestia_2026-07-30_1_DLC.hdf5 b/dj_pipeline/tests/data/w_ephys/vr4mice_Xestia_2026-07-30_1_DLC.hdf5 new file mode 100644 index 00000000..e98a4cc9 --- /dev/null +++ b/dj_pipeline/tests/data/w_ephys/vr4mice_Xestia_2026-07-30_1_DLC.hdf5 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:566186e199d1eca47051b850f2dd2b51de6898bf76e0345e4340d54f2f9ccd60 +size 60025836 diff --git a/dj_pipeline/tests/data/w_ephys/vr4mice_Xestia_2026-07-30_1_PROC b/dj_pipeline/tests/data/w_ephys/vr4mice_Xestia_2026-07-30_1_PROC new file mode 100644 index 00000000..239de594 --- /dev/null +++ b/dj_pipeline/tests/data/w_ephys/vr4mice_Xestia_2026-07-30_1_PROC @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:605fe0a99438aae32c378efbcf331cf594095ff15995889a59cc8976e58333e5 +size 176862800 From 05c75e1465bdb02b24d6ee476dfc50574d093a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 18:18:19 +0200 Subject: [PATCH 21/40] Add new test session to cron scenario yaml --- .github/workflows/cron-scenario.yml | 32 +++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cron-scenario.yml b/.github/workflows/cron-scenario.yml index 0cd3727b..bd2fb94d 100644 --- a/.github/workflows/cron-scenario.yml +++ b/.github/workflows/cron-scenario.yml @@ -41,8 +41,12 @@ jobs: mkdir -p "$RUNNER_TEMP/vr4mice-data/processed" mkdir -p "$RUNNER_TEMP/vr4mice-data/summary_plots" mkdir -p "$RUNNER_TEMP/vr4mice-shared" + mkdir -p "$RUNNER_TEMP/vr4mice-input-data" mkdir -p "$RUNNER_TEMP/vr4mice-screen-recordings/raw_screen_recordings" cp -a "$GITHUB_WORKSPACE/dj_pipeline/tests/data/w_photodiode/." "$RUNNER_TEMP/vr4mice-data/processed/" + cp -a "$GITHUB_WORKSPACE/dj_pipeline/tests/data/w_ephys/." "$RUNNER_TEMP/vr4mice-data/processed/" + cp -a "$GITHUB_WORKSPACE/dj_pipeline/tests/data/w_photodiode/." "$RUNNER_TEMP/vr4mice-input-data/" + cp -a "$GITHUB_WORKSPACE/dj_pipeline/tests/data/w_ephys/." "$RUNNER_TEMP/vr4mice-input-data/" ln -sf \ "$GITHUB_WORKSPACE/dj_pipeline/tests/data/w_photodiode/Imagingsource_Flamingo_2026-02-05_1_VIDEO.avi" \ "$RUNNER_TEMP/vr4mice-screen-recordings/raw_screen_recordings/Flamingo_2026-02-05_1.mkv" @@ -112,8 +116,8 @@ jobs: -e DJ_SUPPORT_ADAPTED_TYPES="${DJ_SUPPORT_ADAPTED_TYPES}" \ -e MPLBACKEND="${MPLBACKEND}" \ -v "$GITHUB_WORKSPACE/dj_pipeline:/app" \ - -v "$GITHUB_WORKSPACE/dj_pipeline/tests/data/w_photodiode/:/data/data/" \ - -v "$GITHUB_WORKSPACE/dj_pipeline/tests/data/w_photodiode/:/data/dlc_video/" \ + -v "$RUNNER_TEMP/vr4mice-input-data:/data/data/" \ + -v "$RUNNER_TEMP/vr4mice-input-data:/data/dlc_video/" \ -v "$RUNNER_TEMP/vr4mice-data/processed:/data/processed" \ -v "$RUNNER_TEMP/vr4mice-data/summary_plots:/data/summary_plots" \ -v "$RUNNER_TEMP/vr4mice-shared:/shared" \ @@ -122,8 +126,32 @@ jobs: bash -lc "set -euo pipefail \ && pip install --no-deps /app/base/base_min_schemas /app/base/base_actions \ && python cron_scenario.py \ + && local_experiment_member_batch1_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch1';\")\" \ + && local_experiment_member_batch2_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch2';\")\" \ + && local_inclusion_status_batch1_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch1';\")\" \ + && local_inclusion_status_batch2_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch2';\")\" \ + && echo \"Local run decision._experiment_member batch1 rows: ${local_experiment_member_batch1_count}\" \ + && echo \"Local run decision._experiment_member batch2 rows: ${local_experiment_member_batch2_count}\" \ + && echo \"Local run decision.__inclusion_status batch1 rows: ${local_inclusion_status_batch1_count}\" \ + && echo \"Local run decision.__inclusion_status batch2 rows: ${local_inclusion_status_batch2_count}\" \ + && test \"${local_experiment_member_batch1_count}\" -eq 0 \ + && test \"${local_inclusion_status_batch1_count}\" -eq 0 \ + && test \"${local_experiment_member_batch2_count}\" -ge 1 \ + && test \"${local_inclusion_status_batch2_count}\" -ge 1 \ && mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -e 'DELETE FROM vr4mice.failed_session;' \ && python cron_scenario.py --aws \ + && aws_experiment_member_batch1_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch1';\")\" \ + && aws_experiment_member_batch2_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch2';\")\" \ + && aws_inclusion_status_batch1_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch1';\")\" \ + && aws_inclusion_status_batch2_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch2';\")\" \ + && echo \"AWS run decision._experiment_member batch1 rows: ${aws_experiment_member_batch1_count}\" \ + && echo \"AWS run decision._experiment_member batch2 rows: ${aws_experiment_member_batch2_count}\" \ + && echo \"AWS run decision.__inclusion_status batch1 rows: ${aws_inclusion_status_batch1_count}\" \ + && echo \"AWS run decision.__inclusion_status batch2 rows: ${aws_inclusion_status_batch2_count}\" \ + && test \"${aws_experiment_member_batch1_count}\" -ge 1 \ + && test \"${aws_inclusion_status_batch1_count}\" -ge 1 \ + && test \"${aws_experiment_member_batch2_count}\" -ge 1 \ + && test \"${aws_inclusion_status_batch2_count}\" -ge 1 \ && cd /app \ && python -u tests/verify_summary_emails_after_cron.py" From a4640f6e6fd74815c537db2dc02c432656e4c27e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 18:24:54 +0200 Subject: [PATCH 22/40] Fix quoting bug --- .github/workflows/cron-scenario.yml | 50 ++++++++++++++--------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/cron-scenario.yml b/.github/workflows/cron-scenario.yml index bd2fb94d..36b1a6b6 100644 --- a/.github/workflows/cron-scenario.yml +++ b/.github/workflows/cron-scenario.yml @@ -126,32 +126,32 @@ jobs: bash -lc "set -euo pipefail \ && pip install --no-deps /app/base/base_min_schemas /app/base/base_actions \ && python cron_scenario.py \ - && local_experiment_member_batch1_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch1';\")\" \ - && local_experiment_member_batch2_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch2';\")\" \ - && local_inclusion_status_batch1_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch1';\")\" \ - && local_inclusion_status_batch2_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch2';\")\" \ - && echo \"Local run decision._experiment_member batch1 rows: ${local_experiment_member_batch1_count}\" \ - && echo \"Local run decision._experiment_member batch2 rows: ${local_experiment_member_batch2_count}\" \ - && echo \"Local run decision.__inclusion_status batch1 rows: ${local_inclusion_status_batch1_count}\" \ - && echo \"Local run decision.__inclusion_status batch2 rows: ${local_inclusion_status_batch2_count}\" \ - && test \"${local_experiment_member_batch1_count}\" -eq 0 \ - && test \"${local_inclusion_status_batch1_count}\" -eq 0 \ - && test \"${local_experiment_member_batch2_count}\" -ge 1 \ - && test \"${local_inclusion_status_batch2_count}\" -ge 1 \ - && mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -e 'DELETE FROM vr4mice.failed_session;' \ + && local_experiment_member_batch1_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch1';\")\" \ + && local_experiment_member_batch2_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch2';\")\" \ + && local_inclusion_status_batch1_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch1';\")\" \ + && local_inclusion_status_batch2_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch2';\")\" \ + && echo \"Local run decision._experiment_member batch1 rows: \${local_experiment_member_batch1_count}\" \ + && echo \"Local run decision._experiment_member batch2 rows: \${local_experiment_member_batch2_count}\" \ + && echo \"Local run decision.__inclusion_status batch1 rows: \${local_inclusion_status_batch1_count}\" \ + && echo \"Local run decision.__inclusion_status batch2 rows: \${local_inclusion_status_batch2_count}\" \ + && test \"\${local_experiment_member_batch1_count}\" -eq 0 \ + && test \"\${local_inclusion_status_batch1_count}\" -eq 0 \ + && test \"\${local_experiment_member_batch2_count}\" -ge 1 \ + && test \"\${local_inclusion_status_batch2_count}\" -ge 1 \ + && mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -e 'DELETE FROM vr4mice.failed_session;' \ && python cron_scenario.py --aws \ - && aws_experiment_member_batch1_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch1';\")\" \ - && aws_experiment_member_batch2_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch2';\")\" \ - && aws_inclusion_status_batch1_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch1';\")\" \ - && aws_inclusion_status_batch2_count=\"$(mysql -h \"${DJ_HOST}\" -P \"${DJ_PORT}\" -u \"${DJ_USER}\" -p\"${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch2';\")\" \ - && echo \"AWS run decision._experiment_member batch1 rows: ${aws_experiment_member_batch1_count}\" \ - && echo \"AWS run decision._experiment_member batch2 rows: ${aws_experiment_member_batch2_count}\" \ - && echo \"AWS run decision.__inclusion_status batch1 rows: ${aws_inclusion_status_batch1_count}\" \ - && echo \"AWS run decision.__inclusion_status batch2 rows: ${aws_inclusion_status_batch2_count}\" \ - && test \"${aws_experiment_member_batch1_count}\" -ge 1 \ - && test \"${aws_inclusion_status_batch1_count}\" -ge 1 \ - && test \"${aws_experiment_member_batch2_count}\" -ge 1 \ - && test \"${aws_inclusion_status_batch2_count}\" -ge 1 \ + && aws_experiment_member_batch1_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch1';\")\" \ + && aws_experiment_member_batch2_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch2';\")\" \ + && aws_inclusion_status_batch1_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch1';\")\" \ + && aws_inclusion_status_batch2_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch2';\")\" \ + && echo \"AWS run decision._experiment_member batch1 rows: \${aws_experiment_member_batch1_count}\" \ + && echo \"AWS run decision._experiment_member batch2 rows: \${aws_experiment_member_batch2_count}\" \ + && echo \"AWS run decision.__inclusion_status batch1 rows: \${aws_inclusion_status_batch1_count}\" \ + && echo \"AWS run decision.__inclusion_status batch2 rows: \${aws_inclusion_status_batch2_count}\" \ + && test \"\${aws_experiment_member_batch1_count}\" -ge 1 \ + && test \"\${aws_inclusion_status_batch1_count}\" -ge 1 \ + && test \"\${aws_experiment_member_batch2_count}\" -ge 1 \ + && test \"\${aws_inclusion_status_batch2_count}\" -ge 1 \ && cd /app \ && python -u tests/verify_summary_emails_after_cron.py" From 8d37637de76b0b73e4cac24ffe9a0579179e82f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 18:44:11 +0200 Subject: [PATCH 23/40] Update SQL queries in cron scenario to join inclusion status with experiment members --- .github/workflows/cron-scenario.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cron-scenario.yml b/.github/workflows/cron-scenario.yml index 36b1a6b6..cf8f2ae3 100644 --- a/.github/workflows/cron-scenario.yml +++ b/.github/workflows/cron-scenario.yml @@ -128,8 +128,8 @@ jobs: && python cron_scenario.py \ && local_experiment_member_batch1_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch1';\")\" \ && local_experiment_member_batch2_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch2';\")\" \ - && local_inclusion_status_batch1_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch1';\")\" \ - && local_inclusion_status_batch2_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch2';\")\" \ + && local_inclusion_status_batch1_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) WHERE m.batch_name = 'batch1';\")\" \ + && local_inclusion_status_batch2_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) WHERE m.batch_name = 'batch2';\")\" \ && echo \"Local run decision._experiment_member batch1 rows: \${local_experiment_member_batch1_count}\" \ && echo \"Local run decision._experiment_member batch2 rows: \${local_experiment_member_batch2_count}\" \ && echo \"Local run decision.__inclusion_status batch1 rows: \${local_inclusion_status_batch1_count}\" \ @@ -142,8 +142,8 @@ jobs: && python cron_scenario.py --aws \ && aws_experiment_member_batch1_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch1';\")\" \ && aws_experiment_member_batch2_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch2';\")\" \ - && aws_inclusion_status_batch1_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch1';\")\" \ - && aws_inclusion_status_batch2_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = 'batch2';\")\" \ + && aws_inclusion_status_batch1_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) WHERE m.batch_name = 'batch1';\")\" \ + && aws_inclusion_status_batch2_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) WHERE m.batch_name = 'batch2';\")\" \ && echo \"AWS run decision._experiment_member batch1 rows: \${aws_experiment_member_batch1_count}\" \ && echo \"AWS run decision._experiment_member batch2 rows: \${aws_experiment_member_batch2_count}\" \ && echo \"AWS run decision.__inclusion_status batch1 rows: \${aws_inclusion_status_batch1_count}\" \ @@ -302,8 +302,8 @@ jobs: experiment_member_batch1_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = "batch1";')" experiment_member_batch2_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = "batch2";')" inclusion_status_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__inclusion_status;')" - inclusion_status_batch1_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = "batch1";')" - inclusion_status_batch2_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__inclusion_status WHERE batch_name = "batch2";')" + inclusion_status_batch1_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) WHERE m.batch_name = "batch1";')" + inclusion_status_batch2_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) WHERE m.batch_name = "batch2";')" label_set_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.`#label_set`;')" prediction_model_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__prediction_model;')" decision_points_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__decision_points;')" From 9b9fbe46cfac540d75b8aba308ef9ebba5ee4a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 5 Aug 2026 18:46:48 +0200 Subject: [PATCH 24/40] Correct over-indentation on two variable-assignment lines --- .github/workflows/cron-scenario.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cron-scenario.yml b/.github/workflows/cron-scenario.yml index cf8f2ae3..40ab45fe 100644 --- a/.github/workflows/cron-scenario.yml +++ b/.github/workflows/cron-scenario.yml @@ -302,8 +302,8 @@ jobs: experiment_member_batch1_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = "batch1";')" experiment_member_batch2_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = "batch2";')" inclusion_status_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__inclusion_status;')" - inclusion_status_batch1_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) WHERE m.batch_name = "batch1";')" - inclusion_status_batch2_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) WHERE m.batch_name = "batch2";')" + inclusion_status_batch1_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) WHERE m.batch_name = "batch1";')" + inclusion_status_batch2_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) WHERE m.batch_name = "batch2";')" label_set_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.`#label_set`;')" prediction_model_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__prediction_model;')" decision_points_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM decision.__decision_points;')" From 43a46efd89d1c1348bb9853c2c5224e3600961c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Thu, 6 Aug 2026 14:53:42 +0200 Subject: [PATCH 25/40] Refactor cron scenario to use compute_locally flag for batch processing and enhance ExcludedDataset pattern matching logic --- .github/workflows/cron-scenario.yml | 48 +++++++++++++------------- dj_pipeline/cron_scenario.py | 40 +++++++++------------ dj_pipeline/vr4mice/schema/barcodes.py | 20 ++++++++--- dj_pipeline/vr4mice/schema/vr4mice.py | 45 ++++++++++++++++++------ tests/integration/test_run_modes.py | 38 ++++++++++++++++++++ 5 files changed, 129 insertions(+), 62 deletions(-) diff --git a/.github/workflows/cron-scenario.yml b/.github/workflows/cron-scenario.yml index 40ab45fe..8b4208fa 100644 --- a/.github/workflows/cron-scenario.yml +++ b/.github/workflows/cron-scenario.yml @@ -126,32 +126,32 @@ jobs: bash -lc "set -euo pipefail \ && pip install --no-deps /app/base/base_min_schemas /app/base/base_actions \ && python cron_scenario.py \ - && local_experiment_member_batch1_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch1';\")\" \ - && local_experiment_member_batch2_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch2';\")\" \ - && local_inclusion_status_batch1_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) WHERE m.batch_name = 'batch1';\")\" \ - && local_inclusion_status_batch2_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) WHERE m.batch_name = 'batch2';\")\" \ - && echo \"Local run decision._experiment_member batch1 rows: \${local_experiment_member_batch1_count}\" \ - && echo \"Local run decision._experiment_member batch2 rows: \${local_experiment_member_batch2_count}\" \ - && echo \"Local run decision.__inclusion_status batch1 rows: \${local_inclusion_status_batch1_count}\" \ - && echo \"Local run decision.__inclusion_status batch2 rows: \${local_inclusion_status_batch2_count}\" \ - && test \"\${local_experiment_member_batch1_count}\" -eq 0 \ - && test \"\${local_inclusion_status_batch1_count}\" -eq 0 \ - && test \"\${local_experiment_member_batch2_count}\" -ge 1 \ - && test \"\${local_inclusion_status_batch2_count}\" -ge 1 \ + && local_experiment_member_nonlocal_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member m JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 0;\")\" \ + && local_experiment_member_local_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member m JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 1;\")\" \ + && local_inclusion_status_nonlocal_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 0;\")\" \ + && local_inclusion_status_local_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 1;\")\" \ + && echo \"Local run decision._experiment_member compute_locally=0 rows: \${local_experiment_member_nonlocal_count}\" \ + && echo \"Local run decision._experiment_member compute_locally=1 rows: \${local_experiment_member_local_count}\" \ + && echo \"Local run decision.__inclusion_status compute_locally=0 rows: \${local_inclusion_status_nonlocal_count}\" \ + && echo \"Local run decision.__inclusion_status compute_locally=1 rows: \${local_inclusion_status_local_count}\" \ + && test \"\${local_experiment_member_nonlocal_count}\" -eq 0 \ + && test \"\${local_inclusion_status_nonlocal_count}\" -eq 0 \ + && test \"\${local_experiment_member_local_count}\" -ge 1 \ + && test \"\${local_inclusion_status_local_count}\" -ge 1 \ && mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -e 'DELETE FROM vr4mice.failed_session;' \ && python cron_scenario.py --aws \ - && aws_experiment_member_batch1_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch1';\")\" \ - && aws_experiment_member_batch2_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member WHERE batch_name = 'batch2';\")\" \ - && aws_inclusion_status_batch1_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) WHERE m.batch_name = 'batch1';\")\" \ - && aws_inclusion_status_batch2_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) WHERE m.batch_name = 'batch2';\")\" \ - && echo \"AWS run decision._experiment_member batch1 rows: \${aws_experiment_member_batch1_count}\" \ - && echo \"AWS run decision._experiment_member batch2 rows: \${aws_experiment_member_batch2_count}\" \ - && echo \"AWS run decision.__inclusion_status batch1 rows: \${aws_inclusion_status_batch1_count}\" \ - && echo \"AWS run decision.__inclusion_status batch2 rows: \${aws_inclusion_status_batch2_count}\" \ - && test \"\${aws_experiment_member_batch1_count}\" -ge 1 \ - && test \"\${aws_inclusion_status_batch1_count}\" -ge 1 \ - && test \"\${aws_experiment_member_batch2_count}\" -ge 1 \ - && test \"\${aws_inclusion_status_batch2_count}\" -ge 1 \ + && aws_experiment_member_nonlocal_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member m JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 0;\")\" \ + && aws_experiment_member_local_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member m JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 1;\")\" \ + && aws_inclusion_status_nonlocal_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 0;\")\" \ + && aws_inclusion_status_local_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 1;\")\" \ + && echo \"AWS run decision._experiment_member compute_locally=0 rows: \${aws_experiment_member_nonlocal_count}\" \ + && echo \"AWS run decision._experiment_member compute_locally=1 rows: \${aws_experiment_member_local_count}\" \ + && echo \"AWS run decision.__inclusion_status compute_locally=0 rows: \${aws_inclusion_status_nonlocal_count}\" \ + && echo \"AWS run decision.__inclusion_status compute_locally=1 rows: \${aws_inclusion_status_local_count}\" \ + && test \"\${aws_experiment_member_nonlocal_count}\" -ge 1 \ + && test \"\${aws_inclusion_status_nonlocal_count}\" -ge 1 \ + && test \"\${aws_experiment_member_local_count}\" -ge 1 \ + && test \"\${aws_inclusion_status_local_count}\" -ge 1 \ && cd /app \ && python -u tests/verify_summary_emails_after_cron.py" diff --git a/dj_pipeline/cron_scenario.py b/dj_pipeline/cron_scenario.py index 500eaea8..d89b8d04 100755 --- a/dj_pipeline/cron_scenario.py +++ b/dj_pipeline/cron_scenario.py @@ -294,52 +294,44 @@ def import_decision_schema(): if not args.aws: from vr4mice.schema import vr4mice as vr4mice_schema - # Local server: do not recompute batch1 decision outputs locally; - # batch1 decision results are produced on AWS, as it needs to be - # combined to other sessions from the other labs. - # batch2 decision outputs are produced locally, as they are only - # for local sessions. + # Decision routing is controlled by Batch.compute_locally: + # local runs only populate datasets whose batch is marked to + # compute decision tables locally. decision_restriction = ( - vr4mice_schema.DatasetBatch & 'batch_name != "batch1"' + vr4mice_schema.DatasetBatch + * vr4mice_schema.Batch + & {"compute_locally": True} ) + decision_populate_args = ( + (decision_restriction,) if decision_restriction is not None else () + ) + run_step( "decision.ExperimentMember.populate", - lambda: decision.ExperimentMember().populate(decision_restriction) - if decision_restriction is not None - else decision.ExperimentMember().populate(), + lambda: decision.ExperimentMember().populate(*decision_populate_args), ) run_step( "decision.InclusionStatus.populate", - lambda: decision.InclusionStatus().populate(decision_restriction) - if decision_restriction is not None - else decision.InclusionStatus().populate(), + lambda: decision.InclusionStatus().populate(*decision_populate_args), ) run_step("decision.LabelSet.fill", lambda: decision.LabelSet.fill()) run_step( "decision.PredictionModel.populate", - lambda: decision.PredictionModel().populate(decision_restriction) - if decision_restriction is not None - else decision.PredictionModel().populate(), + lambda: decision.PredictionModel().populate(*decision_populate_args), ) run_step( "decision.DecisionPoints.populate", - lambda: decision.DecisionPoints().populate(decision_restriction) - if decision_restriction is not None - else decision.DecisionPoints().populate(), + lambda: decision.DecisionPoints().populate(*decision_populate_args), ) run_step( "decision.PredictionModel10Windows.populate", - lambda: decision.PredictionModel10Windows().populate(decision_restriction) - if decision_restriction is not None - else decision.PredictionModel10Windows().populate(), + lambda: decision.PredictionModel10Windows().populate(*decision_populate_args), ) run_step( "decision.DecisionPoints10Windows.populate", - lambda: decision.DecisionPoints10Windows().populate(decision_restriction) - if decision_restriction is not None - else decision.DecisionPoints10Windows().populate(), + lambda: decision.DecisionPoints10Windows().populate(*decision_populate_args), ) if not args.aws: diff --git a/dj_pipeline/vr4mice/schema/barcodes.py b/dj_pipeline/vr4mice/schema/barcodes.py index 6b264f97..da7297e8 100644 --- a/dj_pipeline/vr4mice/schema/barcodes.py +++ b/dj_pipeline/vr4mice/schema/barcodes.py @@ -21,7 +21,14 @@ @schema class TeensyTTL(dj.Imported): - """Raw Teensy barcode channel imported from a DLC PROC file.""" + """Raw Teensy barcode channel imported from a DLC PROC file. + + Threshold semantics: + a session is eligible when its resolved Batch (via DatasetBatch) has + ``has_neural_data=True``. Batch membership boundaries come from + ``Batch.resolve`` (latest ``start_date <= doe``), so boundary dates are + inclusive for the newer batch. + """ definition = """ -> vr4mice.DLC @@ -31,9 +38,14 @@ class TeensyTTL(dj.Imported): has_ttl=0: bool # True when aligned, non-empty Teensy TTL arrays are available """ - # Teensy barcodes were only wired up starting with the batch2 (Neuropixels) cohort - _batch2_start_date = (vr4mice.Batch & {"batch_name": "batch2"}).fetch1("start_date") - key_source = vr4mice.DLC & f"doe > '{_batch2_start_date}'" + # Gate by resolved batch membership (not direct date filtering here): + # DLC session -> DatasetBatch -> Batch(has_neural_data=True). + # Per-session mixed TTL availability is still handled by has_ttl. + key_source = ( + vr4mice.DLC.proj() + * vr4mice.DatasetBatch.proj("batch_name") + * (vr4mice.Batch & {"has_neural_data": True}).proj("batch_name") + ) def make(self, key): """Load the raw Teensy TTL arrays from one DLC PROC file.""" diff --git a/dj_pipeline/vr4mice/schema/vr4mice.py b/dj_pipeline/vr4mice/schema/vr4mice.py index 439b86a6..6d8ae589 100644 --- a/dj_pipeline/vr4mice/schema/vr4mice.py +++ b/dj_pipeline/vr4mice/schema/vr4mice.py @@ -2,6 +2,7 @@ import datetime import os +import re import datajoint as dj import numpy as np @@ -115,12 +116,13 @@ class Batch(dj.Lookup): --- start_date : date # datasets on/after this date fall in this batch description : varchar(255) - has_neural_data : bool + has_neural_data : bool # cohort metadata; sessions may still be mixed + compute_locally : bool # local cron computes decision tables for this batch """ contents = [ - ("batch1", "2000-01-01", "behavioral_cohort", False), - ("batch2", "2026-06-01", "ephys_cohort", True), + ("batch1", "2000-01-01", "behavioral_cohort", False, False), + ("batch2", "2026-06-01", "ephys_cohort", True, True), ] @classmethod @@ -178,8 +180,9 @@ class ExcludedDataset(dj.Lookup): data may still be ingested normally through the rest of the pipeline. dataset_pattern is either an exact dataset name (excludes that one - session) or a SQL LIKE pattern such as "Testmouse%" (excludes every - matching dataset, past and future, without needing a row per session). + session) or a SQL LIKE-style wildcard pattern using "%" such as + "Testmouse%" (excludes every matching dataset, past and future, + without needing a row per session). Not a -> Dataset FK, since a "%" pattern doesn't name a real row. """ @@ -208,10 +211,25 @@ class ExcludedDataset(dj.Lookup): @classmethod def matches(cls, dataset): """Return the matching row's reason, or None if dataset isn't excluded.""" - rows = (cls & f"'{dataset}' LIKE dataset_pattern").fetch(as_dict=True) - if not rows: - return None - return rows[0]["reason"] + rows = cls.fetch("dataset_pattern", "reason", as_dict=True) + for row in rows: + pattern = row["dataset_pattern"] + if cls._dataset_matches_pattern(dataset, pattern): + return row["reason"] + return None + + @staticmethod + def _dataset_matches_pattern(dataset, pattern): + """Match dataset against an exact or '%' wildcard pattern. + + Patterns without '%' are treated as exact dataset names, so '_' stays + literal and does not act as a SQL single-character wildcard. + """ + if "%" not in pattern: + return dataset == pattern + + regex = "^" + re.escape(pattern).replace(r"\%", ".*") + "$" + return re.match(regex, dataset) is not None @classmethod def exclusion_filter(cls): @@ -225,7 +243,14 @@ def exclusion_filter(cls): patterns = cls.fetch("dataset_pattern") if len(patterns) == 0: return "TRUE" - return " AND ".join(f'dataset NOT LIKE "{p}"' for p in patterns) + clauses = [] + for pattern in patterns: + escaped = pattern.replace('"', r'\"') + if "%" in pattern: + clauses.append(f'dataset NOT LIKE "{escaped}"') + else: + clauses.append(f'dataset != "{escaped}"') + return " AND ".join(clauses) @schema diff --git a/tests/integration/test_run_modes.py b/tests/integration/test_run_modes.py index c5bccb24..444ea224 100644 --- a/tests/integration/test_run_modes.py +++ b/tests/integration/test_run_modes.py @@ -465,6 +465,44 @@ def test_barcodes_schema_imports(self, dj_config): assert hasattr(barcodes, "TeensyTTL") assert hasattr(barcodes, "TeensyBarcodes") + def test_excluded_dataset_exact_pattern_treats_underscore_literal(self, dj_config): + """Exact patterns containing '_' should not act as SQL single-char wildcards.""" + from vr4mice.schema import vr4mice + + vr4mice.ExcludedDataset.insert( + vr4mice.ExcludedDataset.contents, + skip_duplicates=True, + ) + + assert ( + vr4mice.ExcludedDataset.matches("Hamster_2026-02-02_1") + == "Missing DLC data" + ) + assert vr4mice.ExcludedDataset.matches("Hamster-2026-02-02-1") is None + + exclusion = vr4mice.ExcludedDataset.exclusion_filter() + assert 'dataset != "Hamster_2026-02-02_1"' in exclusion + + def test_barcodes_schema_imports_without_batch2(self, dj_config, clean_schemas): + """Importing barcodes should not fail when batch2 row is absent.""" + import importlib + import sys + + from vr4mice.schema import vr4mice + + # Ensure lookup rows exist first, then remove batch2 for this scenario. + vr4mice.Batch.insert(vr4mice.Batch.contents, skip_duplicates=True) + (vr4mice.Batch & {"batch_name": "batch2"}).delete() + + # Force a fresh import to exercise module import-time behavior. + sys.modules.pop("vr4mice.schema.barcodes", None) + barcodes = importlib.import_module("vr4mice.schema.barcodes") + + assert hasattr(barcodes, "TeensyTTL") + assert hasattr(barcodes, "TeensyBarcodes") + # Accessing key_source should also remain non-fatal. + _ = barcodes.TeensyTTL().key_source + def test_interpolated_trajectories_schema_imports(self, dj_config): """Verify interpolated_trajectories schema imports without error.""" from vr4mice.schema import interpolated_trajectories From c44d59018a7c48a3312ca9e72d6ffd5495883224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Thu, 6 Aug 2026 15:06:00 +0200 Subject: [PATCH 26/40] Run black --- dj_pipeline/vr4mice/schema/vr4mice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dj_pipeline/vr4mice/schema/vr4mice.py b/dj_pipeline/vr4mice/schema/vr4mice.py index 6d8ae589..4384dd89 100644 --- a/dj_pipeline/vr4mice/schema/vr4mice.py +++ b/dj_pipeline/vr4mice/schema/vr4mice.py @@ -245,7 +245,7 @@ def exclusion_filter(cls): return "TRUE" clauses = [] for pattern in patterns: - escaped = pattern.replace('"', r'\"') + escaped = pattern.replace('"', r"\"") if "%" in pattern: clauses.append(f'dataset NOT LIKE "{escaped}"') else: From 5e2a81bf4d9b10f0bab751f22deb5edc36ae41a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Thu, 6 Aug 2026 15:14:49 +0200 Subject: [PATCH 27/40] Run black #2 --- dj_pipeline/cron_scenario.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dj_pipeline/cron_scenario.py b/dj_pipeline/cron_scenario.py index d89b8d04..85169f40 100755 --- a/dj_pipeline/cron_scenario.py +++ b/dj_pipeline/cron_scenario.py @@ -298,8 +298,7 @@ def import_decision_schema(): # local runs only populate datasets whose batch is marked to # compute decision tables locally. decision_restriction = ( - vr4mice_schema.DatasetBatch - * vr4mice_schema.Batch + vr4mice_schema.DatasetBatch * vr4mice_schema.Batch & {"compute_locally": True} ) @@ -327,11 +326,15 @@ def import_decision_schema(): ) run_step( "decision.PredictionModel10Windows.populate", - lambda: decision.PredictionModel10Windows().populate(*decision_populate_args), + lambda: decision.PredictionModel10Windows().populate( + *decision_populate_args + ), ) run_step( "decision.DecisionPoints10Windows.populate", - lambda: decision.DecisionPoints10Windows().populate(*decision_populate_args), + lambda: decision.DecisionPoints10Windows().populate( + *decision_populate_args + ), ) if not args.aws: From 36341b71380ca9a292fe7265833d0c846a7d9764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Fri, 7 Aug 2026 10:27:44 +0200 Subject: [PATCH 28/40] Fix sql naming in cron yaml --- .github/workflows/cron-scenario.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cron-scenario.yml b/.github/workflows/cron-scenario.yml index 8b4208fa..93a654a9 100644 --- a/.github/workflows/cron-scenario.yml +++ b/.github/workflows/cron-scenario.yml @@ -126,10 +126,10 @@ jobs: bash -lc "set -euo pipefail \ && pip install --no-deps /app/base/base_min_schemas /app/base/base_actions \ && python cron_scenario.py \ - && local_experiment_member_nonlocal_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member m JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 0;\")\" \ - && local_experiment_member_local_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member m JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 1;\")\" \ - && local_inclusion_status_nonlocal_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 0;\")\" \ - && local_inclusion_status_local_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 1;\")\" \ + && local_experiment_member_nonlocal_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member m JOIN vr4mice.\`#batch\` b USING (batch_name) WHERE b.compute_locally = 0;\")\" \ + && local_experiment_member_local_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member m JOIN vr4mice.\`#batch\` b USING (batch_name) WHERE b.compute_locally = 1;\")\" \ + && local_inclusion_status_nonlocal_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) JOIN vr4mice.\`#batch\` b USING (batch_name) WHERE b.compute_locally = 0;\")\" \ + && local_inclusion_status_local_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) JOIN vr4mice.\`#batch\` b USING (batch_name) WHERE b.compute_locally = 1;\")\" \ && echo \"Local run decision._experiment_member compute_locally=0 rows: \${local_experiment_member_nonlocal_count}\" \ && echo \"Local run decision._experiment_member compute_locally=1 rows: \${local_experiment_member_local_count}\" \ && echo \"Local run decision.__inclusion_status compute_locally=0 rows: \${local_inclusion_status_nonlocal_count}\" \ @@ -140,10 +140,10 @@ jobs: && test \"\${local_inclusion_status_local_count}\" -ge 1 \ && mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -e 'DELETE FROM vr4mice.failed_session;' \ && python cron_scenario.py --aws \ - && aws_experiment_member_nonlocal_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member m JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 0;\")\" \ - && aws_experiment_member_local_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member m JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 1;\")\" \ - && aws_inclusion_status_nonlocal_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 0;\")\" \ - && aws_inclusion_status_local_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) JOIN vr4mice.batch b USING (batch_name) WHERE b.compute_locally = 1;\")\" \ + && aws_experiment_member_nonlocal_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member m JOIN vr4mice.\`#batch\` b USING (batch_name) WHERE b.compute_locally = 0;\")\" \ + && aws_experiment_member_local_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision._experiment_member m JOIN vr4mice.\`#batch\` b USING (batch_name) WHERE b.compute_locally = 1;\")\" \ + && aws_inclusion_status_nonlocal_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) JOIN vr4mice.\`#batch\` b USING (batch_name) WHERE b.compute_locally = 0;\")\" \ + && aws_inclusion_status_local_count=\"\$(mysql -h \"\${DJ_HOST}\" -P \"\${DJ_PORT}\" -u \"\${DJ_USER}\" -p\"\${DJ_PASS}\" -N -e \"SELECT COUNT(*) FROM decision.__inclusion_status s JOIN decision._experiment_member m USING (dataset) JOIN vr4mice.\`#batch\` b USING (batch_name) WHERE b.compute_locally = 1;\")\" \ && echo \"AWS run decision._experiment_member compute_locally=0 rows: \${aws_experiment_member_nonlocal_count}\" \ && echo \"AWS run decision._experiment_member compute_locally=1 rows: \${aws_experiment_member_local_count}\" \ && echo \"AWS run decision.__inclusion_status compute_locally=0 rows: \${aws_inclusion_status_nonlocal_count}\" \ From becb3e3ef804fbedc1e4bafd11cc893a2e90016f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Tue, 11 Aug 2026 17:05:52 +0200 Subject: [PATCH 29/40] Run black --- dj_pipeline/vr4mice/schema/np_sync.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dj_pipeline/vr4mice/schema/np_sync.py b/dj_pipeline/vr4mice/schema/np_sync.py index 4e0f18c5..b22dea2a 100644 --- a/dj_pipeline/vr4mice/schema/np_sync.py +++ b/dj_pipeline/vr4mice/schema/np_sync.py @@ -90,9 +90,9 @@ def make(self, key): vr_values, vr_times = (vr_barcodes.TeensyBarcodes.Event & key).fetch( "barcode_value", "onset_time_unity", order_by="barcode_index" ) - np_values, np_times = (np_barcodes.OneBoxBarcodeExtraction.Event & key).fetch( - "barcode_value", "onset_time", order_by="barcode_index" - ) + np_values, np_times = ( + np_barcodes.OneBoxBarcodeExtraction.Event & key + ).fetch("barcode_value", "onset_time", order_by="barcode_index") fit = align_barcodes( vr_times, @@ -154,7 +154,9 @@ def align_timepoints(cls, key, timepoints: list): """ interpol_func = pickle.loads((cls & key).fetch1("interpol_func")) timepoints = np.array(timepoints, dtype=np.float64) - return [float(tx) if not np.isnan(tx) else None for tx in interpol_func(timepoints)] + return [ + float(tx) if not np.isnan(tx) else None for tx in interpol_func(timepoints) + ] @classmethod def align_timepoints_lin(cls, key, timepoints: list): From 1c8f36a464fc4383012a84140f8c4b3bb554716b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Tue, 11 Aug 2026 17:06:13 +0200 Subject: [PATCH 30/40] Add make format --- dj_pipeline/Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dj_pipeline/Makefile b/dj_pipeline/Makefile index fb261f1c..faf35fbe 100644 --- a/dj_pipeline/Makefile +++ b/dj_pipeline/Makefile @@ -37,11 +37,16 @@ BUILD_ARGS := \ .PHONY: build_all git_info git-whoami git-user-env git-user-local up_all base_install down_all client_build client_up \ client_stop client_rm client_down ipython bash mysql aws notebook sync_build \ sync_up add-cron cron-local cron-aws cron-now cron-aws-only cron-reboot import-dumps export-dumps \ - check-compose-project + check-compose-project format check-compose-project: @bash docker/check_compose_conflict.sh "$(COMPOSE_PROJECT)" +# Matches CI's formatter.yml: black==22.6 against dj_pipeline. +format: + pip install --quiet black==22.6 + black . + # Fail with logs if the client service is not running (common after compose project / name mismatch). define verify_client_running @if [ "$$($(DOCKER_COMPOSE) ps --status running -q client 2>/dev/null | wc -l | tr -d ' ')" = "0" ]; then \ From a98623ea9827ba93beb012f1813eadd27156204d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Tue, 11 Aug 2026 17:46:37 +0200 Subject: [PATCH 31/40] Update np_sync schema and cron job to include barcode sync checks --- .github/workflows/cron-scenario.yml | 13 +++++++++++++ dj_pipeline/vr4mice/schema/np_sync.py | 22 +++++++++++----------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cron-scenario.yml b/.github/workflows/cron-scenario.yml index 4f4355a5..c1489828 100644 --- a/.github/workflows/cron-scenario.yml +++ b/.github/workflows/cron-scenario.yml @@ -167,6 +167,7 @@ jobs: interpolated_trajectories session_metrics latency_tests + np_sync summary_emails inputs_videos decision @@ -275,6 +276,10 @@ jobs: dataframe_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM base_analysis.__data_frame;')" summary_plot_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM base_analysis.__summary_plots;')" summary_email_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM summary_emails.summary_plot_email WHERE send_error IS NULL;')" + np_sync_barcode_sync_count="" + if grep -qx 'np_sync' "$RUNNER_TEMP/mysql-dump/schemas.txt"; then + np_sync_barcode_sync_count="$(docker exec vr4mice-mysql-restore mysql -N -uroot -psimple -e 'SELECT COUNT(*) FROM np_sync.__barcode_sync;')" + fi if [ "${summary_email_count}" -lt 1 ] 2>/dev/null; then echo "summary_emails tables:" >&2 docker exec vr4mice-mysql-restore mysql -uroot -psimple -e 'SHOW TABLES FROM summary_emails;' >&2 || true @@ -290,6 +295,11 @@ jobs: echo "base_analysis.__data_frame rows: ${dataframe_count}" echo "base_analysis.__summary_plots rows: ${summary_plot_count}" echo "summary_emails.summary_plot_email successful rows: ${summary_email_count}" + if [ -n "${np_sync_barcode_sync_count}" ]; then + echo "np_sync.__barcode_sync rows: ${np_sync_barcode_sync_count}" + else + echo "np_sync schema not present in dump; skipping np_sync.__barcode_sync row-count check" + fi test "${dataset_count}" -ge 1 test "${batch_count}" -ge 2 @@ -298,6 +308,9 @@ jobs: test "${state_count}" -ge 1 test "${dataframe_count}" -ge 1 test "${summary_plot_count}" -ge 1 + if [ -n "${np_sync_barcode_sync_count}" ]; then + test "${np_sync_barcode_sync_count}" -ge 1 + fi - name: Verify decision tables (--aws populated) run: | diff --git a/dj_pipeline/vr4mice/schema/np_sync.py b/dj_pipeline/vr4mice/schema/np_sync.py index b22dea2a..23ef3ff6 100644 --- a/dj_pipeline/vr4mice/schema/np_sync.py +++ b/dj_pipeline/vr4mice/schema/np_sync.py @@ -40,12 +40,12 @@ @schema class BarcodeSync(dj.Computed): - """Linear fit + interpolator mapping VR Unity/game time to NP OneBox DAQ time. + """Linear fit + interpolator mapping VR Unity/game time to NP probe time. - One row per (dataset, recording, OneBox DAQ stream) triple that has a linked NP - recording and successfully decoded barcodes on both sides. Populated only for - those keys — see ``key_source`` — so ``populate()`` is a no-op, not an error, for - VR-only sessions with no matching neural recording. + One row per (dataset, recording, probe) triple that has a linked NP recording and + successfully decoded barcodes on both sides. Populated only for those keys — see + ``key_source`` — so ``populate()`` is a no-op, not an error, for VR-only sessions + with no matching neural recording. Downstream code should not fetch ``slope``/``intercept``/``interpol_func`` directly; use ``align_timepoints``/``align_timepoints_lin`` instead. @@ -54,7 +54,7 @@ class BarcodeSync(dj.Computed): definition = """ -> base.Base -> session_link.RecordingSessionLink - -> acquisition.OneBoxDaq + -> acquisition.RecordingProbe --- skip_first_n_barcodes: smallint unsigned # leading VR barcode events excluded from the fit slope: float # Slope of the linear fit mapping VR time to NP time @@ -65,9 +65,9 @@ class BarcodeSync(dj.Computed): """ key_source = ( - base.Base * session_link.RecordingSessionLink * acquisition.OneBoxDaq + base.Base * session_link.RecordingSessionLink * acquisition.RecordingProbe & (vr_barcodes.TeensyBarcodes & 'extraction_status = "success"') - & (np_barcodes.OneBoxBarcodeExtraction & 'extraction_status = "success"') + & (np_barcodes.ProbeBarcodeExtraction & 'extraction_status = "success"') ) skip_first_n_barcodes = DEFAULT_SKIP_FIRST_N_BARCODES @@ -75,9 +75,9 @@ class BarcodeSync(dj.Computed): def make(self, key): """Fit and insert one VR-time-to-NP-time alignment. - Fetches decoded barcode events for one dataset/recording/DAQ key from both + Fetches decoded barcode events for one dataset/recording/probe key from both `vr_barcodes.TeensyBarcodes.Event` (VR side) and - `np_barcodes.OneBoxBarcodeExtraction.Event` (NP side), fits the alignment via + `np_barcodes.ProbeBarcodeExtraction.Event` (NP side), fits the alignment via `vr4mice.analysis.np_sync.align_barcodes`, and inserts the resulting fit parameters and pickled interpolator. On failure, records the error in `vr4mice.FailedSession` and logs a warning instead of raising, matching the @@ -91,7 +91,7 @@ def make(self, key): "barcode_value", "onset_time_unity", order_by="barcode_index" ) np_values, np_times = ( - np_barcodes.OneBoxBarcodeExtraction.Event & key + np_barcodes.ProbeBarcodeExtraction.Event & key ).fetch("barcode_value", "onset_time", order_by="barcode_index") fit = align_barcodes( From 122ee2c5c4b690e548e1202c65bddb475bab6d03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 12 Aug 2026 10:13:19 +0200 Subject: [PATCH 32/40] Fix data types --- dj_pipeline/vr4mice/schema/np_sync.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dj_pipeline/vr4mice/schema/np_sync.py b/dj_pipeline/vr4mice/schema/np_sync.py index 23ef3ff6..850edab8 100644 --- a/dj_pipeline/vr4mice/schema/np_sync.py +++ b/dj_pipeline/vr4mice/schema/np_sync.py @@ -56,12 +56,12 @@ class BarcodeSync(dj.Computed): -> session_link.RecordingSessionLink -> acquisition.RecordingProbe --- - skip_first_n_barcodes: smallint unsigned # leading VR barcode events excluded from the fit - slope: float # Slope of the linear fit mapping VR time to NP time - intercept: float # Intercept of the linear fit mapping VR time to NP time - r2: float # R-squared value of the linear fit + skip_first_n_barcodes: int32 # leading VR barcode events excluded from the fit + slope: float64 # Slope of the linear fit mapping VR time to NP time + intercept: float64 # Intercept of the linear fit mapping VR time to NP time + r2: float64 # R-squared value of the linear fit interpol_func: # pickled scipy.interpolate.interp1d, VR time -> NP time - barcode_overlap: float # Fraction of NP barcodes also found on the VR side + barcode_overlap: float64 # Fraction of NP barcodes also found on the VR side """ key_source = ( From ab779ce2474d27c8af5c2dd5de11ba99a212d4e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 12 Aug 2026 14:27:30 +0200 Subject: [PATCH 33/40] Refactor np_sync schema and update Docker Compose settings for improved path handling and module integration --- dj_pipeline/.env.compose.example | 4 +- dj_pipeline/docker-compose.yml | 5 +- dj_pipeline/vr4mice/schema/np_sync.py | 155 +++++++++++++++++++++++--- tests/unit/test_np_sync.py | 100 +++++++++++++++++ 4 files changed, 244 insertions(+), 20 deletions(-) diff --git a/dj_pipeline/.env.compose.example b/dj_pipeline/.env.compose.example index 3e89950d..3cdf8259 100644 --- a/dj_pipeline/.env.compose.example +++ b/dj_pipeline/.env.compose.example @@ -11,9 +11,7 @@ DB_DATA_PATH=/mnt/database/vr4mice/vr4mice_database/database/ SHARED_PATH=/mnt/database/shared DATA_PATH=/mnt/database/vr4mice/vr4mice_database/data SCREEN_RECORDINGS_PATH=/mnt/neuropixel_data/vr4mice/raw_screen_recordings -# Path to the np_pipeline "src" checkout (must contain a np_pipeline/ package dir). -# Optional — leave unset to mount an empty stub; the "np_sync" mode skips gracefully. -# NP_PIPELINE_SRC_PATH=../FreelyMoving_Mouse_NP/src +NP_PIPELINE_SRC_PATH=../../FreelyMoving_Mouse_NP JUPYTER_PORT=8887 CLIENT_IMAGE=mmathislab/vr4mice_app:0.1.0 CLIENT_CONTAINER_NAME=vr4mice_${USER} diff --git a/dj_pipeline/docker-compose.yml b/dj_pipeline/docker-compose.yml index 432fd68d..72908ade 100644 --- a/dj_pipeline/docker-compose.yml +++ b/dj_pipeline/docker-compose.yml @@ -1,3 +1,5 @@ +name: ${COMPOSE_PROJECT:-vr4mice} + services: db: image: ${DB_IMAGE:-mysql:8.0} @@ -23,6 +25,7 @@ services: - MPLCONFIGDIR=/app/.cache/matplotlib - XDG_CONFIG_HOME=/app/.cache - PATH=/app/.local/bin:/usr/local/bin:/usr/bin:/bin + - PYTHONPATH=/np_pipeline/src:${PYTHONPATH} - TAG=${TAG} network_mode: ${CLIENT_NETWORK_MODE:-host} # Run via bash so the bind-mounted script does not need +x on shared/NFS checkouts. @@ -36,7 +39,7 @@ services: - "${SCREEN_RECORDINGS_PATH:-/mnt/neuropixel_data/vr4mice/raw_screen_recordings}:/vr4mice_screen_recordings" - "./base/base_min_schemas:/base_schemas" - "./base/base_actions:/base_actions" - - "${NP_PIPELINE_SRC_PATH:-./np_pipeline_stub}:/np_pipeline" + - "${NP_PIPELINE_SRC_PATH:-../../FreelyMoving_Mouse_NP}:/np_pipeline" ports: - "${JUPYTER_PORT:-8887}:8888" working_dir: /app diff --git a/dj_pipeline/vr4mice/schema/np_sync.py b/dj_pipeline/vr4mice/schema/np_sync.py index 850edab8..8d0dc175 100644 --- a/dj_pipeline/vr4mice/schema/np_sync.py +++ b/dj_pipeline/vr4mice/schema/np_sync.py @@ -1,8 +1,11 @@ """Alignment of VR (behavior) time to Neuropixels (NP) native time via shared barcodes. -Not every VR ``Dataset`` has a corresponding NP recording; ``BarcodeSync.key_source`` -intersects with the NP-side linkage tables so ``populate()`` simply never calls -``make()`` for behavior-only sessions, instead of raising. +Not every VR ``Dataset`` has a corresponding NP recording. ``BarcodeSync.key_source`` +includes datasets with successful VR barcode extraction, while strict NP linkage +matching is performed inside ``make()`` via ``base.Base`` + +``session_link.RecordingSessionLink`` + successful NP probe barcode extraction. +Datasets with no strict NP match are recorded in ``vr4mice.FailedSession`` and +skipped cleanly. Cross-repo foreign keys: ``BarcodeSync`` references ``np_pipeline`` tables directly via ``-> ``. This requires ``vr4mice`` and ``np_pipeline`` to share one @@ -38,14 +41,87 @@ logger = Logger.get_logger() +def _dataset_identity(dataset: str): + """Return (mouse_name, day_iso, attempt) parsed from dataset name.""" + parts = dataset.split("_") + if len(parts) < 3: + return None + try: + return parts[0], parts[1], int(parts[2]) + except ValueError: + return None + + +def _candidate_sort_key(row, identity): + """Sort candidates by identity match first, then NP barcode event count.""" + event_count = int(row["np_event_count"]) + if identity is None: + return ( + -event_count, + str(row.get("recording_id", "")), + str(row.get("probe_serial_number", "")), + ) + + expected_mouse, expected_day, expected_attempt = identity + mouse_miss = 0 if row.get("mouse_name") == expected_mouse else 1 + day = row.get("day") + day_iso = day.isoformat() if hasattr(day, "isoformat") else str(day) + day_miss = 0 if day_iso == expected_day else 1 + attempt_miss = 0 if row.get("attempt") == expected_attempt else 1 + return ( + mouse_miss, + day_miss, + attempt_miss, + -event_count, + str(row.get("recording_id", "")), + str(row.get("probe_serial_number", "")), + ) + + +def _np_module_options(): + """Return available NP schema-module triplets, preferring imported modules.""" + options = [(acquisition, np_barcodes, session_link)] + legacy = ( + dj.VirtualModule("np_acquisition_legacy", "acquisition"), + dj.VirtualModule("np_barcodes_legacy", "barcodes"), + dj.VirtualModule("np_session_link_legacy", "session_link"), + ) + imported_lineage = acquisition.RecordingProbe.heading.attributes[ + "recording_id" + ].lineage + legacy_lineage = legacy[0].RecordingProbe.heading.attributes["recording_id"].lineage + if legacy_lineage != imported_lineage: + options.append(legacy) + return options + + +def _candidate_relation_for_modules(acq_mod, np_barcodes_mod, session_link_mod): + """Rows where one VR dataset has at least one NP probe with successful barcodes.""" + np_success = ( + np_barcodes_mod.ProbeBarcodeExtraction & 'extraction_status = "success"' + ).proj( + "recording_id", + "probe_serial_number", + np_event_count="event_count", + ) + return ( + base.Base + * acq_mod.RecordingProbe + * session_link_mod.RecordingSessionLink + * np_success + & (vr_barcodes.TeensyBarcodes & 'extraction_status = "success"') + ) + + @schema class BarcodeSync(dj.Computed): """Linear fit + interpolator mapping VR Unity/game time to NP probe time. - One row per (dataset, recording, probe) triple that has a linked NP recording and - successfully decoded barcodes on both sides. Populated only for those keys — see - ``key_source`` — so ``populate()`` is a no-op, not an error, for VR-only sessions - with no matching neural recording. + One row per VR dataset that has at least one linked NP recording/probe with + successful barcodes on both sides. ``key_source`` enumerates VR datasets with + successful VR barcode extraction; strict NP-side eligibility is resolved in + ``make()``. Sessions with no strict NP match are recorded to + ``vr4mice.FailedSession`` and skipped. Downstream code should not fetch ``slope``/``intercept``/``interpol_func`` directly; use ``align_timepoints``/``align_timepoints_lin`` instead. @@ -53,9 +129,9 @@ class BarcodeSync(dj.Computed): definition = """ -> base.Base - -> session_link.RecordingSessionLink - -> acquisition.RecordingProbe --- + recording_id: varchar(255) # Linked NP recording selected for this dataset + probe_serial_number: varchar(64) # Selected NP probe used for alignment skip_first_n_barcodes: int32 # leading VR barcode events excluded from the fit slope: float64 # Slope of the linear fit mapping VR time to NP time intercept: float64 # Intercept of the linear fit mapping VR time to NP time @@ -64,11 +140,11 @@ class BarcodeSync(dj.Computed): barcode_overlap: float64 # Fraction of NP barcodes also found on the VR side """ - key_source = ( - base.Base * session_link.RecordingSessionLink * acquisition.RecordingProbe - & (vr_barcodes.TeensyBarcodes & 'extraction_status = "success"') - & (np_barcodes.ProbeBarcodeExtraction & 'extraction_status = "success"') - ) + @property + def key_source(self): + return dj.U("dataset") & ( + base.Base * (vr_barcodes.TeensyBarcodes & 'extraction_status = "success"') + ) skip_first_n_barcodes = DEFAULT_SKIP_FIRST_N_BARCODES @@ -87,11 +163,51 @@ def make(self, key): return try: + selected = None + selected_modules = None + identity = _dataset_identity(key["dataset"]) + for modules in _np_module_options(): + acq_mod, np_barcodes_mod, session_link_mod = modules + candidates = ( + _candidate_relation_for_modules( + acq_mod, np_barcodes_mod, session_link_mod + ) + & key + ) + if len(candidates) == 0: + continue + selected_modules = modules + candidate_rows = candidates.fetch(as_dict=True) + candidate_rows = sorted( + candidate_rows, + key=lambda row: _candidate_sort_key(row, identity), + ) + selected = candidate_rows[0] + break + + if selected is None or selected_modules is None: + reason = "No eligible NP barcode source for dataset from strict session-link matching" + vr4mice.FailedSession().add_entry( + f"{key['dataset']}", f"{self.__class__.__name__}", reason + ) + logger.warning( + "%s %s for dataset %s", + self.__class__.__name__, + reason, + key["dataset"], + ) + return + + _, np_barcodes_mod, _ = selected_modules + np_key = { + "recording_id": selected["recording_id"], + "probe_serial_number": selected["probe_serial_number"], + } vr_values, vr_times = (vr_barcodes.TeensyBarcodes.Event & key).fetch( "barcode_value", "onset_time_unity", order_by="barcode_index" ) np_values, np_times = ( - np_barcodes.ProbeBarcodeExtraction.Event & key + np_barcodes_mod.ProbeBarcodeExtraction.Event & np_key ).fetch("barcode_value", "onset_time", order_by="barcode_index") fit = align_barcodes( @@ -102,9 +218,16 @@ def make(self, key): skip_first_n_barcodes=self.skip_first_n_barcodes, ) + insert_row = {} + for name in self.heading.names: + if name in key: + insert_row[name] = key[name] + elif name in selected: + insert_row[name] = selected[name] + self.insert1( { - **key, + **insert_row, "skip_first_n_barcodes": self.skip_first_n_barcodes, "slope": fit.slope, "intercept": fit.intercept, diff --git a/tests/unit/test_np_sync.py b/tests/unit/test_np_sync.py index a5cb6bf8..b2dbd6bf 100644 --- a/tests/unit/test_np_sync.py +++ b/tests/unit/test_np_sync.py @@ -4,6 +4,7 @@ import importlib import sys +import ast from pathlib import Path import numpy as np @@ -14,6 +15,7 @@ REPO_ROOT = Path(__file__).parent.parent.parent RUN_PY = REPO_ROOT / "dj_pipeline" / "run.py" CRON_SCENARIO_PY = REPO_ROOT / "dj_pipeline" / "cron_scenario.py" +SCHEMA_NP_SYNC_PY = REPO_ROOT / "dj_pipeline" / "vr4mice" / "schema" / "np_sync.py" def _linear_barcode_streams(*, n=20, slope=2.0, intercept=100.0, skip_vr=0, skip_np=0): @@ -108,6 +110,104 @@ def _function_source(file_text: str, def_line: str) -> str: return def_line + (rest if end == -1 else rest[:end]) +def _top_level_function_source(file_text: str, function_name: str) -> str: + """Return source for a top-level function using AST locations.""" + module = ast.parse(file_text) + for node in module.body: + if isinstance(node, ast.FunctionDef) and node.name == function_name: + return ast.get_source_segment(file_text, node) + raise ValueError(f"Function not found: {function_name}") + + +def _load_schema_helpers(): + """Load pure helper functions from schema np_sync source without importing DB deps.""" + text = SCHEMA_NP_SYNC_PY.read_text() + namespace = {} + for function_name in ( + "_dataset_identity", + "_candidate_sort_key", + ): + exec(_top_level_function_source(text, function_name), namespace) + return namespace + + +def test_schema_dataset_identity_parses(): + dataset_identity = _load_schema_helpers()["_dataset_identity"] + + assert dataset_identity("Xestia_2026-07-01_1") == ( + "Xestia", + "2026-07-01", + 1, + ) + assert dataset_identity("bad_name") is None + + +def test_schema_candidate_sort_key_prefers_identity_then_event_count(): + helper_ns = _load_schema_helpers() + candidate_sort_key = helper_ns["_candidate_sort_key"] + + identity = ("Xestia", "2026-07-01", 2) + + # Exact identity match should beat higher event count with wrong attempt. + exact = { + "mouse_name": "Xestia", + "day": "2026-07-01", + "attempt": 2, + "np_event_count": 10, + "recording_id": "rec_a", + "probe_serial_number": "p1", + } + wrong_attempt_but_more_events = { + "mouse_name": "Xestia", + "day": "2026-07-01", + "attempt": 1, + "np_event_count": 999, + "recording_id": "rec_b", + "probe_serial_number": "p1", + } + + ranked = sorted( + [wrong_attempt_but_more_events, exact], + key=lambda row: candidate_sort_key(row, identity), + ) + assert ranked[0]["recording_id"] == "rec_a" + + # Within the same identity, higher np_event_count should win. + same_identity_low = { + **exact, + "np_event_count": 5, + "recording_id": "rec_low", + } + same_identity_high = { + **exact, + "np_event_count": 25, + "recording_id": "rec_high", + } + ranked_same = sorted( + [same_identity_low, same_identity_high], + key=lambda row: candidate_sort_key(row, identity), + ) + assert ranked_same[0]["recording_id"] == "rec_high" + + +def test_schema_candidate_sort_key_without_identity_prefers_event_count(): + candidate_sort_key = _load_schema_helpers()["_candidate_sort_key"] + + low = { + "np_event_count": 5, + "recording_id": "rec_low", + "probe_serial_number": "p1", + } + high = { + "np_event_count": 50, + "recording_id": "rec_high", + "probe_serial_number": "p2", + } + + ranked = sorted([low, high], key=lambda row: candidate_sort_key(row, None)) + assert ranked[0]["recording_id"] == "rec_high" + + def test_cron_scenario_core_schemas_import_excludes_np_sync(): """import_core_schemas() must not import np_sync, so a missing np_pipeline can't null out the whole behavioral core_schemas tuple (see import_np_sync_schema, From 9875eb2b6537ddf943eb54c1d373d2d18e5b8408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 12 Aug 2026 14:34:01 +0200 Subject: [PATCH 34/40] Update Docker Compose paths for NP pipeline and enhance README documentation --- dj_pipeline/.env.compose.example | 4 +++- dj_pipeline/README.md | 1 + dj_pipeline/docker-compose.yml | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dj_pipeline/.env.compose.example b/dj_pipeline/.env.compose.example index 3cdf8259..f6e73f51 100644 --- a/dj_pipeline/.env.compose.example +++ b/dj_pipeline/.env.compose.example @@ -11,7 +11,9 @@ DB_DATA_PATH=/mnt/database/vr4mice/vr4mice_database/database/ SHARED_PATH=/mnt/database/shared DATA_PATH=/mnt/database/vr4mice/vr4mice_database/data SCREEN_RECORDINGS_PATH=/mnt/neuropixel_data/vr4mice/raw_screen_recordings -NP_PIPELINE_SRC_PATH=../../FreelyMoving_Mouse_NP +# Optional: point to a real FreelyMoving_Mouse_NP checkout to enable np_sync. +# If omitted, docker-compose defaults to ./np_pipeline_stub and np_sync is skipped. +NP_PIPELINE_SRC_PATH=./np_pipeline_stub JUPYTER_PORT=8887 CLIENT_IMAGE=mmathislab/vr4mice_app:0.1.0 CLIENT_CONTAINER_NAME=vr4mice_${USER} diff --git a/dj_pipeline/README.md b/dj_pipeline/README.md index 37abd391..462bb33d 100644 --- a/dj_pipeline/README.md +++ b/dj_pipeline/README.md @@ -297,6 +297,7 @@ When using the local Docker database, `DJ_HOST` port must match `DB_PORT` in `.e - `COMPOSE_PROJECT` (default `vr4mice`) — must match between `make`, cron scripts, and manual `docker compose -p …` calls - `DB_BIND_IP`, `DB_PORT`, `MYSQL_ROOT_PASSWORD` - `DB_DATA_PATH`, `SHARED_PATH`, `DATA_PATH`, `SCREEN_RECORDINGS_PATH` +- `NP_PIPELINE_SRC_PATH` (optional): path to a real `FreelyMoving_Mouse_NP` checkout. If unset, compose mounts `./np_pipeline_stub`, and only `run.py np_sync` is skipped while the rest of the pipeline remains functional. - `CLIENT_IMAGE`, `CLIENT_CONTAINER_NAME`, `DB_CONTAINER_NAME`, `CLIENT_NETWORK_MODE`, `JUPYTER_PORT` Remote/AWS DB credentials for scheduled AWS runs live in `.env-aws` (copy from `.env-aws.example`); this file is **not** committed. diff --git a/dj_pipeline/docker-compose.yml b/dj_pipeline/docker-compose.yml index 72908ade..23a0ceba 100644 --- a/dj_pipeline/docker-compose.yml +++ b/dj_pipeline/docker-compose.yml @@ -39,7 +39,7 @@ services: - "${SCREEN_RECORDINGS_PATH:-/mnt/neuropixel_data/vr4mice/raw_screen_recordings}:/vr4mice_screen_recordings" - "./base/base_min_schemas:/base_schemas" - "./base/base_actions:/base_actions" - - "${NP_PIPELINE_SRC_PATH:-../../FreelyMoving_Mouse_NP}:/np_pipeline" + - "${NP_PIPELINE_SRC_PATH:-./np_pipeline_stub}:/np_pipeline" ports: - "${JUPYTER_PORT:-8887}:8888" working_dir: /app From 4635af2b399905d2f9e730101ac1e9e7db7f5dbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Wed, 12 Aug 2026 15:16:03 +0200 Subject: [PATCH 35/40] Change barcodesync to be related to barcodes np and vr tables --- dj_pipeline/vr4mice/schema/np_sync.py | 86 ++++++++++----------------- 1 file changed, 31 insertions(+), 55 deletions(-) diff --git a/dj_pipeline/vr4mice/schema/np_sync.py b/dj_pipeline/vr4mice/schema/np_sync.py index 8d0dc175..a1abe3f6 100644 --- a/dj_pipeline/vr4mice/schema/np_sync.py +++ b/dj_pipeline/vr4mice/schema/np_sync.py @@ -52,6 +52,11 @@ def _dataset_identity(dataset: str): return None +def _row_day_iso(row): + day = row.get("day") + return day.isoformat() if hasattr(day, "isoformat") else str(day) + + def _candidate_sort_key(row, identity): """Sort candidates by identity match first, then NP barcode event count.""" event_count = int(row["np_event_count"]) @@ -64,9 +69,7 @@ def _candidate_sort_key(row, identity): expected_mouse, expected_day, expected_attempt = identity mouse_miss = 0 if row.get("mouse_name") == expected_mouse else 1 - day = row.get("day") - day_iso = day.isoformat() if hasattr(day, "isoformat") else str(day) - day_miss = 0 if day_iso == expected_day else 1 + day_miss = 0 if _row_day_iso(row) == expected_day else 1 attempt_miss = 0 if row.get("attempt") == expected_attempt else 1 return ( mouse_miss, @@ -78,27 +81,11 @@ def _candidate_sort_key(row, identity): ) -def _np_module_options(): - """Return available NP schema-module triplets, preferring imported modules.""" - options = [(acquisition, np_barcodes, session_link)] - legacy = ( - dj.VirtualModule("np_acquisition_legacy", "acquisition"), - dj.VirtualModule("np_barcodes_legacy", "barcodes"), - dj.VirtualModule("np_session_link_legacy", "session_link"), - ) - imported_lineage = acquisition.RecordingProbe.heading.attributes[ - "recording_id" - ].lineage - legacy_lineage = legacy[0].RecordingProbe.heading.attributes["recording_id"].lineage - if legacy_lineage != imported_lineage: - options.append(legacy) - return options - - -def _candidate_relation_for_modules(acq_mod, np_barcodes_mod, session_link_mod): +def _candidate_relation(): """Rows where one VR dataset has at least one NP probe with successful barcodes.""" + vr_success = vr_barcodes.TeensyBarcodes & 'extraction_status = "success"' np_success = ( - np_barcodes_mod.ProbeBarcodeExtraction & 'extraction_status = "success"' + np_barcodes.ProbeBarcodeExtraction & 'extraction_status = "success"' ).proj( "recording_id", "probe_serial_number", @@ -106,10 +93,10 @@ def _candidate_relation_for_modules(acq_mod, np_barcodes_mod, session_link_mod): ) return ( base.Base - * acq_mod.RecordingProbe - * session_link_mod.RecordingSessionLink + * vr_success + * acquisition.RecordingProbe + * session_link.RecordingSessionLink * np_success - & (vr_barcodes.TeensyBarcodes & 'extraction_status = "success"') ) @@ -117,21 +104,20 @@ def _candidate_relation_for_modules(acq_mod, np_barcodes_mod, session_link_mod): class BarcodeSync(dj.Computed): """Linear fit + interpolator mapping VR Unity/game time to NP probe time. - One row per VR dataset that has at least one linked NP recording/probe with - successful barcodes on both sides. ``key_source`` enumerates VR datasets with - successful VR barcode extraction; strict NP-side eligibility is resolved in - ``make()``. Sessions with no strict NP match are recorded to - ``vr4mice.FailedSession`` and skipped. + One row per related VR/NP barcode pair with successful extraction on both sides. + The table is explicitly keyed by ``TeensyBarcodes`` (VR side) and + ``ProbeBarcodeExtraction`` (NP side). ``key_source`` enforces strict matching + through ``base.Base`` + ``RecordingSessionLink`` + ``recording_id`` so + ``populate()`` only visits related rows. Downstream code should not fetch ``slope``/``intercept``/``interpol_func`` directly; use ``align_timepoints``/``align_timepoints_lin`` instead. """ definition = """ - -> base.Base + -> vr_barcodes.TeensyBarcodes + -> np_barcodes.ProbeBarcodeExtraction --- - recording_id: varchar(255) # Linked NP recording selected for this dataset - probe_serial_number: varchar(64) # Selected NP probe used for alignment skip_first_n_barcodes: int32 # leading VR barcode events excluded from the fit slope: float64 # Slope of the linear fit mapping VR time to NP time intercept: float64 # Intercept of the linear fit mapping VR time to NP time @@ -142,9 +128,7 @@ class BarcodeSync(dj.Computed): @property def key_source(self): - return dj.U("dataset") & ( - base.Base * (vr_barcodes.TeensyBarcodes & 'extraction_status = "success"') - ) + return dj.U(*self.primary_key) & _candidate_relation() skip_first_n_barcodes = DEFAULT_SKIP_FIRST_N_BARCODES @@ -164,28 +148,16 @@ def make(self, key): try: selected = None - selected_modules = None identity = _dataset_identity(key["dataset"]) - for modules in _np_module_options(): - acq_mod, np_barcodes_mod, session_link_mod = modules - candidates = ( - _candidate_relation_for_modules( - acq_mod, np_barcodes_mod, session_link_mod - ) - & key - ) - if len(candidates) == 0: - continue - selected_modules = modules + candidates = _candidate_relation() & key + if len(candidates) != 0: candidate_rows = candidates.fetch(as_dict=True) - candidate_rows = sorted( + selected = min( candidate_rows, key=lambda row: _candidate_sort_key(row, identity), ) - selected = candidate_rows[0] - break - if selected is None or selected_modules is None: + if selected is None: reason = "No eligible NP barcode source for dataset from strict session-link matching" vr4mice.FailedSession().add_entry( f"{key['dataset']}", f"{self.__class__.__name__}", reason @@ -198,16 +170,20 @@ def make(self, key): ) return - _, np_barcodes_mod, _ = selected_modules np_key = { "recording_id": selected["recording_id"], "probe_serial_number": selected["probe_serial_number"], } - vr_values, vr_times = (vr_barcodes.TeensyBarcodes.Event & key).fetch( + vr_key = { + name: key[name] + for name in vr_barcodes.TeensyBarcodes.primary_key + if name in key + } + vr_values, vr_times = (vr_barcodes.TeensyBarcodes.Event & vr_key).fetch( "barcode_value", "onset_time_unity", order_by="barcode_index" ) np_values, np_times = ( - np_barcodes_mod.ProbeBarcodeExtraction.Event & np_key + np_barcodes.ProbeBarcodeExtraction.Event & np_key ).fetch("barcode_value", "onset_time", order_by="barcode_index") fit = align_barcodes( From 6d260d9df657cd93ef20d3cba7d5c8a2341b465c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Thu, 13 Aug 2026 09:40:21 +0200 Subject: [PATCH 36/40] Fix BarcodeSync documentation and tests for VR-only dataset handling and NP event absence --- dj_pipeline/vr4mice/schema/np_sync.py | 38 ++++++++++++++---- docs/software/datajoint.md | 6 +-- tests/unit/test_np_sync.py | 57 +++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 11 deletions(-) diff --git a/dj_pipeline/vr4mice/schema/np_sync.py b/dj_pipeline/vr4mice/schema/np_sync.py index a1abe3f6..240b88dc 100644 --- a/dj_pipeline/vr4mice/schema/np_sync.py +++ b/dj_pipeline/vr4mice/schema/np_sync.py @@ -1,11 +1,10 @@ """Alignment of VR (behavior) time to Neuropixels (NP) native time via shared barcodes. Not every VR ``Dataset`` has a corresponding NP recording. ``BarcodeSync.key_source`` -includes datasets with successful VR barcode extraction, while strict NP linkage -matching is performed inside ``make()`` via ``base.Base`` + -``session_link.RecordingSessionLink`` + successful NP probe barcode extraction. -Datasets with no strict NP match are recorded in ``vr4mice.FailedSession`` and -skipped cleanly. +contains only strict VR/NP matches (via ``base.Base`` + +``session_link.RecordingSessionLink`` + successful NP probe barcode extraction), +so VR-only datasets are excluded from this table and are not visited by +``populate()``. Cross-repo foreign keys: ``BarcodeSync`` references ``np_pipeline`` tables directly via ``-> ``. This requires ``vr4mice`` and ``np_pipeline`` to share one @@ -108,7 +107,8 @@ class BarcodeSync(dj.Computed): The table is explicitly keyed by ``TeensyBarcodes`` (VR side) and ``ProbeBarcodeExtraction`` (NP side). ``key_source`` enforces strict matching through ``base.Base`` + ``RecordingSessionLink`` + ``recording_id`` so - ``populate()`` only visits related rows. + ``populate()`` only visits related rows; sessions with no strict NP match are + invisible to this table. Downstream code should not fetch ``slope``/``intercept``/``interpol_func`` directly; use ``align_timepoints``/``align_timepoints_lin`` instead. @@ -158,7 +158,10 @@ def make(self, key): ) if selected is None: - reason = "No eligible NP barcode source for dataset from strict session-link matching" + reason = ( + "No eligible NP barcode source after key selection " + "(possible concurrent data change)" + ) vr4mice.FailedSession().add_entry( f"{key['dataset']}", f"{self.__class__.__name__}", reason ) @@ -186,6 +189,23 @@ def make(self, key): np_barcodes.ProbeBarcodeExtraction.Event & np_key ).fetch("barcode_value", "onset_time", order_by="barcode_index") + if len(np_values) == 0: + reason = ( + "No NP barcode events found for key at populate time " + "(events may have been removed after key_source selection)" + ) + vr4mice.FailedSession().add_entry( + f"{key['dataset']}", f"{self.__class__.__name__}", reason + ) + logger.warning( + "%s %s. key: %s, np_key: %s", + self.__class__.__name__, + reason, + key, + np_key, + ) + return + fit = align_barcodes( vr_times, vr_values, @@ -252,7 +272,9 @@ def align_timepoints(cls, key, timepoints: list): A list of NP-side times (or `None`), same length and order as `timepoints`. """ interpol_func = pickle.loads((cls & key).fetch1("interpol_func")) - timepoints = np.array(timepoints, dtype=np.float64) + timepoints = np.array( + [np.nan if tx is None else tx for tx in timepoints], dtype=np.float64 + ) return [ float(tx) if not np.isnan(tx) else None for tx in interpol_func(timepoints) ] diff --git a/docs/software/datajoint.md b/docs/software/datajoint.md index f7cab513..e3b6a382 100644 --- a/docs/software/datajoint.md +++ b/docs/software/datajoint.md @@ -421,10 +421,10 @@ Barcode events decoded from the TTL signal sampled by the Teensy. ```python class BarcodeSync(dj.Computed) ``` -**Depends on:** `base.Base`, `np_pipeline.schemas.session_link.RecordingSessionLink`, `np_pipeline.schemas.acquisition.OneBoxDaq` -Fits a linear regression + interpolator mapping VR Unity/game time to NP OneBox DAQ +**Depends on:** `base.Base`, `np_pipeline.schemas.acquisition.RecordingProbe`, `np_pipeline.schemas.session_link.RecordingSessionLink`, `np_pipeline.schemas.barcodes.ProbeBarcodeExtraction` +Fits a linear regression + interpolator mapping VR Unity/game time to NP probe time, from barcode values shared between `barcodes.TeensyBarcodes.Event` and -`np_pipeline`'s `OneBoxBarcodeExtraction.Event`. `key_source` is restricted to +`np_pipeline`'s `ProbeBarcodeExtraction.Event`. `key_source` is restricted to datasets with a linked, successfully barcode-decoded NP recording, so `populate()` is a no-op — not an error — for VR-only sessions with no neural data. Use `align_timepoints`/`align_timepoints_lin` to convert VR times to NP times. diff --git a/tests/unit/test_np_sync.py b/tests/unit/test_np_sync.py index b2dbd6bf..ad6cfa9c 100644 --- a/tests/unit/test_np_sync.py +++ b/tests/unit/test_np_sync.py @@ -5,6 +5,8 @@ import importlib import sys import ast +import textwrap +import pickle from pathlib import Path import numpy as np @@ -125,12 +127,35 @@ def _load_schema_helpers(): namespace = {} for function_name in ( "_dataset_identity", + "_row_day_iso", "_candidate_sort_key", ): exec(_top_level_function_source(text, function_name), namespace) return namespace +def _load_schema_align_timepoints_method(): + """Load BarcodeSync.align_timepoints from source without importing DB deps.""" + text = SCHEMA_NP_SYNC_PY.read_text() + module = ast.parse(text) + + method_src = None + for node in module.body: + if isinstance(node, ast.ClassDef) and node.name == "BarcodeSync": + for item in node.body: + if isinstance(item, ast.FunctionDef) and item.name == "align_timepoints": + method_src = ast.get_source_segment(text, item) + break + break + + if method_src is None: + raise ValueError("BarcodeSync.align_timepoints not found") + + namespace = {"np": np, "pickle": pickle} + exec(textwrap.dedent(method_src), namespace) + return namespace["align_timepoints"] + + def test_schema_dataset_identity_parses(): dataset_identity = _load_schema_helpers()["_dataset_identity"] @@ -208,6 +233,23 @@ def test_schema_candidate_sort_key_without_identity_prefers_event_count(): assert ranked[0]["recording_id"] == "rec_high" +def test_schema_align_timepoints_preserves_none_entries(): + align_timepoints = _load_schema_align_timepoints_method() + + class _FakeRelation: + def fetch1(self, field_name): + assert field_name == "interpol_func" + return pickle.dumps(np.square) + + class _FakeCls: + def __and__(self, key): + assert key == {"dataset": "dummy"} + return _FakeRelation() + + aligned = align_timepoints(_FakeCls(), {"dataset": "dummy"}, [1.0, None, 2.0]) + assert aligned == [1.0, None, 4.0] + + def test_cron_scenario_core_schemas_import_excludes_np_sync(): """import_core_schemas() must not import np_sync, so a missing np_pipeline can't null out the whole behavioral core_schemas tuple (see import_np_sync_schema, @@ -230,3 +272,18 @@ def test_run_py_np_sync_mode_catches_broad_exception(): assert "from vr4mice.schema import np_sync" in block assert "except Exception as err:" in block assert "except ModuleNotFoundError" not in block + + +def test_schema_make_handles_empty_np_events_with_clear_reason(): + text = SCHEMA_NP_SYNC_PY.read_text() + make_src = _function_source(text, " def make(self, key):") + + assert "if len(np_values) == 0:" in make_src + assert "No NP barcode events found for key at populate time" in make_src + + +def test_schema_module_docstring_states_vr_only_sessions_excluded_from_key_source(): + text = SCHEMA_NP_SYNC_PY.read_text() + + assert "so VR-only datasets are excluded from this table" in text + assert "are not visited by" in text From b8942b1b5f4bdaf4ae6bc79e32c78e3726814f6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Thu, 13 Aug 2026 10:22:38 +0200 Subject: [PATCH 37/40] Fix cron scenario np_sync absent --- dj_pipeline/cron_scenario.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/dj_pipeline/cron_scenario.py b/dj_pipeline/cron_scenario.py index 3823d86d..f6e3e479 100755 --- a/dj_pipeline/cron_scenario.py +++ b/dj_pipeline/cron_scenario.py @@ -290,17 +290,27 @@ def import_core_schemas(): lambda: summary_emails.send_pending_summary_emails(logger=logger), ) + def import_decision_schema(): + from vr4mice.schema import decision + + return decision + def import_np_sync_schema(): - # np_sync depends on the separate np_pipeline package/repo, unlike the - # core_schemas above. Imported in its own run_import call (not bundled - # into import_core_schemas) so a missing/unreachable NP repo only skips - # this one step, without nulling out the rest of the behavioral pipeline. from vr4mice.schema import np_sync return np_sync - np_sync = run_import("import np_sync schema", import_np_sync_schema) - if np_sync: + # np_sync depends on the separate np_pipeline package/repo and should not + # fail the whole cron scenario when unavailable in a deployment/CI setup. + try: + np_sync = import_np_sync_schema() + except Exception as err: + logger.warning( + "Skipping np_sync: np_pipeline is not available (%s). " + "Behavioral analysis is unaffected.", + err, + ) + else: from vr4mice.utils.populate_helpers import populate_pending run_step( @@ -312,11 +322,6 @@ def import_np_sync_schema(): ), ) - def import_decision_schema(): - from vr4mice.schema import decision - - return decision - decision = run_import("import decision schema", import_decision_schema) if decision: run_step( From d1a96c70a63e3d03ece5964e523616a453611aae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Thu, 13 Aug 2026 13:15:40 +0200 Subject: [PATCH 38/40] Implement artur's comments --- dj_pipeline/cron_scenario.py | 2 +- dj_pipeline/docker-compose.yml | 2 +- dj_pipeline/run.py | 2 +- dj_pipeline/vr4mice/schema/np_sync.py | 130 ++++++++------------------ tests/unit/test_np_sync.py | 117 ++++------------------- 5 files changed, 60 insertions(+), 193 deletions(-) diff --git a/dj_pipeline/cron_scenario.py b/dj_pipeline/cron_scenario.py index f6e3e479..a46dde35 100755 --- a/dj_pipeline/cron_scenario.py +++ b/dj_pipeline/cron_scenario.py @@ -317,7 +317,7 @@ def import_np_sync_schema(): "np_sync.BarcodeSync.populate", lambda: populate_pending( np_sync.BarcodeSync, - np_sync.BarcodeSync.key_source, + np_sync.BarcodeSync().key_source, logger=logger, ), ) diff --git a/dj_pipeline/docker-compose.yml b/dj_pipeline/docker-compose.yml index 23a0ceba..a61b01c8 100644 --- a/dj_pipeline/docker-compose.yml +++ b/dj_pipeline/docker-compose.yml @@ -39,7 +39,7 @@ services: - "${SCREEN_RECORDINGS_PATH:-/mnt/neuropixel_data/vr4mice/raw_screen_recordings}:/vr4mice_screen_recordings" - "./base/base_min_schemas:/base_schemas" - "./base/base_actions:/base_actions" - - "${NP_PIPELINE_SRC_PATH:-./np_pipeline_stub}:/np_pipeline" + - "${NP_PIPELINE_SRC_PATH:-./np_pipeline_stub}:/app/np_pipeline" ports: - "${JUPYTER_PORT:-8887}:8888" working_dir: /app diff --git a/dj_pipeline/run.py b/dj_pipeline/run.py index 90c70b3a..ff6f5214 100644 --- a/dj_pipeline/run.py +++ b/dj_pipeline/run.py @@ -304,7 +304,7 @@ def check_folder_existence(folder_path): populate_pending( np_sync.BarcodeSync, - np_sync.BarcodeSync.key_source, + np_sync.BarcodeSync().key_source, logger=logger, ) diff --git a/dj_pipeline/vr4mice/schema/np_sync.py b/dj_pipeline/vr4mice/schema/np_sync.py index 240b88dc..3534146d 100644 --- a/dj_pipeline/vr4mice/schema/np_sync.py +++ b/dj_pipeline/vr4mice/schema/np_sync.py @@ -20,12 +20,12 @@ import numpy as np try: - from np_pipeline.schemas import acquisition, barcodes as np_barcodes, session_link + from np_pipeline.schemas import barcodes as np_barcodes, session_link except ModuleNotFoundError: import sys sys.path.insert(0, "/np_pipeline") - from np_pipeline.schemas import acquisition, barcodes as np_barcodes, session_link + from np_pipeline.schemas import barcodes as np_barcodes, session_link from vr4mice.analysis.np_sync import DEFAULT_SKIP_FIRST_N_BARCODES, align_barcodes from vr4mice.schema import base @@ -40,64 +40,19 @@ logger = Logger.get_logger() -def _dataset_identity(dataset: str): - """Return (mouse_name, day_iso, attempt) parsed from dataset name.""" - parts = dataset.split("_") - if len(parts) < 3: - return None - try: - return parts[0], parts[1], int(parts[2]) - except ValueError: - return None - - -def _row_day_iso(row): - day = row.get("day") - return day.isoformat() if hasattr(day, "isoformat") else str(day) - - -def _candidate_sort_key(row, identity): - """Sort candidates by identity match first, then NP barcode event count.""" - event_count = int(row["np_event_count"]) - if identity is None: - return ( - -event_count, - str(row.get("recording_id", "")), - str(row.get("probe_serial_number", "")), - ) - - expected_mouse, expected_day, expected_attempt = identity - mouse_miss = 0 if row.get("mouse_name") == expected_mouse else 1 - day_miss = 0 if _row_day_iso(row) == expected_day else 1 - attempt_miss = 0 if row.get("attempt") == expected_attempt else 1 - return ( - mouse_miss, - day_miss, - attempt_miss, - -event_count, - str(row.get("recording_id", "")), - str(row.get("probe_serial_number", "")), - ) - - def _candidate_relation(): """Rows where one VR dataset has at least one NP probe with successful barcodes.""" - vr_success = vr_barcodes.TeensyBarcodes & 'extraction_status = "success"' - np_success = ( - np_barcodes.ProbeBarcodeExtraction & 'extraction_status = "success"' - ).proj( - "recording_id", - "probe_serial_number", - np_event_count="event_count", - ) - return ( + vr = ( base.Base - * vr_success - * acquisition.RecordingProbe - * session_link.RecordingSessionLink - * np_success + * (vr_barcodes.TeensyBarcodes & 'extraction_status = "success"').proj() + ) + npx = ( + session_link.RecordingSessionLink + * (np_barcodes.ProbeBarcodeExtraction & 'extraction_status = "success"').proj() ) + return vr * npx + @schema class BarcodeSync(dj.Computed): @@ -128,7 +83,13 @@ class BarcodeSync(dj.Computed): @property def key_source(self): - return dj.U(*self.primary_key) & _candidate_relation() + source = _candidate_relation() + unexpected = sorted(set(source.heading.primary_key) - set(self.primary_key)) + if unexpected: + raise dj.DataJointError( + f"key_source primary key has attributes BarcodeSync lacks: {unexpected}" + ) + return source skip_first_n_barcodes = DEFAULT_SKIP_FIRST_N_BARCODES @@ -147,21 +108,21 @@ def make(self, key): return try: - selected = None - identity = _dataset_identity(key["dataset"]) - candidates = _candidate_relation() & key - if len(candidates) != 0: - candidate_rows = candidates.fetch(as_dict=True) - selected = min( - candidate_rows, - key=lambda row: _candidate_sort_key(row, identity), + candidate_rows = (_candidate_relation() & key).to_dicts() + if not candidate_rows: + reason = "No NP candidate matched key fields" + vr4mice.FailedSession().add_entry( + f"{key['dataset']}", f"{self.__class__.__name__}", reason ) - - if selected is None: - reason = ( - "No eligible NP barcode source after key selection " - "(possible concurrent data change)" + logger.warning( + "%s %s for dataset %s", + self.__class__.__name__, + reason, + key["dataset"], ) + return + if len(candidate_rows) > 1: + reason = "Ambiguous NP candidates matched key fields" vr4mice.FailedSession().add_entry( f"{key['dataset']}", f"{self.__class__.__name__}", reason ) @@ -173,21 +134,12 @@ def make(self, key): ) return - np_key = { - "recording_id": selected["recording_id"], - "probe_serial_number": selected["probe_serial_number"], - } - vr_key = { - name: key[name] - for name in vr_barcodes.TeensyBarcodes.primary_key - if name in key - } - vr_values, vr_times = (vr_barcodes.TeensyBarcodes.Event & vr_key).fetch( + vr_values, vr_times = (vr_barcodes.TeensyBarcodes.Event & key).to_arrays( "barcode_value", "onset_time_unity", order_by="barcode_index" ) np_values, np_times = ( - np_barcodes.ProbeBarcodeExtraction.Event & np_key - ).fetch("barcode_value", "onset_time", order_by="barcode_index") + np_barcodes.ProbeBarcodeExtraction.Event & key + ).to_arrays("barcode_value", "onset_time", order_by="barcode_index") if len(np_values) == 0: reason = ( @@ -198,11 +150,10 @@ def make(self, key): f"{key['dataset']}", f"{self.__class__.__name__}", reason ) logger.warning( - "%s %s. key: %s, np_key: %s", + "%s %s. key: %s", self.__class__.__name__, reason, key, - np_key, ) return @@ -214,22 +165,19 @@ def make(self, key): skip_first_n_barcodes=self.skip_first_n_barcodes, ) - insert_row = {} - for name in self.heading.names: - if name in key: - insert_row[name] = key[name] - elif name in selected: - insert_row[name] = selected[name] + # Measure overlap on full streams (independent of fit-time skipping) + # so DEFAULT_SKIP_FIRST_N_BARCODES does not bias this quality metric. + full_shared_barcodes = np.intersect1d(vr_values, np_values) self.insert1( { - **insert_row, + **key, "skip_first_n_barcodes": self.skip_first_n_barcodes, "slope": fit.slope, "intercept": fit.intercept, "r2": fit.r2, "interpol_func": pickle.dumps(fit.interpol_func), - "barcode_overlap": len(fit.shared_barcodes) / len(np_values), + "barcode_overlap": len(full_shared_barcodes) / len(np_values), } ) logger.info( diff --git a/tests/unit/test_np_sync.py b/tests/unit/test_np_sync.py index ad6cfa9c..6fef02f8 100644 --- a/tests/unit/test_np_sync.py +++ b/tests/unit/test_np_sync.py @@ -112,28 +112,6 @@ def _function_source(file_text: str, def_line: str) -> str: return def_line + (rest if end == -1 else rest[:end]) -def _top_level_function_source(file_text: str, function_name: str) -> str: - """Return source for a top-level function using AST locations.""" - module = ast.parse(file_text) - for node in module.body: - if isinstance(node, ast.FunctionDef) and node.name == function_name: - return ast.get_source_segment(file_text, node) - raise ValueError(f"Function not found: {function_name}") - - -def _load_schema_helpers(): - """Load pure helper functions from schema np_sync source without importing DB deps.""" - text = SCHEMA_NP_SYNC_PY.read_text() - namespace = {} - for function_name in ( - "_dataset_identity", - "_row_day_iso", - "_candidate_sort_key", - ): - exec(_top_level_function_source(text, function_name), namespace) - return namespace - - def _load_schema_align_timepoints_method(): """Load BarcodeSync.align_timepoints from source without importing DB deps.""" text = SCHEMA_NP_SYNC_PY.read_text() @@ -156,83 +134,6 @@ def _load_schema_align_timepoints_method(): return namespace["align_timepoints"] -def test_schema_dataset_identity_parses(): - dataset_identity = _load_schema_helpers()["_dataset_identity"] - - assert dataset_identity("Xestia_2026-07-01_1") == ( - "Xestia", - "2026-07-01", - 1, - ) - assert dataset_identity("bad_name") is None - - -def test_schema_candidate_sort_key_prefers_identity_then_event_count(): - helper_ns = _load_schema_helpers() - candidate_sort_key = helper_ns["_candidate_sort_key"] - - identity = ("Xestia", "2026-07-01", 2) - - # Exact identity match should beat higher event count with wrong attempt. - exact = { - "mouse_name": "Xestia", - "day": "2026-07-01", - "attempt": 2, - "np_event_count": 10, - "recording_id": "rec_a", - "probe_serial_number": "p1", - } - wrong_attempt_but_more_events = { - "mouse_name": "Xestia", - "day": "2026-07-01", - "attempt": 1, - "np_event_count": 999, - "recording_id": "rec_b", - "probe_serial_number": "p1", - } - - ranked = sorted( - [wrong_attempt_but_more_events, exact], - key=lambda row: candidate_sort_key(row, identity), - ) - assert ranked[0]["recording_id"] == "rec_a" - - # Within the same identity, higher np_event_count should win. - same_identity_low = { - **exact, - "np_event_count": 5, - "recording_id": "rec_low", - } - same_identity_high = { - **exact, - "np_event_count": 25, - "recording_id": "rec_high", - } - ranked_same = sorted( - [same_identity_low, same_identity_high], - key=lambda row: candidate_sort_key(row, identity), - ) - assert ranked_same[0]["recording_id"] == "rec_high" - - -def test_schema_candidate_sort_key_without_identity_prefers_event_count(): - candidate_sort_key = _load_schema_helpers()["_candidate_sort_key"] - - low = { - "np_event_count": 5, - "recording_id": "rec_low", - "probe_serial_number": "p1", - } - high = { - "np_event_count": 50, - "recording_id": "rec_high", - "probe_serial_number": "p2", - } - - ranked = sorted([low, high], key=lambda row: candidate_sort_key(row, None)) - assert ranked[0]["recording_id"] == "rec_high" - - def test_schema_align_timepoints_preserves_none_entries(): align_timepoints = _load_schema_align_timepoints_method() @@ -282,6 +183,24 @@ def test_schema_make_handles_empty_np_events_with_clear_reason(): assert "No NP barcode events found for key at populate time" in make_src +def test_schema_make_uses_key_only_without_identity_parsing(): + text = SCHEMA_NP_SYNC_PY.read_text() + make_src = _function_source(text, " def make(self, key):") + + assert "_dataset_identity" not in make_src + assert "_identity_matches_row" not in make_src + assert "_select_rows_by_np_event_count" not in make_src + assert "No NP candidate matched key fields" in make_src + + +def test_schema_make_fails_ambiguous_matches_directly_without_event_count_tie_break(): + text = SCHEMA_NP_SYNC_PY.read_text() + make_src = _function_source(text, " def make(self, key):") + + assert "Ambiguous NP candidates matched key fields" in make_src + assert "event_count matching" not in make_src + + def test_schema_module_docstring_states_vr_only_sessions_excluded_from_key_source(): text = SCHEMA_NP_SYNC_PY.read_text() From dfae17f3412c97bad103d49d3cee3f44843b33b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Thu, 13 Aug 2026 15:04:18 +0200 Subject: [PATCH 39/40] Update pythonpath --- dj_pipeline/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dj_pipeline/docker-compose.yml b/dj_pipeline/docker-compose.yml index a61b01c8..db0a74ae 100644 --- a/dj_pipeline/docker-compose.yml +++ b/dj_pipeline/docker-compose.yml @@ -25,7 +25,7 @@ services: - MPLCONFIGDIR=/app/.cache/matplotlib - XDG_CONFIG_HOME=/app/.cache - PATH=/app/.local/bin:/usr/local/bin:/usr/bin:/bin - - PYTHONPATH=/np_pipeline/src:${PYTHONPATH} + - PYTHONPATH=app/np_pipeline/src:${PYTHONPATH} - TAG=${TAG} network_mode: ${CLIENT_NETWORK_MODE:-host} # Run via bash so the bind-mounted script does not need +x on shared/NFS checkouts. From 83421b6daf0e2ff7910f1d76acdbed228a1dfb03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lia=20Benquet?= Date: Thu, 13 Aug 2026 15:22:25 +0200 Subject: [PATCH 40/40] Enhance NP-VR synchronization with quality gates and resilient imports - Updated PYTHONPATH in docker-compose for better path resolution. - Added minimum shared barcodes and overlap checks in np_sync for reliable alignment. - Included np_sync schema in maintenance utility. - Revised documentation to reflect updated dependencies and functionality. - Added unit tests for new quality gate features in np_sync. --- dj_pipeline/docker-compose.yml | 2 +- dj_pipeline/vr4mice/schema/np_sync.py | 44 ++++++++++++++++++++++-- dj_pipeline/vr4mice/utils/maintenance.py | 2 ++ docs/software/datajoint.md | 2 +- tests/unit/test_np_sync.py | 10 ++++++ 5 files changed, 56 insertions(+), 4 deletions(-) diff --git a/dj_pipeline/docker-compose.yml b/dj_pipeline/docker-compose.yml index db0a74ae..5fe01ab6 100644 --- a/dj_pipeline/docker-compose.yml +++ b/dj_pipeline/docker-compose.yml @@ -25,7 +25,7 @@ services: - MPLCONFIGDIR=/app/.cache/matplotlib - XDG_CONFIG_HOME=/app/.cache - PATH=/app/.local/bin:/usr/local/bin:/usr/bin:/bin - - PYTHONPATH=app/np_pipeline/src:${PYTHONPATH} + - PYTHONPATH=/app/np_pipeline/src:/app/np_pipeline:${PYTHONPATH} - TAG=${TAG} network_mode: ${CLIENT_NETWORK_MODE:-host} # Run via bash so the bind-mounted script does not need +x on shared/NFS checkouts. diff --git a/dj_pipeline/vr4mice/schema/np_sync.py b/dj_pipeline/vr4mice/schema/np_sync.py index 3534146d..62ddcecf 100644 --- a/dj_pipeline/vr4mice/schema/np_sync.py +++ b/dj_pipeline/vr4mice/schema/np_sync.py @@ -24,7 +24,9 @@ except ModuleNotFoundError: import sys - sys.path.insert(0, "/np_pipeline") + # Keep np_pipeline imports resilient when process CWD is not /app. + sys.path.insert(0, "/app/np_pipeline/src") + sys.path.insert(0, "/app/np_pipeline") from np_pipeline.schemas import barcodes as np_barcodes, session_link from vr4mice.analysis.np_sync import DEFAULT_SKIP_FIRST_N_BARCODES, align_barcodes @@ -92,6 +94,8 @@ def key_source(self): return source skip_first_n_barcodes = DEFAULT_SKIP_FIRST_N_BARCODES + min_shared_barcodes = 20 + min_barcode_overlap = 0.90 def make(self, key): """Fit and insert one VR-time-to-NP-time alignment. @@ -168,6 +172,42 @@ def make(self, key): # Measure overlap on full streams (independent of fit-time skipping) # so DEFAULT_SKIP_FIRST_N_BARCODES does not bias this quality metric. full_shared_barcodes = np.intersect1d(vr_values, np_values) + np_unique_barcodes = np.unique(np_values) + barcode_overlap = len(full_shared_barcodes) / len(np_unique_barcodes) + + if len(fit.shared_barcodes) < self.min_shared_barcodes: + reason = ( + "Insufficient shared barcodes for reliable NP-VR alignment " + f"(shared={len(fit.shared_barcodes)}, " + f"min_required={self.min_shared_barcodes})" + ) + vr4mice.FailedSession().add_entry( + f"{key['dataset']}", f"{self.__class__.__name__}", reason + ) + logger.warning( + "%s %s for dataset %s", + self.__class__.__name__, + reason, + key["dataset"], + ) + return + + if barcode_overlap <= self.min_barcode_overlap: + reason = ( + "Insufficient NP-VR barcode overlap for reliable alignment " + f"(overlap={barcode_overlap:.4f}, " + f"min_required>{self.min_barcode_overlap:.4f})" + ) + vr4mice.FailedSession().add_entry( + f"{key['dataset']}", f"{self.__class__.__name__}", reason + ) + logger.warning( + "%s %s for dataset %s", + self.__class__.__name__, + reason, + key["dataset"], + ) + return self.insert1( { @@ -177,7 +217,7 @@ def make(self, key): "intercept": fit.intercept, "r2": fit.r2, "interpol_func": pickle.dumps(fit.interpol_func), - "barcode_overlap": len(full_shared_barcodes) / len(np_values), + "barcode_overlap": barcode_overlap, } ) logger.info( diff --git a/dj_pipeline/vr4mice/utils/maintenance.py b/dj_pipeline/vr4mice/utils/maintenance.py index 67c1c7c5..654915f7 100644 --- a/dj_pipeline/vr4mice/utils/maintenance.py +++ b/dj_pipeline/vr4mice/utils/maintenance.py @@ -41,6 +41,7 @@ def _schema_pairs() -> Iterable[Tuple[str, object]]: inputs_videos, interpolated_trajectories, latency_tests, + np_sync, session_metrics, summary_emails, vr4mice, @@ -55,6 +56,7 @@ def _schema_pairs() -> Iterable[Tuple[str, object]]: ("summary_emails", summary_emails.schema), ("dlc", dlc.schema), ("barcodes", barcodes.schema), + ("np_sync", np_sync.schema), ("session_metrics", session_metrics.schema), ("interpolated_trajectories", interpolated_trajectories.schema), ("latency_tests", latency_tests.schema), diff --git a/docs/software/datajoint.md b/docs/software/datajoint.md index e3b6a382..2ae410d2 100644 --- a/docs/software/datajoint.md +++ b/docs/software/datajoint.md @@ -421,7 +421,7 @@ Barcode events decoded from the TTL signal sampled by the Teensy. ```python class BarcodeSync(dj.Computed) ``` -**Depends on:** `base.Base`, `np_pipeline.schemas.acquisition.RecordingProbe`, `np_pipeline.schemas.session_link.RecordingSessionLink`, `np_pipeline.schemas.barcodes.ProbeBarcodeExtraction` +**Depends on:** `barcodes.TeensyBarcodes`, `np_pipeline.schemas.barcodes.ProbeBarcodeExtraction` (table key), plus `base.Base` and `np_pipeline.schemas.session_link.RecordingSessionLink` in `key_source` filtering Fits a linear regression + interpolator mapping VR Unity/game time to NP probe time, from barcode values shared between `barcodes.TeensyBarcodes.Event` and `np_pipeline`'s `ProbeBarcodeExtraction.Event`. `key_source` is restricted to diff --git a/tests/unit/test_np_sync.py b/tests/unit/test_np_sync.py index 6fef02f8..991c7f29 100644 --- a/tests/unit/test_np_sync.py +++ b/tests/unit/test_np_sync.py @@ -201,6 +201,16 @@ def test_schema_make_fails_ambiguous_matches_directly_without_event_count_tie_br assert "event_count matching" not in make_src +def test_schema_make_has_quality_gate_for_min_shared_and_overlap(): + text = SCHEMA_NP_SYNC_PY.read_text() + make_src = _function_source(text, " def make(self, key):") + + assert "min_shared_barcodes = 20" in text + assert "min_barcode_overlap = 0.90" in text + assert "Insufficient shared barcodes for reliable NP-VR alignment" in make_src + assert "Insufficient NP-VR barcode overlap for reliable alignment" in make_src + + def test_schema_module_docstring_states_vr_only_sessions_excluded_from_key_source(): text = SCHEMA_NP_SYNC_PY.read_text()