From 6ccb5b3948c0510a29a85cc51e9b42f1fe61516a Mon Sep 17 00:00:00 2001
From: yodem <yotam@sefaria.org>
Date: Wed, 12 Aug 2026 18:04:56 +0300
Subject: [PATCH] fix(helm): verify mongo restore completeness and allow
 skipping collections

Nothing in the restore pipeline ever checked that mongorestore produced a
complete database, so a truncated restore was indistinguishable from a good
one. A cauldron came up with 8 of 73 collections and still reported healthy.

Two changes to the restore hook.

restore.verify (default true) compares the collections extracted from the
dump against what mongo actually holds afterwards, and fails the job listing
what is missing. It is self-calibrating -- the expected set is derived from
the .bson files on disk, so excluded collections are not expected and no
collection list is hardcoded.

restore.excludeCollections drops collections at tar-extraction time rather
than at restore time, so their bytes are never written to local disk. The
untar stage is disk-bound and is the slowest part of a restore, so this saves
time twice. `sheets` is ~14GB of the ~27GB dump and most cauldrons never
exercise user sheets.

Note this catches an incomplete restore that *finishes*. A restore killed
mid-flight by a Helm hook timeout never reaches the check; that path is
handled separately by install.timeout and install.remediation on the
HelmRelease (Sefaria/cauldrons#152).

Also switches tar from xzvf to xzf -- the file list was 148 lines of noise in
the pod log.

Not changed here: the `[[ ! -z "MONGO_REPLICASET_NAME" ]]` test just above is
missing its `$` and so is always true, appending an empty replicaSet param.
Restores work today, so fixing it would change connection semantics for every
environment; left for its own change.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HFb3rAtzXJ4Ue7w4G3c4VW
---
 .../templates/configmap/mongo-restore.yaml    | 39 ++++++++++++++++++-
 helm-chart/sefaria/values.yaml                |  5 +++
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/helm-chart/sefaria/templates/configmap/mongo-restore.yaml b/helm-chart/sefaria/templates/configmap/mongo-restore.yaml
index 885634d687..fff4160439 100644
--- a/helm-chart/sefaria/templates/configmap/mongo-restore.yaml
+++ b/helm-chart/sefaria/templates/configmap/mongo-restore.yaml
@@ -14,8 +14,17 @@ data:
   restore-mongo.sh: |-
     #!/bin/bash
     set -e
+    set -o pipefail
+
+    # Excluded at extraction time, so skipped bytes never hit the (disk-bound) untar.
+    TAR_EXCLUDES=()
+    {{- range .Values.restore.excludeCollections }}
+    echo "Skipping collection: {{ . }}"
+    TAR_EXCLUDES+=( "--exclude=./dump/sefaria/{{ . }}.bson" "--exclude=./dump/sefaria/{{ . }}.metadata.json" )
+    {{- end }}
+
+    tar xzf /storage/dump.tar.gz -C /storage ${TAR_EXCLUDES[@]+"${TAR_EXCLUDES[@]}"}
 
-    tar xzvf /storage/dump.tar.gz -C /storage 
     if [[ -z "$MONGO_HOST" ]]; then
       echo "Mongo Host not specified"
       exit 1
@@ -48,4 +57,32 @@ data:
     {{- end }}
 
     mongorestore --drop --uri="$URI" -v -d "${DATABASE}" --dir=/storage/dump/sefaria
+
+    {{- if .Values.restore.verify }}
+
+    # Compare what was extracted against what mongo actually has, so a truncated
+    # restore fails instead of passing for a good one. Excluded collections are
+    # not on disk, so they are not expected -- nothing to hardcode.
+    echo "Verifying restore completeness..."
+
+    EXPECTED=$(cd /storage/dump/sefaria && ls -1 ./*.bson 2>/dev/null | sed -e 's|^\./||' -e 's|\.bson$||' | sort)
+    if [[ -z "$EXPECTED" ]]; then
+      echo "ERROR: no .bson files found in the extracted dump -- nothing was restored"
+      exit 1
+    fi
+
+    ACTUAL=$(mongo --quiet "$URI" --eval \
+      "db.getSiblingDB('${DATABASE}').getCollectionNames().forEach(function(c){print(c)})" | sort)
+
+    MISSING=$(comm -23 <(echo "$EXPECTED") <(echo "$ACTUAL") || true)
+
+    if [[ -n "$MISSING" ]]; then
+      echo "ERROR: restore is incomplete. Expected $(echo "$EXPECTED" | wc -l) collections, found $(echo "$ACTUAL" | wc -l)."
+      echo "Missing collections:"
+      echo "$MISSING" | sed 's/^/  - /'
+      exit 1
+    fi
+
+    echo "Restore verified: $(echo "$ACTUAL" | wc -l) collections present in ${DATABASE}"
+    {{- end }}
 {{- end }}
diff --git a/helm-chart/sefaria/values.yaml b/helm-chart/sefaria/values.yaml
index 0872247aba..9c1825eabf 100644
--- a/helm-chart/sefaria/values.yaml
+++ b/helm-chart/sefaria/values.yaml
@@ -55,6 +55,11 @@ restore:
   bucket: sefaria-mongo-backup
   # tarball:
   serviceAccount: database-backup-read
+  # Collections to skip, excluded at extraction time so they never hit disk.
+  # `sheets` alone is ~14GB of the ~27GB dump.
+  excludeCollections: []
+  # Fail the job if mongo is missing any collection that was extracted.
+  verify: true
 # config to backup environment DB
 backup:
   mongo: