diff --git a/helm-chart/sefaria/templates/configmap/mongo-restore.yaml b/helm-chart/sefaria/templates/configmap/mongo-restore.yaml index 885634d687..fff4160439 100644 --- a/helm-chart/sefaria/templates/configmap/mongo-restore.yaml +++ b/helm-chart/sefaria/templates/configmap/mongo-restore.yaml @@ -14,8 +14,17 @@ data: restore-mongo.sh: |- #!/bin/bash set -e + set -o pipefail + + # Excluded at extraction time, so skipped bytes never hit the (disk-bound) untar. + TAR_EXCLUDES=() + {{- range .Values.restore.excludeCollections }} + echo "Skipping collection: {{ . }}" + TAR_EXCLUDES+=( "--exclude=./dump/sefaria/{{ . }}.bson" "--exclude=./dump/sefaria/{{ . }}.metadata.json" ) + {{- end }} + + tar xzf /storage/dump.tar.gz -C /storage ${TAR_EXCLUDES[@]+"${TAR_EXCLUDES[@]}"} - tar xzvf /storage/dump.tar.gz -C /storage if [[ -z "$MONGO_HOST" ]]; then echo "Mongo Host not specified" exit 1 @@ -48,4 +57,32 @@ data: {{- end }} mongorestore --drop --uri="$URI" -v -d "${DATABASE}" --dir=/storage/dump/sefaria + + {{- if .Values.restore.verify }} + + # Compare what was extracted against what mongo actually has, so a truncated + # restore fails instead of passing for a good one. Excluded collections are + # not on disk, so they are not expected -- nothing to hardcode. + echo "Verifying restore completeness..." + + EXPECTED=$(cd /storage/dump/sefaria && ls -1 ./*.bson 2>/dev/null | sed -e 's|^\./||' -e 's|\.bson$||' | sort) + if [[ -z "$EXPECTED" ]]; then + echo "ERROR: no .bson files found in the extracted dump -- nothing was restored" + exit 1 + fi + + ACTUAL=$(mongo --quiet "$URI" --eval \ + "db.getSiblingDB('${DATABASE}').getCollectionNames().forEach(function(c){print(c)})" | sort) + + MISSING=$(comm -23 <(echo "$EXPECTED") <(echo "$ACTUAL") || true) + + if [[ -n "$MISSING" ]]; then + echo "ERROR: restore is incomplete. Expected $(echo "$EXPECTED" | wc -l) collections, found $(echo "$ACTUAL" | wc -l)." + echo "Missing collections:" + echo "$MISSING" | sed 's/^/ - /' + exit 1 + fi + + echo "Restore verified: $(echo "$ACTUAL" | wc -l) collections present in ${DATABASE}" + {{- end }} {{- end }} diff --git a/helm-chart/sefaria/values.yaml b/helm-chart/sefaria/values.yaml index 0872247aba..9c1825eabf 100644 --- a/helm-chart/sefaria/values.yaml +++ b/helm-chart/sefaria/values.yaml @@ -55,6 +55,11 @@ restore: bucket: sefaria-mongo-backup # tarball: serviceAccount: database-backup-read + # Collections to skip, excluded at extraction time so they never hit disk. + # `sheets` alone is ~14GB of the ~27GB dump. + excludeCollections: [] + # Fail the job if mongo is missing any collection that was extracted. + verify: true # config to backup environment DB backup: mongo: