Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .containerignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
**/obj/
SecondDimensionWatcherReDive.Test/
packaging/
deployments/
deployments/*
!deployments/sdw-backup
docs/
SecondDimensionWatcherReDive.Client/dist/
SecondDimensionWatcherReDive.Client/.parcel-cache/
Expand Down
247 changes: 247 additions & 0 deletions .github/workflows/backup-restore.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,247 @@
name: Backup Restore Drill

on:
workflow_call:
workflow_dispatch:

permissions:
contents: read

jobs:
drill:
name: PostgreSQL ${{ matrix.postgres }} rollback-safe drill
runs-on: ubuntu-latest
timeout-minutes: 25
strategy:
fail-fast: false
matrix:
postgres: ["16", "17"]
services:
postgres:
image: postgres:${{ matrix.postgres }}-alpine
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U postgres -d postgres"
--health-interval 5s
--health-timeout 5s
--health-retries 12
env:
PGHOST: 127.0.0.1
PGPORT: 5432
ASPNETCORE_ENVIRONMENT: Production
JwtSecret: backup-restore-drill-jwt-secret-with-at-least-32-bytes
steps:
- uses: actions/checkout@v7

- uses: actions/setup-dotnet@v6
with:
dotnet-version: "10.0.x"

- name: Install PostgreSQL client
run: sudo apt-get update && sudo apt-get install -y postgresql-client

- name: Configure distinct database roles and measured capacity
shell: bash
env:
SERVICE_CONTAINER: ${{ job.services.postgres.id }}
run: |
set -Eeuo pipefail
export PGUSER=postgres PGPASSWORD=postgres PGDATABASE=postgres
psql --no-psqlrc --set=ON_ERROR_STOP=1 <<'SQL'
CREATE ROLE sdw_app LOGIN PASSWORD 'app-password';
CREATE ROLE sdw_restore_admin LOGIN CREATEDB PASSWORD 'restore-password';
GRANT sdw_app TO sdw_restore_admin;
CREATE DATABASE sdw_source OWNER sdw_app;
SQL
available_kib=$(docker exec "$SERVICE_CONTAINER" \
df -Pk /var/lib/postgresql/data | awk 'NR == 2 {print $4}')
[[ "$available_kib" =~ ^[0-9]+$ ]]
printf 'POSTGRES_AVAILABLE_BYTES=%s\n' "$((available_kib * 1024))" >> "$GITHUB_ENV"
printf 'WORK_DIR=%s\n' "$RUNNER_TEMP/sdw-backup-drill-${{ matrix.postgres }}" >> "$GITHUB_ENV"

- name: Build backend
run: >-
dotnet build SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj
-c Release /p:Version="$(tr -d '[:space:]' < VERSION)"

- name: Migrate, register, and persist an encrypted runtime secret
shell: bash
run: |
set -Eeuo pipefail
mkdir -p "$WORK_DIR/source/keys" "$WORK_DIR/plugins/example" "$WORK_DIR/backups"
printf 'DisableCors: true\n' > "$WORK_DIR/source/appsettings.yml"
printf '{"name":"example","version":"1"}\n' > "$WORK_DIR/plugins/example/manifest.json"
export PGUSER=sdw_app PGPASSWORD=app-password PGDATABASE=sdw_source
export ConnectionStrings__sdw='Host=127.0.0.1;Port=5432;Username=sdw_app;Password=app-password;Database=sdw_source'
export PasswordFile="$WORK_DIR/source/password.json"
export DataProtection__KeyRingPath="$WORK_DIR/source/keys"
export Config="$WORK_DIR/source/appsettings.yml"
dotnet run --project SecondDimensionWatcherReDive -c Release --no-build \
--urls http://127.0.0.1:5097 >"$WORK_DIR/source-app.log" 2>&1 &
app_pid=$!
trap 'kill "$app_pid" 2>/dev/null || true' EXIT
for attempt in {1..60}; do
curl --silent --fail http://127.0.0.1:5097/api/auth/allowRegister >/dev/null && break
sleep 1
done
registration=$(curl --silent --show-error --fail \
-H 'Content-Type: application/json' \
-d '{"password":"drill-password"}' \
http://127.0.0.1:5097/api/auth/register)
token=$(jq -er .token <<<"$registration")
curl --silent --show-error --fail \
-H 'Content-Type: application/json' \
-d '{"password":"drill-password"}' \
http://127.0.0.1:5097/api/auth/login >/dev/null
revision=$(curl --silent --show-error --fail \
-H "Authorization: Bearer $token" \
http://127.0.0.1:5097/api/settings | jq -er .revision)
jq -n --argjson revision "$revision" \
'{expectedRevision:$revision,tmdb:{apiKey:{operation:"set",value:"encrypted-drill-secret"}}}' |
curl --silent --show-error --fail -X PATCH \
-H "Authorization: Bearer $token" -H 'Content-Type: application/json' \
--data-binary @- http://127.0.0.1:5097/api/settings >/dev/null
kill "$app_pid"
wait "$app_pid" || true
trap - EXIT
psql --no-psqlrc --set=ON_ERROR_STOP=1 --command \
"INSERT INTO \"Feeds\" (\"Id\", \"Url\", \"Name\", \"CreatedAt\") VALUES ('11111111-1111-1111-1111-111111111111', 'https://example.com/feed', 'drill', now())"
schema=$(psql --no-psqlrc -Atc \
'SELECT "MigrationId" FROM "__EFMigrationsHistory" ORDER BY "MigrationId" DESC LIMIT 1')
[[ "$schema" =~ ^[A-Za-z0-9._:+/@-]+$ ]]
printf 'BACKUP_SCHEMA=%s\n' "$schema" >> "$GITHUB_ENV"
test -s "$WORK_DIR/source/password.json"
test "$(find "$WORK_DIR/source/keys" -type f | wc -l)" -gt 0

- name: Create, verify, and reject archive corruption
shell: bash
run: |
set -Eeuo pipefail
export PGUSER=sdw_app PGPASSWORD=app-password PGDATABASE=sdw_source
archive=$(deployments/sdw-backup create \
--output "$WORK_DIR/backups" \
--config "$WORK_DIR/source/appsettings.yml" \
--password-file "$WORK_DIR/source/password.json" \
--key-ring "$WORK_DIR/source/keys" \
--plugin-dir "$WORK_DIR/plugins" \
--retention-days 7 \
--app-version "$(tr -d '[:space:]' < VERSION)")
deployments/sdw-backup verify "$archive"
cp "$archive" "$WORK_DIR/corrupt.tar.gz"
archive_size=$(stat --format=%s "$WORK_DIR/corrupt.tar.gz")
printf CORRUPT | dd of="$WORK_DIR/corrupt.tar.gz" bs=1 \
seek=$((archive_size / 2)) conv=notrunc status=none
if deployments/sdw-backup verify "$WORK_DIR/corrupt.tar.gz"; then
echo "corrupted archive unexpectedly verified" >&2
exit 1
fi
printf '%s\n' "$archive" > "$WORK_DIR/archive-path"

- name: Reject failures without changing the target database
shell: bash
run: |
set -Eeuo pipefail
export PGUSER=postgres PGPASSWORD=postgres PGDATABASE=postgres
createdb --template=sdw_source --owner=sdw_app sdw_restore
export PGUSER=sdw_restore_admin PGPASSWORD=restore-password PGDATABASE=sdw_restore
export PGMAINTENANCEDATABASE=postgres
psql --no-psqlrc --set=ON_ERROR_STOP=1 --command \
'CREATE TABLE restore_guard (value integer NOT NULL); INSERT INTO restore_guard VALUES (1)'
archive=$(cat "$WORK_DIR/archive-path")
mkdir -p "$WORK_DIR/failure-state"
common=(--confirm-replace --expected-version "$(tr -d '[:space:]' < VERSION)" \
--expected-schema "$BACKUP_SCHEMA" \
--postgres-available-bytes "$POSTGRES_AVAILABLE_BYTES" \
--config-destination "$WORK_DIR/failure-state/appsettings.yml" \
--password-destination "$WORK_DIR/failure-state/password.json" \
--key-ring-destination "$WORK_DIR/failure-state/keys" \
--plugin-destination "$WORK_DIR/failure-state/plugins" \
--safety-directory "$WORK_DIR/backups")
if deployments/sdw-backup restore "$WORK_DIR/corrupt.tar.gz" "${common[@]}"; then
echo "corrupted restore unexpectedly started" >&2
exit 1
fi
test "$(psql --no-psqlrc -Atc 'SELECT value FROM restore_guard')" = 1
test "$(psql --no-psqlrc -Atc \
"SELECT count(*) FROM pg_database WHERE datname LIKE 'sdw_restore_%'")" = 0
if deployments/sdw-backup restore "$archive" "${common[@]}" \
--expected-schema definitely-not-this-schema; then
echo "incompatible schema unexpectedly restored" >&2
exit 1
fi
test "$(psql --no-psqlrc -Atc 'SELECT value FROM restore_guard')" = 1

mkdir -p "$WORK_DIR/fault-bin"
real_pg_restore=$(command -v pg_restore)
cat > "$WORK_DIR/fault-bin/pg_restore" <<EOF
#!/usr/bin/env bash
case " \$* " in
*" sdw_restore_"*) exit 73 ;;
esac
exec "$real_pg_restore" "\$@"
EOF
chmod +x "$WORK_DIR/fault-bin/pg_restore"
if PATH="$WORK_DIR/fault-bin:$PATH" deployments/sdw-backup restore \
"$archive" "${common[@]}"; then
echo "injected pg_restore failure unexpectedly succeeded" >&2
exit 1
fi
test "$(psql --no-psqlrc -Atc 'SELECT value FROM restore_guard')" = 1
test "$(psql --no-psqlrc -Atc \
"SELECT count(*) FROM pg_database WHERE datname LIKE 'sdw_restore_%'")" = 0

- name: Restore, verify owners, and start as the application role
shell: bash
run: |
set -Eeuo pipefail
export PGUSER=sdw_restore_admin PGPASSWORD=restore-password PGDATABASE=sdw_restore
export PGMAINTENANCEDATABASE=postgres
archive=$(cat "$WORK_DIR/archive-path")
mkdir -p "$WORK_DIR/restored"
deployments/sdw-backup restore "$archive" \
--confirm-replace \
--expected-version "$(tr -d '[:space:]' < VERSION)" \
--expected-schema "$BACKUP_SCHEMA" \
--postgres-available-bytes "$POSTGRES_AVAILABLE_BYTES" \
--config-destination "$WORK_DIR/restored/appsettings.yml" \
--password-destination "$WORK_DIR/restored/password.json" \
--key-ring-destination "$WORK_DIR/restored/keys" \
--plugin-destination "$WORK_DIR/restored/plugins" \
--safety-directory "$WORK_DIR/backups"
test "$(psql --no-psqlrc -Atc "SELECT to_regclass('public.restore_guard') IS NULL")" = t
test "$(psql --no-psqlrc -Atc 'SELECT count(*) FROM "Feeds"')" = 1
test "$(psql --no-psqlrc -Atc \
"SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relkind IN ('r','p','v','m','S','f') AND pg_get_userbyid(c.relowner) <> 'sdw_app'")" = 0
cmp "$WORK_DIR/source/password.json" "$WORK_DIR/restored/password.json"
cmp "$WORK_DIR/source/appsettings.yml" "$WORK_DIR/restored/appsettings.yml"
cmp "$WORK_DIR/plugins/example/manifest.json" \
"$WORK_DIR/restored/plugins/example/manifest.json"

export PGUSER=sdw_app PGPASSWORD=app-password PGDATABASE=sdw_restore
export ConnectionStrings__sdw='Host=127.0.0.1;Port=5432;Username=sdw_app;Password=app-password;Database=sdw_restore'
export PasswordFile="$WORK_DIR/restored/password.json"
export DataProtection__KeyRingPath="$WORK_DIR/restored/keys"
export Config="$WORK_DIR/restored/appsettings.yml"
dotnet run --project SecondDimensionWatcherReDive -c Release --no-build \
--urls http://127.0.0.1:5098 >"$WORK_DIR/restored-app.log" 2>&1 &
app_pid=$!
trap 'kill "$app_pid" 2>/dev/null || true' EXIT
for attempt in {1..60}; do
curl --silent --fail http://127.0.0.1:5098/api/auth/allowRegister >/dev/null && break
sleep 1
done
login=$(curl --silent --show-error --fail \
-H 'Content-Type: application/json' -d '{"password":"drill-password"}' \
http://127.0.0.1:5098/api/auth/login)
token=$(jq -er .token <<<"$login")
test "$(curl --silent --show-error --fail \
-H "Authorization: Bearer $token" http://127.0.0.1:5098/api/settings |
jq -r '.tmdb.apiKey.isConfigured')" = true
kill "$app_pid"
wait "$app_pid" || true
trap - EXIT
25 changes: 22 additions & 3 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ permissions:
contents: read

jobs:
backup-restore:
uses: ./.github/workflows/backup-restore.yml

test:
runs-on: ubuntu-latest
steps:
Expand All @@ -42,9 +45,24 @@ jobs:
**/TestResults/*.trx
**/TestResults/**/coverage.cobertura.xml

quality_gate:
name: Required quality gate
if: always()
needs: [test, backup-restore]
runs-on: ubuntu-latest
steps:
- name: Require tests and rollback-safe backup drill
env:
BACKUP_RESULT: ${{ needs.backup-restore.result }}
TEST_RESULT: ${{ needs.test.result }}
run: |
set -euo pipefail
test "$TEST_RESULT" = success
test "$BACKUP_RESULT" = success

build-frontend:
if: github.event_name != 'pull_request'
needs: test
needs: quality_gate
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
Expand Down Expand Up @@ -110,6 +128,7 @@ jobs:
--self-contained false \
/p:Version=${VERSION} \
-o publish
printf '%s\n' "$VERSION" > publish/VERSION

- name: Install nfpm
run: |
Expand Down Expand Up @@ -257,8 +276,8 @@ jobs:
# NativeAOT cross-compile is not supported, so each architecture builds on a
# native runner. ubuntu-24.04-arm is GA for public repos; private repos pay
# for arm minutes — switch to a self-hosted arm runner if that becomes an
# issue. The job depends on `test` only (no frontend artifact needed).
needs: test
# issue. The job needs the quality gate but no frontend artifact.
needs: quality_gate
strategy:
fail-fast: false
matrix:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/container.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,5 @@ jobs:
${{ env.TAG2 }}
cache-from: type=gha
cache-to: type=gha,mode=max
build-args: |
VERSION=${{ env.VERSION }}
16 changes: 15 additions & 1 deletion Containerfile
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
ARG VERSION

# Stage 1: Build frontend
FROM node:24 AS frontend-build
WORKDIR /app
Expand All @@ -8,20 +10,32 @@ RUN yarn build

# Stage 2: Build backend
FROM mcr.microsoft.com/dotnet/sdk:10.0 AS backend-build
ARG VERSION
WORKDIR /src
COPY SecondDimensionWatcherReDive.slnx .
COPY VERSION .
COPY SecondDimensionWatcherReDive.Framework/ SecondDimensionWatcherReDive.Framework/
COPY SecondDimensionWatcherReDive/ SecondDimensionWatcherReDive/
COPY Plugins/ Plugins/
COPY Share/ Share/
COPY --from=frontend-build /app/dist SecondDimensionWatcherReDive/wwwroot/
RUN dotnet restore SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj
RUN dotnet publish SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj -c Release -o /app --no-restore
RUN effective_version="${VERSION:-$(tr -d '[:space:]' < VERSION)}" \
&& dotnet publish SecondDimensionWatcherReDive/SecondDimensionWatcherReDive.csproj \
-c Release -o /app --no-restore \
/p:Version="${effective_version}" \
&& printf '%s\n' "${effective_version}" > /app/VERSION

# Stage 3: Runtime
FROM mcr.microsoft.com/dotnet/aspnet:10.0
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends postgresql-client curl \
&& rm -rf /var/lib/apt/lists/*
COPY --from=backend-build /app .
COPY deployments/sdw-backup /usr/local/bin/sdw-backup
RUN mkdir -p /usr/lib/sdw-redive \
&& install -m 0644 /app/VERSION /usr/lib/sdw-redive/VERSION
EXPOSE 8080
# Optional: read-only NFSv4 export (set Nfs:Enabled=true to activate; publish port at run time).
EXPOSE 2049
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
namespace SecondDimensionWatcherReDive.Framework.DataRepository;

public interface ILogicalDataTransferRepository
{
Task<LogicalDataBundle> ExportAsync(
LogicalDataCategory categories,
Guid userId,
string applicationVersion,
CancellationToken cancellationToken);

Task<LogicalImportResult> ImportAsync(
LogicalDataBundle bundle,
LogicalImportConflictStrategy conflictStrategy,
Guid userId,
CancellationToken cancellationToken);
}
Loading