From 2b7b3a0f3912f2d56617125da12c2e0e2de491a1 Mon Sep 17 00:00:00 2001 From: Aleksandr Minkin Date: Wed, 8 Apr 2026 20:58:15 +0300 Subject: [PATCH] feat: add production auto deploy --- .github/workflows/deploy.yml | 285 ++++++++++-------- README.md | 451 +++++++++++++++++++++++++++- scripts/deploy.sh | 566 +++++++++++++++++++++++++++++++++++ scripts/rollback.sh | 311 +++++++++++++++++++ 4 files changed, 1476 insertions(+), 137 deletions(-) create mode 100755 scripts/deploy.sh create mode 100755 scripts/rollback.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f0532b9..8b4cf63 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,158 +1,191 @@ -name: Deploy Stand +name: Production Deploy on: + push: + branches: ["main", "master"] workflow_dispatch: inputs: - environment: - description: "Target environment" + repo_name: + description: "Repository that triggered deploy" required: true - default: staging + default: infra type: choice options: - - staging - - production - image_tag: - description: "Docker tag for all apps" + - infra + - npp-web + - npp-backend + - processing-worker + - scrape-helper + - contracts + branch: + description: "Git branch to deploy on the server" required: true - default: edge - deploy_path: - description: "Remote path for compose files" + default: main + target_sha: + description: "Optional commit SHA from CI for traceability" + required: false + default: "" + services: + description: "Optional comma-separated docker compose services override" + required: false + default: "" + auto_rollback: + description: "Rollback automatically if deploy or health checks fail" required: true - default: /opt/nppweb - ssh_port: - description: "SSH port" + type: boolean + default: true + workflow_call: + inputs: + repo_name: + description: "Repository that triggered deploy" required: true - default: "22" + type: string + branch: + description: "Git branch to deploy on the server" + required: false + default: main + type: string + target_sha: + description: "Optional commit SHA from CI for traceability" + required: false + default: "" + type: string + services: + description: "Optional comma-separated docker compose services override" + required: false + default: "" + type: string + auto_rollback: + description: "Rollback automatically if deploy or health checks fail" + required: false + default: true + type: boolean permissions: contents: read +concurrency: + group: production-deploy + cancel-in-progress: false + jobs: deploy: runs-on: ubuntu-latest - environment: ${{ inputs.environment }} + timeout-minutes: 45 + environment: production steps: - - name: Checkout - uses: actions/checkout@v4 + - name: Resolve deploy parameters + id: params + env: + INPUT_REPO_NAME: ${{ inputs.repo_name }} + INPUT_BRANCH: ${{ inputs.branch }} + INPUT_TARGET_SHA: ${{ inputs.target_sha }} + INPUT_SERVICES: ${{ inputs.services }} + INPUT_AUTO_ROLLBACK: ${{ inputs.auto_rollback }} + run: | + set -Eeuo pipefail + + if [[ "${GITHUB_EVENT_NAME}" == "push" ]]; then + repo_name="infra" + branch="${GITHUB_REF_NAME}" + target_sha="${GITHUB_SHA}" + services="" + auto_rollback="true" + else + repo_name="${INPUT_REPO_NAME:-infra}" + branch="${INPUT_BRANCH:-main}" + target_sha="${INPUT_TARGET_SHA:-}" + services="${INPUT_SERVICES:-}" + auto_rollback="${INPUT_AUTO_ROLLBACK:-true}" + fi + + { + echo "repo_name=${repo_name}" + echo "branch=${branch}" + echo "target_sha=${target_sha}" + echo "services=${services}" + echo "auto_rollback=${auto_rollback}" + } >>"$GITHUB_OUTPUT" + + echo "Resolved deploy target: repo=${repo_name}, branch=${branch}, sha=${target_sha:-n/a}, services=${services:-default}" - name: Configure SSH env: - SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} SSH_HOST: ${{ secrets.SSH_HOST }} - SSH_PORT: ${{ inputs.ssh_port }} + SSH_PORT_INPUT: ${{ secrets.SSH_PORT }} + SSH_USER: ${{ secrets.SSH_USER }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }} run: | - set -euo pipefail + set -Eeuo pipefail + + [[ -n "${SSH_HOST}" ]] || { echo "SSH_HOST secret is required"; exit 1; } + [[ -n "${SSH_USER}" ]] || { echo "SSH_USER secret is required"; exit 1; } + [[ -n "${SSH_PRIVATE_KEY}" ]] || { echo "SSH_PRIVATE_KEY secret is required"; exit 1; } + + ssh_port="${SSH_PORT_INPUT:-22}" + mkdir -p ~/.ssh chmod 700 ~/.ssh - printf "%s\n" "$SSH_PRIVATE_KEY" > ~/.ssh/id_ed25519 + printf '%s\n' "${SSH_PRIVATE_KEY}" > ~/.ssh/id_ed25519 chmod 600 ~/.ssh/id_ed25519 - ssh-keyscan -p "$SSH_PORT" -H "$SSH_HOST" >> ~/.ssh/known_hosts - chmod 644 ~/.ssh/known_hosts - - name: Build deploy env file - env: - IMAGE_TAG: ${{ inputs.image_tag }} - POSTGRES_DB: ${{ secrets.POSTGRES_DB }} - POSTGRES_USER: ${{ secrets.POSTGRES_USER }} - POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} - JWT_ACCESS_SECRET: ${{ secrets.JWT_ACCESS_SECRET }} - RABBITMQ_DEFAULT_USER: ${{ secrets.RABBITMQ_DEFAULT_USER }} - RABBITMQ_DEFAULT_PASS: ${{ secrets.RABBITMQ_DEFAULT_PASS }} - MINIO_ROOT_USER: ${{ secrets.MINIO_ROOT_USER }} - MINIO_ROOT_PASSWORD: ${{ secrets.MINIO_ROOT_PASSWORD }} - MINIO_BUCKET: ${{ secrets.MINIO_BUCKET }} - INGEST_API_TOKEN: ${{ secrets.INGEST_API_TOKEN }} - API_INGEST_TOKEN: ${{ secrets.API_INGEST_TOKEN }} - ADMIN_EMAIL: ${{ secrets.ADMIN_EMAIL }} - ADMIN_PASSWORD: ${{ secrets.ADMIN_PASSWORD }} - ADMIN_FULL_NAME: ${{ secrets.ADMIN_FULL_NAME }} - ENABLED_SOURCES: ${{ vars.ENABLED_SOURCES }} - COMPOSE_PROFILES: ${{ vars.COMPOSE_PROFILES }} - REQUEST_TIMEOUT_MS: ${{ vars.REQUEST_TIMEOUT_MS }} - SCRAPER_HTTP_PROXY: ${{ vars.SCRAPER_HTTP_PROXY }} - SCRAPER_HTTPS_PROXY: ${{ vars.SCRAPER_HTTPS_PROXY }} - SCRAPER_NO_PROXY: ${{ vars.SCRAPER_NO_PROXY }} - BACKEND_API_IMAGE: ${{ vars.BACKEND_API_IMAGE }} - SCRAPER_SERVICE_IMAGE: ${{ vars.SCRAPER_SERVICE_IMAGE }} - PROCESSING_WORKER_IMAGE: ${{ vars.PROCESSING_WORKER_IMAGE }} - FRONTEND_IMAGE: ${{ vars.FRONTEND_IMAGE }} - run: | - set -euo pipefail - cat > .env.deploy < ~/.ssh/known_hosts + else + ssh-keyscan -p "${ssh_port}" -H "${SSH_HOST}" >> ~/.ssh/known_hosts + fi - - name: Sync compose files to server - env: - SSH_HOST: ${{ secrets.SSH_HOST }} - SSH_USER: ${{ secrets.SSH_USER }} - SSH_PORT: ${{ inputs.ssh_port }} - DEPLOY_PATH: ${{ inputs.deploy_path }} - run: | - set -euo pipefail - ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "mkdir -p '$DEPLOY_PATH' '$DEPLOY_PATH/xray-local'" - rsync -az --delete -e "ssh -p $SSH_PORT" \ - docker-compose.yml \ - docker-compose.deploy.yml \ - contracts \ - "$SSH_USER@$SSH_HOST:$DEPLOY_PATH/" - scp -P "$SSH_PORT" .env.deploy "$SSH_USER@$SSH_HOST:$DEPLOY_PATH/.env" - - - name: Deploy stack on server + chmod 644 ~/.ssh/known_hosts + + - name: Run deploy on server env: SSH_HOST: ${{ secrets.SSH_HOST }} + SSH_PORT_INPUT: ${{ secrets.SSH_PORT }} SSH_USER: ${{ secrets.SSH_USER }} - SSH_PORT: ${{ inputs.ssh_port }} - DEPLOY_PATH: ${{ inputs.deploy_path }} - GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }} - GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} + DEPLOY_PATH_INPUT: ${{ secrets.DEPLOY_PATH }} + REPO_NAME: ${{ steps.params.outputs.repo_name }} + BRANCH: ${{ steps.params.outputs.branch }} + TARGET_SHA: ${{ steps.params.outputs.target_sha }} + SERVICES: ${{ steps.params.outputs.services }} + AUTO_ROLLBACK: ${{ steps.params.outputs.auto_rollback }} run: | - set -euo pipefail - echo "$GHCR_TOKEN" | ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" \ - "docker login ghcr.io -u '$GHCR_USERNAME' --password-stdin" - ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" " - set -euo pipefail - cd '$DEPLOY_PATH' - docker compose --env-file .env -f docker-compose.yml -f docker-compose.deploy.yml pull - docker compose --env-file .env -f docker-compose.yml -f docker-compose.deploy.yml up -d - docker image prune -f - " + set -Eeuo pipefail + + ssh_port="${SSH_PORT_INPUT:-22}" + deploy_path="${DEPLOY_PATH_INPUT:-/home/deploy}" + + ssh -p "${ssh_port}" "${SSH_USER}@${SSH_HOST}" bash -s -- \ + "${REPO_NAME}" \ + "${BRANCH}" \ + "${TARGET_SHA}" \ + "${SERVICES}" \ + "${AUTO_ROLLBACK}" \ + "${deploy_path}" <<'EOF' + set -Eeuo pipefail + + repo_name="$1" + branch="$2" + target_sha="$3" + services="$4" + auto_rollback="$5" + deploy_path="$6" + + args=(--repo "$repo_name" --branch "$branch") + + if [[ -n "$target_sha" ]]; then + args+=(--sha "$target_sha") + fi + + if [[ -n "$services" ]]; then + args+=(--services "$services") + fi + + if [[ "$auto_rollback" != "true" ]]; then + args+=(--no-auto-rollback) + fi + + export DEPLOY_PATH="$deploy_path" + cd "$deploy_path/infra" + ./scripts/deploy.sh "${args[@]}" + EOF diff --git a/README.md b/README.md index d18e89e..12e384c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # infra -Docker Compose инфраструктура для локального запуска NPPWEB. +Docker Compose инфраструктура для локального запуска и production deploy проекта NPPWEB. ## Что поднимается @@ -17,7 +17,9 @@ Docker Compose инфраструктура для локального запу Все сервисы работают в сети `platform-net` и обращаются друг к другу по service name. -## Быстрый старт +## Локальный запуск + +Быстрый вариант через общий стек: ```bash cp .env.example .env @@ -65,9 +67,9 @@ docker compose --env-file .env -f docker-compose.yml -f docker-compose.apps.yml Ожидаемо: -- `postgres`, `rabbitmq`, `minio`, `backend-api`, `frontend` имеют статус `healthy`; -- `minio-init` завершился со статусом `Exited (0)`; -- `processing-worker` и `scraper-service` находятся в `Up`. +- `postgres`, `rabbitmq`, `minio`, `backend-api`, `frontend` имеют статус `healthy` +- `minio-init` завершился со статусом `Exited (0)` +- `processing-worker` и `scraper-service` находятся в `Up` Проверки вручную: @@ -105,9 +107,436 @@ docker compose --env-file .env -f docker-compose.yml -f docker-compose.apps.yml ## Что обеспечивает устойчивый старт -- `postgres`, `rabbitmq`, `minio`, `backend-api` имеют healthcheck'и; -- `minio-init` создаёт bucket до старта `scraper-service`; -- `backend-api` ждёт Postgres и запускает migrations/seed перед стартом; -- `processing-worker` ждёт `rabbitmq` и `backend-api`; -- `scraper-service` ждёт `rabbitmq`, `minio` и `minio-init`. -- `scraper-service` может идти наружу через `xray-proxy`, если в `.env` заданы `HTTP_PROXY/HTTPS_PROXY`. +- `postgres`, `rabbitmq`, `minio`, `backend-api` имеют healthcheck'и +- `minio-init` создаёт bucket до старта `scraper-service` +- `backend-api` ждёт Postgres и запускает migrations/seed перед стартом +- `processing-worker` ждёт `rabbitmq` и `backend-api` +- `scraper-service` ждёт `rabbitmq`, `minio` и `minio-init` +- `scraper-service` может идти наружу через `xray-proxy`, если в `.env` заданы `HTTP_PROXY/HTTPS_PROXY` + +## Production Auto-Deploy + +### Выбранная архитектура + +Выбран прагматичный вариант без Kubernetes и без внешней DevOps-платформы: + +- единый reusable workflow живёт в `infra` +- `npp-web`, `npp-backend`, `processing-worker`, `scrape-helper` и `contracts` имеют тонкие workflows-обёртки +- каждый такой workflow при `push` в `main` вызывает reusable workflow из `infra` +- reusable workflow по SSH заходит на production VPS под пользователем `deploy` +- на сервере запускается [`scripts/deploy.sh`](/home/minkin/vkrdiff/infra/scripts/deploy.sh), который: + - берёт глобальный lock через `flock` + - проверяет директории и git-репозитории + - печатает commit hash до и после обновления + - делает `git fetch` и `git pull --ff-only` + - запускает `docker compose --env-file .env -f docker-compose.yml -f docker-compose.apps.yml up -d --build` + - ждёт health checks и HTTP checks + - на провале автоматически откатывает изменённые репозитории назад и повторно поднимает стек +- успешный деплой сохраняет release snapshot в `/home/deploy/.deploy-state/releases/*.env` +- ручной откат выполняется через [`scripts/rollback.sh`](/home/minkin/vkrdiff/infra/scripts/rollback.sh) + +Почему так: + +- логика деплоя сосредоточена в одном месте +- нет `repository_dispatch` и отдельного PAT +- сервер остаётся source-of-truth для production checkout'ов +- решение нормально работает на обычном VPS с уже существующим Docker и nginx + +### Как триггерится деплой из нескольких репозиториев + +- push в `infra/main` запускает [`infra/.github/workflows/deploy.yml`](/home/minkin/vkrdiff/infra/.github/workflows/deploy.yml) напрямую и делает full-stack deploy +- push в `npp-web/main` запускает [`npp-web/.github/workflows/production-deploy.yml`](/home/minkin/vkrdiff/npp-web/.github/workflows/production-deploy.yml), который вызывает reusable workflow из `infra` и пересобирает только `frontend` +- push в `npp-backend/main` запускает [`npp-backend/.github/workflows/production-deploy.yml`](/home/minkin/vkrdiff/npp-backend/.github/workflows/production-deploy.yml), который пересобирает `backend-api` +- push в `processing-worker/main` пересобирает только `processing-worker` +- push в `scrape-helper/main` пересобирает только `scraper-service` +- push в `contracts/main` обновляет `contracts` и пересобирает `processing-worker` и `scraper-service` + +Таким образом: + +- приложение деплоится из нескольких отдельных репозиториев +- compose всё равно исполняется только на сервере из `infra` +- `infra` repo на сервере тоже обновляется перед deploy, чтобы compose и скрипты не расходились с текущим `main` + +### Minimal Downtime + +Полного zero-downtime с одиночным VPS и обычным `docker compose` тут нет: контейнер сервиса всё равно пересоздаётся. Но downtime сводится к минимуму: + +- образ собирается перед финальной заменой контейнера +- при изменении `npp-web` пересобирается только `frontend` +- при изменении `npp-backend` пересобирается только `backend-api` +- full-stack rebuild выполняется только для `infra` +- после перезапуска скрипт ждёт health checks и HTTP checks + +### Rollback Strategy + +Есть два уровня rollback: + +- автоматический rollback в `deploy.sh`, если `git pull`, `docker compose up`, container health или HTTP health провалились +- ручной rollback на предыдущий успешный release через `rollback.sh` + +Release snapshots лежат в: + +```text +/home/deploy/.deploy-state/releases/ +``` + +Каждый snapshot хранит commit SHA всех production repo на момент успешного deploy. + +## Какие файлы добавлены или изменены + +### Infra + +- [`infra/.github/workflows/deploy.yml`](/home/minkin/vkrdiff/infra/.github/workflows/deploy.yml) +- [`infra/scripts/deploy.sh`](/home/minkin/vkrdiff/infra/scripts/deploy.sh) +- [`infra/scripts/rollback.sh`](/home/minkin/vkrdiff/infra/scripts/rollback.sh) +- [`infra/README.md`](/home/minkin/vkrdiff/infra/README.md) + +### App Repos + +- [`npp-web/.github/workflows/production-deploy.yml`](/home/minkin/vkrdiff/npp-web/.github/workflows/production-deploy.yml) +- [`npp-backend/.github/workflows/production-deploy.yml`](/home/minkin/vkrdiff/npp-backend/.github/workflows/production-deploy.yml) +- [`processing-worker/.github/workflows/production-deploy.yml`](/home/minkin/vkrdiff/processing-worker/.github/workflows/production-deploy.yml) +- [`scrape-helper/.github/workflows/production-deploy.yml`](/home/minkin/vkrdiff/scrape-helper/.github/workflows/production-deploy.yml) +- [`contracts/.github/workflows/production-deploy.yml`](/home/minkin/vkrdiff/contracts/.github/workflows/production-deploy.yml) + +## GitHub Secrets + +Рекомендуемый вариант: завести secrets на уровне GitHub Organization `nppweb` и открыть их для нужных репозиториев. Тогда `secrets: inherit` будет работать без дублирования. + +Обязательные secrets: + +- `SSH_HOST` - hostname или IP production VPS +- `SSH_USER` - `deploy` +- `SSH_PRIVATE_KEY` - приватный deploy key для GitHub Actions + +Рекомендуемые secrets: + +- `SSH_PORT` - если не `22` +- `DEPLOY_PATH` - если production checkout живёт не в `/home/deploy` +- `SSH_KNOWN_HOSTS` - заранее сохранённый `known_hosts`; если не задан, workflow сам сделает `ssh-keyscan` + +Если org secrets использовать нельзя, заведите те же secrets в каждом repo: + +- `infra` +- `npp-web` +- `npp-backend` +- `processing-worker` +- `scrape-helper` +- `contracts` + +## Команды для deploy SSH key + +Сгенерировать отдельный ключ для GitHub Actions: + +```bash +ssh-keygen -t ed25519 -f ~/.ssh/nppweb-gha-deploy -C "github-actions@nppweb-production" +``` + +Положить приватный ключ в GitHub Secret `SSH_PRIVATE_KEY`: + +```bash +cat ~/.ssh/nppweb-gha-deploy +``` + +Публичный ключ: + +```bash +cat ~/.ssh/nppweb-gha-deploy.pub +``` + +## Как добавить публичный ключ на сервер + +Под root или пользователем с sudo: + +```bash +sudo install -d -m 700 -o deploy -g deploy /home/deploy/.ssh +sudo touch /home/deploy/.ssh/authorized_keys +sudo chmod 600 /home/deploy/.ssh/authorized_keys +sudo chown deploy:deploy /home/deploy/.ssh/authorized_keys +``` + +Добавить ключ: + +```bash +cat ~/.ssh/nppweb-gha-deploy.pub | ssh deploy@YOUR_SERVER "cat >> ~/.ssh/authorized_keys" +``` + +Проверить вход: + +```bash +ssh -i ~/.ssh/nppweb-gha-deploy deploy@YOUR_SERVER +``` + +## Пример `known_hosts` + +Снять host key и сохранить в secret `SSH_KNOWN_HOSTS`: + +```bash +ssh-keyscan -H nppweb.minkin.tech +``` + +Если SSH идёт на нестандартный порт: + +```bash +ssh-keyscan -p 2222 -H nppweb.minkin.tech +``` + +Пример значения для secret: + +```text +nppweb.minkin.tech ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI... +``` + +## Первичная настройка сервера + +### 1. Убедиться, что у `deploy` есть доступ к Docker + +```bash +sudo usermod -aG docker deploy +``` + +После этого нужно перелогиниться под `deploy`. + +### 2. Поставить базовые зависимости + +На VPS нужны: + +- `git` +- `docker` +- `docker compose` plugin +- `curl` +- `flock` из `util-linux` + +### 3. Подготовить рабочую директорию + +```bash +sudo mkdir -p /home/deploy +sudo chown deploy:deploy /home/deploy +``` + +### 4. Склонировать репозитории + +Под пользователем `deploy`: + +```bash +cd /home/deploy +git clone git@github.com:nppweb/infra.git +git clone git@github.com:nppweb/npp-web.git +git clone git@github.com:nppweb/npp-backend.git +git clone git@github.com:nppweb/processing-worker.git +git clone git@github.com:nppweb/scrape-helper.git +git clone git@github.com:nppweb/contracts.git +``` + +Если какие-то repo реально не используются в production, их можно не клонировать и убрать соответствующий workflow. + +### 5. Сделать deploy scripts исполняемыми + +```bash +chmod +x /home/deploy/infra/scripts/deploy.sh +chmod +x /home/deploy/infra/scripts/rollback.sh +``` + +### 6. Подготовить production `.env` + +```bash +cd /home/deploy/infra +cp .env.example .env +``` + +Минимально проверьте и поправьте: + +```dotenv +CORS_ALLOWED_ORIGINS=http://localhost:8080,http://nppweb.minkin.tech +NUXT_PUBLIC_GRAPHQL_ENDPOINT=/graphql +BACKEND_PORT=3000 +FRONTEND_PORT=8080 +``` + +Остальные production secrets должны лежать только в `/home/deploy/infra/.env`, а не в git. + +### 7. Если нужен proxy для scraper + +```bash +cp /home/deploy/infra/xray-local/config.example.json /home/deploy/infra/xray-local/config.json +``` + +Заполните приватные параметры и при необходимости добавьте в `.env`: + +```dotenv +COMPOSE_PROFILES=proxy +HTTP_PROXY=http://xray-proxy:8080 +HTTPS_PROXY=http://xray-proxy:8080 +``` + +### 8. Выполнить первый ручной deploy + +```bash +cd /home/deploy/infra +./scripts/deploy.sh --repo infra +``` + +Это проверит, что сервер готов ещё до первого GitHub Actions запуска. + +## Пошаговый запуск + +1. Добавьте workflows из этого README в соответствующие repo. +2. Добавьте GitHub secrets. +3. Подготовьте сервер по шагам выше. +4. Выполните первый ручной deploy на сервере: `./scripts/deploy.sh --repo infra`. +5. Сделайте тестовый push в `main` одного из repo. +6. Убедитесь, что workflow дошёл до SSH-этапа и завершился успешно. + +## Как проверить, что автодеплой работает + +### Через GitHub Actions + +- в `npp-web` запушьте тестовый commit в `main` +- откройте workflow `Production Deploy` +- убедитесь, что reusable workflow в `infra` отработал успешно + +### На сервере + +Проверить commit в целевом repo: + +```bash +git -C /home/deploy/npp-web rev-parse HEAD +``` + +Проверить статус контейнеров: + +```bash +cd /home/deploy/infra +docker compose --env-file .env -f docker-compose.yml -f docker-compose.apps.yml ps +``` + +Проверить backend: + +```bash +curl -f http://127.0.0.1:3000/api/health/ready +``` + +Проверить frontend: + +```bash +curl -I http://127.0.0.1:8080 +``` + +Проверить, что release snapshot сохранился: + +```bash +ls -1 /home/deploy/.deploy-state/releases +cat /home/deploy/.deploy-state/last-success.env +``` + +## Как откатиться + +### Быстрый откат на предыдущий successful release + +```bash +cd /home/deploy/infra +./scripts/rollback.sh +``` + +### Посмотреть доступные release snapshots + +```bash +cd /home/deploy/infra +./scripts/rollback.sh --list +``` + +### Откатиться на конкретный snapshot + +```bash +cd /home/deploy/infra +./scripts/rollback.sh --release-file /home/deploy/.deploy-state/releases/20260408T120501Z.env +``` + +После отката скрипт: + +- переводит все production repo на сохранённые commit SHA +- выполняет `docker compose up -d --build --remove-orphans` +- ждёт health checks +- обновляет `/home/deploy/.deploy-state/last-success.env` + +## Типичные проблемы и как диагностировать + +### SSH не подключается + +Проверьте: + +- правильность `SSH_HOST`, `SSH_PORT`, `SSH_USER` +- что `SSH_PRIVATE_KEY` соответствует ключу в `authorized_keys` +- что `SSH_KNOWN_HOSTS` не устарел + +Диагностика: + +```bash +ssh -i ~/.ssh/nppweb-gha-deploy -p 22 deploy@YOUR_SERVER +``` + +### Workflow падает на reusable workflow access + +Проверьте настройки GitHub Actions в организации и в repo: + +- reusable workflows из `nppweb/infra` должны быть разрешены +- caller repo должен видеть repo `infra` + +### `deploy.sh` ругается на dirty worktree + +На production checkout не должно быть ручных правок. Проверить: + +```bash +git -C /home/deploy/npp-web status --short +``` + +Если на сервере есть ручные изменения, уберите их и повторите deploy. + +### Падает `docker compose config -q` + +Обычно это: + +- синтаксическая ошибка в compose +- отсутствует нужный repo рядом с `infra` +- отсутствует `.env` + +Проверка: + +```bash +cd /home/deploy/infra +docker compose --env-file .env -f docker-compose.yml -f docker-compose.apps.yml config +``` + +### Контейнер поднялся, но health check не проходит + +Смотреть статус и логи: + +```bash +cd /home/deploy/infra +docker compose --env-file .env -f docker-compose.yml -f docker-compose.apps.yml ps +docker compose --env-file .env -f docker-compose.yml -f docker-compose.apps.yml logs --tail=100 backend-api frontend processing-worker scraper-service +``` + +### Проблема с frontend или CORS после deploy + +Проверьте production `.env`: + +```dotenv +CORS_ALLOWED_ORIGINS=http://localhost:8080,http://nppweb.minkin.tech +NUXT_PUBLIC_GRAPHQL_ENDPOINT=/graphql +``` + +Также проверьте nginx-маршрутизацию до: + +- `http://127.0.0.1:8080` для frontend +- `http://127.0.0.1:3000/graphql` и `http://127.0.0.1:3000/api/health/ready` для backend + +### Два деплоя стартовали одновременно + +Это штатно. `deploy.sh` использует глобальный lock: + +```text +/home/deploy/.deploy-state/deploy.lock +``` + +Второй deploy завершится fail-fast с понятной ошибкой вместо гонки между `git pull` и `docker compose`. diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..a1a8f6e --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,566 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +IFS=$'\n\t' + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +INFRA_DIR=$(cd -- "$SCRIPT_DIR/.." && pwd) +DEPLOY_ROOT_DEFAULT=$(cd -- "$INFRA_DIR/.." && pwd) +DEPLOY_ROOT="${DEPLOY_PATH:-$DEPLOY_ROOT_DEFAULT}" +STATE_DIR="${DEPLOY_STATE_DIR:-$DEPLOY_ROOT/.deploy-state}" +LOCK_FILE="${DEPLOY_LOCK_FILE:-$STATE_DIR/deploy.lock}" +COMPOSE_ENV_FILE="${DEPLOY_ENV_FILE:-$INFRA_DIR/.env}" +DEFAULT_BRANCH="${DEPLOY_BRANCH:-main}" +WAIT_TIMEOUT="${DEPLOY_WAIT_TIMEOUT:-300}" +POLL_INTERVAL="${DEPLOY_POLL_INTERVAL:-5}" + +MANAGED_REPOS=(infra npp-web npp-backend processing-worker scrape-helper contracts) + +TARGET_REPO="" +TARGET_BRANCH="$DEFAULT_BRANCH" +TARGET_SHA="" +SERVICES_OVERRIDE="" +AUTO_ROLLBACK=1 +UPDATE_INFRA=1 + +declare -a UPDATED_REPOS=() +declare -a DEPLOY_SERVICES=() +declare -A BEFORE_SHA=() +declare -A AFTER_SHA=() + +HANDLING_ERROR=0 + +log() { + printf '[%s] %s\n' "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" "$*" +} + +warn() { + log "WARN: $*" +} + +fail() { + log "ERROR: $*" + return 1 +} + +usage() { + cat <<'EOF' +Usage: + ./scripts/deploy.sh --repo [options] + +Options: + --repo Repository to update. One of: + infra, npp-web, npp-backend, processing-worker, + scrape-helper, contracts. + --branch Git branch to deploy. Default: main. + --sha Commit SHA from CI for logging/verification. + --services Comma-separated docker compose services override. + --no-auto-rollback Disable automatic rollback on failure. + --no-update-infra Do not refresh infra repo before deploy. + --help Show this help. + +Examples: + ./scripts/deploy.sh --repo infra + ./scripts/deploy.sh --repo npp-web --sha 0123abcd + ./scripts/deploy.sh --repo contracts --services processing-worker,scraper-service +EOF +} + +require_command() { + local command_name="$1" + command -v "$command_name" >/dev/null 2>&1 || fail "Required command not found: $command_name" +} + +join_by() { + local delimiter="$1" + shift + + local first=1 + local item + + for item in "$@"; do + if ((first == 1)); then + printf '%s' "$item" + first=0 + else + printf '%s%s' "$delimiter" "$item" + fi + done +} + +repo_path() { + local repo_name="$1" + + case "$repo_name" in + infra) + printf '%s\n' "$INFRA_DIR" + ;; + npp-web | npp-backend | processing-worker | scrape-helper | contracts) + printf '%s/%s\n' "$DEPLOY_ROOT" "$repo_name" + ;; + *) + fail "Unknown repo: $repo_name" + ;; + esac +} + +snapshot_var_name() { + local repo_name="$1" + + case "$repo_name" in + infra) + printf 'REPO_SHA_INFRA\n' + ;; + npp-web) + printf 'REPO_SHA_NPP_WEB\n' + ;; + npp-backend) + printf 'REPO_SHA_NPP_BACKEND\n' + ;; + processing-worker) + printf 'REPO_SHA_PROCESSING_WORKER\n' + ;; + scrape-helper) + printf 'REPO_SHA_SCRAPE_HELPER\n' + ;; + contracts) + printf 'REPO_SHA_CONTRACTS\n' + ;; + *) + fail "Unknown repo for snapshot var: $repo_name" + ;; + esac +} + +parse_args() { + while (($# > 0)); do + case "$1" in + --repo) + TARGET_REPO="${2:-}" + shift 2 + ;; + --branch) + TARGET_BRANCH="${2:-}" + shift 2 + ;; + --sha) + TARGET_SHA="${2:-}" + shift 2 + ;; + --services) + SERVICES_OVERRIDE="${2:-}" + shift 2 + ;; + --no-auto-rollback) + AUTO_ROLLBACK=0 + shift + ;; + --no-update-infra) + UPDATE_INFRA=0 + shift + ;; + --help) + usage + exit 0 + ;; + *) + fail "Unknown argument: $1" + ;; + esac + done + + [[ -n "$TARGET_REPO" ]] || fail "--repo is required" + [[ -n "$TARGET_BRANCH" ]] || fail "--branch cannot be empty" +} + +ensure_repo_ready() { + local repo_name="$1" + local repo_dir + + repo_dir=$(repo_path "$repo_name") + + [[ -d "$repo_dir" ]] || fail "Repository directory not found: $repo_dir" + [[ -d "$repo_dir/.git" ]] || fail "Directory is not a git repository: $repo_dir" + + if [[ -n "$(git -C "$repo_dir" status --porcelain)" ]]; then + fail "Repository has uncommitted changes: $repo_dir" + fi +} + +compose() { + ( + cd "$INFRA_DIR" + docker compose \ + --env-file "$COMPOSE_ENV_FILE" \ + -f docker-compose.yml \ + -f docker-compose.apps.yml \ + "$@" + ) +} + +read_env_var() { + local key="$1" + local default_value="$2" + local value="" + + if [[ -f "$COMPOSE_ENV_FILE" ]]; then + value=$(awk -F= -v key="$key" ' + $0 !~ /^[[:space:]]*#/ && $1 == key { + sub(/^[^=]*=/, "", $0) + print $0 + exit + } + ' "$COMPOSE_ENV_FILE") + fi + + if [[ -n "$value" ]]; then + printf '%s\n' "$value" + else + printf '%s\n' "$default_value" + fi +} + +git_sha() { + local repo_dir="$1" + git -C "$repo_dir" rev-parse HEAD +} + +update_repo() { + local repo_name="$1" + local branch_name="$2" + local repo_dir current_branch before_sha after_sha + + repo_dir=$(repo_path "$repo_name") + before_sha=$(git_sha "$repo_dir") + current_branch=$(git -C "$repo_dir" symbolic-ref -q --short HEAD || true) + + log "Updating repo '$repo_name' in '$repo_dir'" + log "Repo '$repo_name' before: ${before_sha} (branch: ${current_branch:-detached})" + + git -C "$repo_dir" fetch --prune origin "$branch_name" + + if [[ -z "$current_branch" ]]; then + warn "Repo '$repo_name' is on detached HEAD, switching back to '$branch_name'" + if git -C "$repo_dir" show-ref --verify --quiet "refs/heads/$branch_name"; then + git -C "$repo_dir" checkout "$branch_name" + else + git -C "$repo_dir" checkout -b "$branch_name" "origin/$branch_name" + fi + elif [[ "$current_branch" != "$branch_name" ]]; then + warn "Repo '$repo_name' is on branch '$current_branch', switching to '$branch_name'" + if git -C "$repo_dir" show-ref --verify --quiet "refs/heads/$branch_name"; then + git -C "$repo_dir" checkout "$branch_name" + else + git -C "$repo_dir" checkout -b "$branch_name" "origin/$branch_name" + fi + fi + + git -C "$repo_dir" pull --ff-only origin "$branch_name" + + after_sha=$(git_sha "$repo_dir") + BEFORE_SHA["$repo_name"]="$before_sha" + AFTER_SHA["$repo_name"]="$after_sha" + UPDATED_REPOS+=("$repo_name") + + log "Repo '$repo_name' after: ${after_sha}" + + if [[ "$repo_name" == "$TARGET_REPO" && -n "$TARGET_SHA" ]]; then + if git -C "$repo_dir" cat-file -e "${TARGET_SHA}^{commit}" 2>/dev/null; then + if git -C "$repo_dir" merge-base --is-ancestor "$TARGET_SHA" "$after_sha"; then + log "Target SHA ${TARGET_SHA} is included in deployed commit for '$repo_name'" + else + warn "Target SHA ${TARGET_SHA} is not an ancestor of deployed commit ${after_sha}" + fi + else + warn "Target SHA ${TARGET_SHA} is not present in local git object database for '$repo_name'" + fi + fi +} + +resolve_services() { + if [[ -n "$SERVICES_OVERRIDE" ]]; then + IFS=',' read -r -a DEPLOY_SERVICES <<<"$SERVICES_OVERRIDE" + return + fi + + case "$TARGET_REPO" in + infra) + DEPLOY_SERVICES=() + ;; + npp-web) + DEPLOY_SERVICES=(frontend) + ;; + npp-backend) + DEPLOY_SERVICES=(backend-api) + ;; + processing-worker) + DEPLOY_SERVICES=(processing-worker) + ;; + scrape-helper) + DEPLOY_SERVICES=(scraper-service) + ;; + contracts) + DEPLOY_SERVICES=(processing-worker scraper-service) + ;; + *) + fail "Unsupported target repo: $TARGET_REPO" + ;; + esac +} + +service_status() { + local service_name="$1" + local container_id + + container_id=$(compose ps -q "$service_name") + [[ -n "$container_id" ]] || fail "Container ID not found for service '$service_name'" + + docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id" +} + +wait_for_service() { + local service_name="$1" + local waited=0 + local status="" + + while ((waited < WAIT_TIMEOUT)); do + status=$(service_status "$service_name") + + case "$status" in + healthy | running) + log "Service '$service_name' status: $status" + return 0 + ;; + exited | dead | unhealthy) + fail "Service '$service_name' entered bad state: $status" + ;; + *) + log "Waiting for service '$service_name' (status: $status)" + ;; + esac + + sleep "$POLL_INTERVAL" + waited=$((waited + POLL_INTERVAL)) + done + + fail "Timed out waiting for service '$service_name' after ${WAIT_TIMEOUT}s" +} + +wait_for_compose_services() { + local services=("$@") + + if ((${#services[@]} == 0)); then + services=(postgres redis rabbitmq minio backend-api processing-worker scraper-service frontend) + fi + + for service_name in "${services[@]}"; do + wait_for_service "$service_name" + done +} + +check_http() { + local name="$1" + local url="$2" + local waited=0 + + while ((waited < WAIT_TIMEOUT)); do + if curl --silent --show-error --fail "$url" >/dev/null; then + log "HTTP check passed for '$name': $url" + return 0 + fi + + log "Waiting for HTTP check '$name': $url" + sleep "$POLL_INTERVAL" + waited=$((waited + POLL_INTERVAL)) + done + + fail "HTTP check failed for '$name' after ${WAIT_TIMEOUT}s: $url" +} + +run_http_checks() { + local backend_port frontend_port + + backend_port=$(read_env_var BACKEND_PORT 3000) + frontend_port=$(read_env_var FRONTEND_PORT 8080) + + if ((${#DEPLOY_SERVICES[@]} == 0)); then + check_http "backend-ready" "http://127.0.0.1:${backend_port}/api/health/ready" + check_http "frontend-home" "http://127.0.0.1:${frontend_port}/" + return + fi + + for service_name in "${DEPLOY_SERVICES[@]}"; do + case "$service_name" in + backend-api) + check_http "backend-ready" "http://127.0.0.1:${backend_port}/api/health/ready" + ;; + frontend) + check_http "frontend-home" "http://127.0.0.1:${frontend_port}/" + ;; + esac + done +} + +print_compose_status() { + log "docker compose ps" + compose ps || true +} + +print_recent_logs() { + local services=("$@") + + if ((${#services[@]} == 0)); then + services=(backend-api processing-worker scraper-service frontend) + fi + + log "Recent docker compose logs" + compose logs --tail=100 "${services[@]}" || true +} + +write_release_snapshot() { + local release_id release_dir release_file repo_name repo_dir repo_sha var_name services_value + + release_id=$(date -u +'%Y%m%dT%H%M%SZ') + release_dir="$STATE_DIR/releases" + release_file="$release_dir/${release_id}.env" + services_value="" + + if ((${#DEPLOY_SERVICES[@]} > 0)); then + services_value=$(join_by ',' "${DEPLOY_SERVICES[@]}") + fi + + mkdir -p "$release_dir" + + { + printf 'RELEASE_ID=%s\n' "$release_id" + printf 'DEPLOYED_AT=%s\n' "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" + printf 'TRIGGER_REPO=%s\n' "$TARGET_REPO" + printf 'TRIGGER_BRANCH=%s\n' "$TARGET_BRANCH" + printf 'TRIGGER_SHA=%s\n' "$TARGET_SHA" + printf 'DEPLOY_SERVICES=%s\n' "${SERVICES_OVERRIDE:-$services_value}" + for repo_name in "${MANAGED_REPOS[@]}"; do + repo_dir=$(repo_path "$repo_name") + repo_sha=$(git_sha "$repo_dir") + var_name=$(snapshot_var_name "$repo_name") + printf '%s=%s\n' "$var_name" "$repo_sha" + done + } >"$release_file" + + cp "$release_file" "$STATE_DIR/last-success.env" + log "Saved release snapshot: $release_file" +} + +restore_repo_to_sha() { + local repo_name="$1" + local sha="$2" + local repo_dir + + repo_dir=$(repo_path "$repo_name") + log "Restoring repo '$repo_name' to ${sha}" + git -C "$repo_dir" checkout --detach "$sha" +} + +rollback_failed_deploy() { + local repo_name + + ((AUTO_ROLLBACK == 1)) || return 0 + ((${#UPDATED_REPOS[@]} > 0)) || return 0 + + warn "Deployment failed, starting automatic rollback" + + for repo_name in "${UPDATED_REPOS[@]}"; do + restore_repo_to_sha "$repo_name" "${BEFORE_SHA[$repo_name]}" + done + + if ((${#DEPLOY_SERVICES[@]} == 0)); then + compose up -d --build --remove-orphans + else + compose up -d --build --remove-orphans "${DEPLOY_SERVICES[@]}" + fi + + wait_for_compose_services "${DEPLOY_SERVICES[@]}" + run_http_checks + warn "Automatic rollback completed successfully" +} + +handle_error() { + local exit_code=$? + + trap - ERR + + if ((HANDLING_ERROR == 1)); then + exit "$exit_code" + fi + + HANDLING_ERROR=1 + + print_compose_status + print_recent_logs "${DEPLOY_SERVICES[@]}" + rollback_failed_deploy || true + print_compose_status + print_recent_logs "${DEPLOY_SERVICES[@]}" + + exit "$exit_code" +} + +perform_deploy() { + if ((UPDATE_INFRA == 1)) && [[ "$TARGET_REPO" != "infra" ]]; then + ensure_repo_ready infra + update_repo infra "$TARGET_BRANCH" + fi + + ensure_repo_ready "$TARGET_REPO" + update_repo "$TARGET_REPO" "$TARGET_BRANCH" + resolve_services + + log "Validating docker compose configuration" + compose config -q + + if ((${#DEPLOY_SERVICES[@]} == 0)); then + log "Deploying full stack" + compose up -d --build --remove-orphans + else + log "Deploying services: $(join_by ', ' "${DEPLOY_SERVICES[@]}")" + compose up -d --build --remove-orphans "${DEPLOY_SERVICES[@]}" + fi + + print_compose_status + wait_for_compose_services "${DEPLOY_SERVICES[@]}" + run_http_checks + write_release_snapshot + log "Production deploy finished successfully" +} + +main() { + parse_args "$@" + + require_command git + require_command docker + require_command curl + require_command flock + + mkdir -p "$STATE_DIR" + exec 9>"$LOCK_FILE" + + if ! flock -n 9; then + log "ERROR: Another deploy is already running. Lock file: $LOCK_FILE" + exit 1 + fi + + trap handle_error ERR + + log "Starting deploy" + log "Target repo: $TARGET_REPO" + log "Target branch: $TARGET_BRANCH" + log "Target sha: ${TARGET_SHA:-not provided}" + log "Deploy root: $DEPLOY_ROOT" + log "Infra dir: $INFRA_DIR" + log "Compose env file: $COMPOSE_ENV_FILE" + log "Auto rollback: $AUTO_ROLLBACK" + + perform_deploy + trap - ERR +} + +main "$@" diff --git a/scripts/rollback.sh b/scripts/rollback.sh new file mode 100755 index 0000000..d08c243 --- /dev/null +++ b/scripts/rollback.sh @@ -0,0 +1,311 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +IFS=$'\n\t' + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +INFRA_DIR=$(cd -- "$SCRIPT_DIR/.." && pwd) +DEPLOY_ROOT_DEFAULT=$(cd -- "$INFRA_DIR/.." && pwd) +DEPLOY_ROOT="${DEPLOY_PATH:-$DEPLOY_ROOT_DEFAULT}" +STATE_DIR="${DEPLOY_STATE_DIR:-$DEPLOY_ROOT/.deploy-state}" +LOCK_FILE="${DEPLOY_LOCK_FILE:-$STATE_DIR/deploy.lock}" +COMPOSE_ENV_FILE="${DEPLOY_ENV_FILE:-$INFRA_DIR/.env}" +WAIT_TIMEOUT="${DEPLOY_WAIT_TIMEOUT:-300}" +POLL_INTERVAL="${DEPLOY_POLL_INTERVAL:-5}" + +MANAGED_REPOS=(infra npp-web npp-backend processing-worker scrape-helper contracts) + +RELEASE_FILE="" +LIST_ONLY=0 + +log() { + printf '[%s] %s\n' "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" "$*" +} + +fail() { + log "ERROR: $*" + exit 1 +} + +usage() { + cat <<'EOF' +Usage: + ./scripts/rollback.sh [options] + +Options: + --release-file Roll back to an explicit release snapshot file. + --list List available release snapshots. + --help Show this help. + +Default behavior: + Rolls back to the previous successful release snapshot. +EOF +} + +require_command() { + local command_name="$1" + command -v "$command_name" >/dev/null 2>&1 || fail "Required command not found: $command_name" +} + +repo_path() { + local repo_name="$1" + + case "$repo_name" in + infra) + printf '%s\n' "$INFRA_DIR" + ;; + npp-web | npp-backend | processing-worker | scrape-helper | contracts) + printf '%s/%s\n' "$DEPLOY_ROOT" "$repo_name" + ;; + *) + fail "Unknown repo: $repo_name" + ;; + esac +} + +sha_var_name() { + local repo_name="$1" + + case "$repo_name" in + infra) + printf 'REPO_SHA_INFRA\n' + ;; + npp-web) + printf 'REPO_SHA_NPP_WEB\n' + ;; + npp-backend) + printf 'REPO_SHA_NPP_BACKEND\n' + ;; + processing-worker) + printf 'REPO_SHA_PROCESSING_WORKER\n' + ;; + scrape-helper) + printf 'REPO_SHA_SCRAPE_HELPER\n' + ;; + contracts) + printf 'REPO_SHA_CONTRACTS\n' + ;; + *) + fail "Unknown repo: $repo_name" + ;; + esac +} + +parse_args() { + while (($# > 0)); do + case "$1" in + --release-file) + RELEASE_FILE="${2:-}" + shift 2 + ;; + --list) + LIST_ONLY=1 + shift + ;; + --help) + usage + exit 0 + ;; + *) + fail "Unknown argument: $1" + ;; + esac + done +} + +list_releases() { + local release_dir + + release_dir="$STATE_DIR/releases" + [[ -d "$release_dir" ]] || fail "No release directory found: $release_dir" + + ls -1 "$release_dir" +} + +resolve_release_file() { + local release_dir selected_release + + if [[ -n "$RELEASE_FILE" ]]; then + [[ -f "$RELEASE_FILE" ]] || fail "Release file not found: $RELEASE_FILE" + return + fi + + release_dir="$STATE_DIR/releases" + [[ -d "$release_dir" ]] || fail "No release directory found: $release_dir" + + selected_release=$(ls -1 "$release_dir"/*.env 2>/dev/null | sort | tail -n 2 | head -n 1 || true) + [[ -n "$selected_release" ]] || fail "Could not resolve previous release snapshot" + + RELEASE_FILE="$selected_release" +} + +compose() { + ( + cd "$INFRA_DIR" + docker compose \ + --env-file "$COMPOSE_ENV_FILE" \ + -f docker-compose.yml \ + -f docker-compose.apps.yml \ + "$@" + ) +} + +service_status() { + local service_name="$1" + local container_id + + container_id=$(compose ps -q "$service_name") + [[ -n "$container_id" ]] || fail "Container ID not found for service '$service_name'" + + docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id" +} + +wait_for_service() { + local service_name="$1" + local waited=0 + local status="" + + while ((waited < WAIT_TIMEOUT)); do + status=$(service_status "$service_name") + + case "$status" in + healthy | running) + log "Service '$service_name' status: $status" + return 0 + ;; + exited | dead | unhealthy) + fail "Service '$service_name' entered bad state: $status" + ;; + *) + log "Waiting for service '$service_name' (status: $status)" + ;; + esac + + sleep "$POLL_INTERVAL" + waited=$((waited + POLL_INTERVAL)) + done + + fail "Timed out waiting for service '$service_name' after ${WAIT_TIMEOUT}s" +} + +wait_for_stack() { + local services=(postgres redis rabbitmq minio backend-api processing-worker scraper-service frontend) + local service_name + + for service_name in "${services[@]}"; do + wait_for_service "$service_name" + done +} + +read_env_var() { + local key="$1" + local default_value="$2" + local value="" + + if [[ -f "$COMPOSE_ENV_FILE" ]]; then + value=$(awk -F= -v key="$key" ' + $0 !~ /^[[:space:]]*#/ && $1 == key { + sub(/^[^=]*=/, "", $0) + print $0 + exit + } + ' "$COMPOSE_ENV_FILE") + fi + + if [[ -n "$value" ]]; then + printf '%s\n' "$value" + else + printf '%s\n' "$default_value" + fi +} + +check_http() { + local name="$1" + local url="$2" + local waited=0 + + while ((waited < WAIT_TIMEOUT)); do + if curl --silent --show-error --fail "$url" >/dev/null; then + log "HTTP check passed for '$name': $url" + return 0 + fi + + log "Waiting for HTTP check '$name': $url" + sleep "$POLL_INTERVAL" + waited=$((waited + POLL_INTERVAL)) + done + + fail "HTTP check failed for '$name' after ${WAIT_TIMEOUT}s: $url" +} + +rollback_repo() { + local repo_name="$1" + local sha_var repo_sha repo_dir + + sha_var=$(sha_var_name "$repo_name") + repo_sha="${!sha_var:-}" + [[ -n "$repo_sha" ]] || fail "Release snapshot does not contain SHA for '$repo_name'" + + repo_dir=$(repo_path "$repo_name") + + [[ -d "$repo_dir/.git" ]] || fail "Missing git repo for '$repo_name': $repo_dir" + if [[ -n "$(git -C "$repo_dir" status --porcelain)" ]]; then + fail "Repository has uncommitted changes: $repo_dir" + fi + + git -C "$repo_dir" fetch --prune origin + log "Rolling back '$repo_name' to ${repo_sha}" + git -C "$repo_dir" checkout --detach "$repo_sha" +} + +main() { + local backend_port frontend_port repo_name + + parse_args "$@" + + require_command git + require_command docker + require_command curl + require_command flock + + if ((LIST_ONLY == 1)); then + list_releases + exit 0 + fi + + resolve_release_file + + mkdir -p "$STATE_DIR" + exec 9>"$LOCK_FILE" + flock -n 9 || fail "Another deploy or rollback is already running. Lock file: $LOCK_FILE" + + log "Using release snapshot: $RELEASE_FILE" + + # shellcheck disable=SC1090 + source "$RELEASE_FILE" + + for repo_name in "${MANAGED_REPOS[@]}"; do + rollback_repo "$repo_name" + done + + log "Validating docker compose configuration" + compose config -q + + log "Recreating full stack from selected release" + compose up -d --build --remove-orphans + compose ps + + wait_for_stack + + backend_port=$(read_env_var BACKEND_PORT 3000) + frontend_port=$(read_env_var FRONTEND_PORT 8080) + + check_http "backend-ready" "http://127.0.0.1:${backend_port}/api/health/ready" + check_http "frontend-home" "http://127.0.0.1:${frontend_port}/" + + cp "$RELEASE_FILE" "$STATE_DIR/last-success.env" + log "Rollback completed successfully" +} + +main "$@"