deploy-dev.yml

# GHO-26: Health checks after deployment
name: Deploy to Dev Environment

on:
  push:
    branches:
      - develop
  workflow_dispatch:
    
permissions:
  contents: read
  packages: read

jobs:
  # Detect which files changed to determine if we need to deploy
  changes:
    runs-on: ubuntu-latest
    outputs:
      infra: ${{ steps.filter.outputs.infra }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Check for infrastructure file changes
        uses: dorny/paths-filter@v4
        id: filter
        with:
          base: ${{ github.event.before }}
          filters: |
            infra:
              - 'opentofu/**/*.tofu'
              - 'opentofu/**/*.tfvars'
              - 'opentofu/**/*.bu'
              - 'opentofu/**/*.tftpl'
              - 'opentofu/**/*.sh'
              - 'opentofu/**/userdata/**'
              - 'docker/scripts/**'

  deploy:
    needs: changes
    if: needs.changes.outputs.infra == 'true'
    runs-on: ubuntu-latest
    environment: dev

    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
        with:
          fetch-depth: 0

      # Required if the GHCR image is private
      - name: Log in to GHCR
        env:
          GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
        run: |
          docker login ghcr.io -u "noahwhite" --password-stdin <<< "$GHCR_TOKEN"

      - name: Pull deployment tools image
        run: |
          docker pull ghcr.io/noahwhite/ghost-stack-shell:latest

      - name: Verify deployment context
        run: |
          echo "✅ Deployment workflow triggered for develop branch"
          echo "   Environment: dev"
          echo "   Working directory: ${GITHUB_WORKSPACE}"
          echo "   Runner: $(uname -a)"

      - name: Verify tooling availability
        run: |
          docker run --rm \
            -v "${GITHUB_WORKSPACE}:/home/devops/app" \
            -w /home/devops/app \
            ghcr.io/noahwhite/ghost-stack-shell:latest \
            bash -c "git config --global --add safe.directory /home/devops/app && echo '✅ OpenTofu tooling verified' && tofu version"

      - name: Extract PR number from merge commit
        id: pr
        env:
          COMMIT_MSG: ${{ github.event.head_commit.message }}
        run: |
          PR_NUM=$(echo "$COMMIT_MSG" | grep -oP '\(#\K\d+(?=\))' | head -1)
          if [ -z "$PR_NUM" ]; then
            echo "❌ No PR number found in commit message"
            echo "   This deployment requires a PR merge commit"
            exit 1
          fi
          echo "number=$PR_NUM" >> $GITHUB_OUTPUT
          echo "✅ Found PR #$PR_NUM"

      - name: Download plan artifact from PR
        id: download-artifact
        uses: dawidd6/action-download-artifact@v20
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          workflow: pr-tofu-plan-develop.yml
          pr: ${{ steps.pr.outputs.number }}
          name: tofu-plan-dev-${{ steps.pr.outputs.number }}
          path: ./pr-plan
          if_no_artifact_found: warn

      - name: Check if deployment should proceed
        id: skip-check
        run: |
          # Check if plan artifact exists (no artifact = no infra changes in PR)
          if [ ! -f ./pr-plan/plan-output.txt ]; then
            echo "✅ No plan artifact found for PR #${{ steps.pr.outputs.number }}"
            echo "   No infrastructure files were changed in this PR."
            echo "   Skipping deployment."
            echo "skip=true" >> $GITHUB_OUTPUT
          else
            echo "✅ Plan artifact verified"
            echo "Plan from PR #${{ steps.pr.outputs.number }}:"
            head -20 ./pr-plan/plan-output.txt
            echo "skip=false" >> $GITHUB_OUTPUT
          fi

      - name: Install Bitwarden Secrets Manager CLI (bws)
        if: steps.skip-check.outputs.skip != 'true'
        run: |
          set -euo pipefail
          curl -fsSL \
            https://github.com/bitwarden/sdk-sm/releases/download/bws-v1.0.0/bws-x86_64-unknown-linux-gnu-1.0.0.zip \
            -o /tmp/bws.zip
          unzip -q /tmp/bws.zip -d /tmp/bws
          sudo mv /tmp/bws/bws /usr/local/bin/bws
          sudo chmod +x /usr/local/bin/bws
          bws --version

      - name: Retrieve secrets via infra-shell.sh (CI mode)
        if: steps.skip-check.outputs.skip != 'true'
        env:
          BWS_ACCESS_TOKEN: ${{ secrets.BWS_ACCESS_TOKEN }}
          BOOTSTRAP_R2_BUCKET: ${{ vars.BOOTSTRAP_R2_BUCKET }}
          ADMIN_IP: ${{ secrets.ADMIN_IP }}
          CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }}
        run: |
          ./docker/scripts/infra-shell.sh --ci --secrets-only --export-github-env

      - name: Fix workspace permissions for Docker container
        if: steps.skip-check.outputs.skip != 'true'
        run: |
          chmod -R a+w "${GITHUB_WORKSPACE}/opentofu"

      - name: Run OpenTofu init
        if: steps.skip-check.outputs.skip != 'true'
        env:
          TF_VAR_cloudflare_account_id: ${{ env.TF_VAR_cloudflare_account_id }}
          TF_VAR_cloudflare_api_token: ${{ env.TF_VAR_cloudflare_api_token }}
          R2_ACCESS_KEY_ID: ${{ env.R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ env.R2_SECRET_ACCESS_KEY }}
          TF_VAR_vultr_api_key: ${{ env.TF_VAR_vultr_api_key }}
          TAILSCALE_API_KEY: ${{ env.TAILSCALE_API_KEY }}
          TAILSCALE_TAILNET: ${{ env.TAILSCALE_TAILNET }}
          TF_VAR_tailscale_tailnet: ${{ env.TF_VAR_tailscale_tailnet }}
          TF_VAR_PD_CLIENT_ID: ${{ env.TF_VAR_PD_CLIENT_ID }}
          TF_VAR_PD_CLIENT_SECRET: ${{ env.TF_VAR_PD_CLIENT_SECRET }}
          TF_VAR_pd_subdomain: ${{ env.TF_VAR_pd_subdomain }}
          TF_VAR_pd_user_tok: ${{ env.TF_VAR_pd_user_tok }}
          TF_VAR_GC_ACCESS_TOK: ${{ env.TF_VAR_GC_ACCESS_TOK }}
          TF_VAR_SOC_DEV_TERRAFORM_SA_TOK: ${{ env.TF_VAR_SOC_DEV_TERRAFORM_SA_TOK }}
          TF_BACKEND_BUCKET: ${{ env.TF_BACKEND_BUCKET }}
        run: |
          docker run --rm \
            -v "${GITHUB_WORKSPACE}:/home/devops/app" \
            -w /home/devops/app \
            -e TF_VAR_cloudflare_account_id \
            -e TF_VAR_cloudflare_api_token \
            -e R2_ACCESS_KEY_ID \
            -e R2_SECRET_ACCESS_KEY \
            -e TF_VAR_vultr_api_key \
            -e TAILSCALE_API_KEY \
            -e TAILSCALE_TAILNET \
            -e TF_VAR_tailscale_tailnet \
            -e TF_VAR_PD_CLIENT_ID \
            -e TF_VAR_PD_CLIENT_SECRET \
            -e TF_VAR_pd_subdomain \
            -e TF_VAR_pd_user_tok \
            -e TF_VAR_GC_ACCESS_TOK \
            -e TF_VAR_SOC_DEV_TERRAFORM_SA_TOK \
            -e TF_BACKEND_BUCKET \
            ghcr.io/noahwhite/ghost-stack-shell:latest \
            bash -c "git config --global --add safe.directory /home/devops/app && ./opentofu/scripts/tofu.sh dev init"

      - name: Generate fresh plan for comparison
        if: steps.skip-check.outputs.skip != 'true'
        env:
          TF_VAR_cloudflare_account_id: ${{ env.TF_VAR_cloudflare_account_id }}
          TF_VAR_cloudflare_api_token: ${{ env.TF_VAR_cloudflare_api_token }}
          R2_ACCESS_KEY_ID: ${{ env.R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ env.R2_SECRET_ACCESS_KEY }}
          TF_VAR_vultr_api_key: ${{ env.TF_VAR_vultr_api_key }}
          TAILSCALE_API_KEY: ${{ env.TAILSCALE_API_KEY }}
          TAILSCALE_TAILNET: ${{ env.TAILSCALE_TAILNET }}
          TF_VAR_tailscale_tailnet: ${{ env.TF_VAR_tailscale_tailnet }}
          TF_VAR_PD_CLIENT_ID: ${{ env.TF_VAR_PD_CLIENT_ID }}
          TF_VAR_PD_CLIENT_SECRET: ${{ env.TF_VAR_PD_CLIENT_SECRET }}
          TF_VAR_pd_subdomain: ${{ env.TF_VAR_pd_subdomain }}
          TF_VAR_pd_user_tok: ${{ env.TF_VAR_pd_user_tok }}
          TF_VAR_GC_ACCESS_TOK: ${{ env.TF_VAR_GC_ACCESS_TOK }}
          TF_VAR_SOC_DEV_TERRAFORM_SA_TOK: ${{ env.TF_VAR_SOC_DEV_TERRAFORM_SA_TOK }}
          TF_BACKEND_BUCKET: ${{ env.TF_BACKEND_BUCKET }}
          TF_VAR_admin_subnets: ${{ env.TF_VAR_admin_subnets }}
          TF_VAR_admin_ip: ${{ env.TF_VAR_admin_ip }}
          TF_VAR_ssh_public_key: ${{ env.TF_VAR_ssh_public_key }}
          TF_VAR_cloudflare_zone_id: ${{ env.TF_VAR_cloudflare_zone_id }}
          TF_VAR_infisical_client_id: ${{ env.TF_VAR_infisical_client_id }}
          TF_VAR_infisical_client_secret: ${{ env.TF_VAR_infisical_client_secret }}
          TF_VAR_infisical_org_id: ${{ env.TF_VAR_infisical_org_id }}
        run: |
          docker run --rm \
            -v "${GITHUB_WORKSPACE}:/home/devops/app" \
            -w /home/devops/app \
            -e TF_VAR_cloudflare_account_id \
            -e TF_VAR_cloudflare_api_token \
            -e R2_ACCESS_KEY_ID \
            -e R2_SECRET_ACCESS_KEY \
            -e TF_VAR_vultr_api_key \
            -e TAILSCALE_API_KEY \
            -e TAILSCALE_TAILNET \
            -e TF_VAR_tailscale_tailnet \
            -e TF_VAR_PD_CLIENT_ID \
            -e TF_VAR_PD_CLIENT_SECRET \
            -e TF_VAR_pd_subdomain \
            -e TF_VAR_pd_user_tok \
            -e TF_VAR_GC_ACCESS_TOK \
            -e TF_VAR_SOC_DEV_TERRAFORM_SA_TOK \
            -e TF_BACKEND_BUCKET \
            -e TF_VAR_admin_subnets \
            -e TF_VAR_admin_ip \
            -e TF_VAR_ssh_public_key \
            -e TF_VAR_cloudflare_zone_id \
            -e TF_VAR_infisical_client_id \
            -e TF_VAR_infisical_client_secret \
            -e TF_VAR_infisical_org_id \
            ghcr.io/noahwhite/ghost-stack-shell:latest \
            bash -c "git config --global --add safe.directory /home/devops/app && ./opentofu/scripts/tofu.sh dev plan -out=tfplan"

      - name: Generate human-readable current plan
        if: steps.skip-check.outputs.skip != 'true'
        run: |
          docker run --rm \
            -v "${GITHUB_WORKSPACE}:/home/devops/app" \
            -w /home/devops/app \
            ghcr.io/noahwhite/ghost-stack-shell:latest \
            bash -c "git config --global --add safe.directory /home/devops/app && tofu -chdir=opentofu/envs/dev show -no-color tfplan > opentofu/envs/dev/current-plan.txt"

      - name: Compare PR plan with current plan
        if: steps.skip-check.outputs.skip != 'true'
        run: |
          echo "Comparing PR plan with current state..."

          # Filter out ephemeral values that change between plan runs
          # - kvm: Vultr KVM console URL contains session tokens that regenerate on every API call
          # - allowed_bandwidth: Vultr bandwidth allocation changes over time (monthly reset/accumulation)
          filter_ephemeral() {
            grep -v '~ kvm' | grep -v '~ allowed_bandwidth'
          }

          filter_ephemeral < ./pr-plan/plan-output.txt > /tmp/pr-plan-filtered.txt
          filter_ephemeral < ./opentofu/envs/dev/current-plan.txt > /tmp/current-plan-filtered.txt

          if diff -u /tmp/pr-plan-filtered.txt /tmp/current-plan-filtered.txt; then
            echo "✅ Plans match - no drift detected"
            echo "PLANS_MATCH=true" >> $GITHUB_ENV
          else
            echo "❌ Plans differ - state has drifted since PR approval"
            echo "   PR was approved based on a different plan"
            echo "   Manual review required"
            echo "PLANS_MATCH=false" >> $GITHUB_ENV
            exit 1
          fi

      - name: Apply infrastructure changes
        if: steps.skip-check.outputs.skip != 'true' && env.PLANS_MATCH == 'true'
        env:
          TF_VAR_cloudflare_account_id: ${{ env.TF_VAR_cloudflare_account_id }}
          TF_VAR_cloudflare_api_token: ${{ env.TF_VAR_cloudflare_api_token }}
          R2_ACCESS_KEY_ID: ${{ env.R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ env.R2_SECRET_ACCESS_KEY }}
          TF_VAR_vultr_api_key: ${{ env.TF_VAR_vultr_api_key }}
          TAILSCALE_API_KEY: ${{ env.TAILSCALE_API_KEY }}
          TAILSCALE_TAILNET: ${{ env.TAILSCALE_TAILNET }}
          TF_VAR_tailscale_tailnet: ${{ env.TF_VAR_tailscale_tailnet }}
          TF_VAR_PD_CLIENT_ID: ${{ env.TF_VAR_PD_CLIENT_ID }}
          TF_VAR_PD_CLIENT_SECRET: ${{ env.TF_VAR_PD_CLIENT_SECRET }}
          TF_VAR_pd_subdomain: ${{ env.TF_VAR_pd_subdomain }}
          TF_VAR_pd_user_tok: ${{ env.TF_VAR_pd_user_tok }}
          TF_VAR_GC_ACCESS_TOK: ${{ env.TF_VAR_GC_ACCESS_TOK }}
          TF_VAR_SOC_DEV_TERRAFORM_SA_TOK: ${{ env.TF_VAR_SOC_DEV_TERRAFORM_SA_TOK }}
          TF_BACKEND_BUCKET: ${{ env.TF_BACKEND_BUCKET }}
          TF_VAR_admin_subnets: ${{ env.TF_VAR_admin_subnets }}
          TF_VAR_admin_ip: ${{ env.TF_VAR_admin_ip }}
          TF_VAR_ssh_public_key: ${{ env.TF_VAR_ssh_public_key }}
          TF_VAR_cloudflare_zone_id: ${{ env.TF_VAR_cloudflare_zone_id }}
          TF_VAR_infisical_client_id: ${{ env.TF_VAR_infisical_client_id }}
          TF_VAR_infisical_client_secret: ${{ env.TF_VAR_infisical_client_secret }}
          TF_VAR_infisical_org_id: ${{ env.TF_VAR_infisical_org_id }}
        run: |
          docker run --rm \
            -v "${GITHUB_WORKSPACE}:/home/devops/app" \
            -w /home/devops/app \
            -e TF_VAR_cloudflare_account_id \
            -e TF_VAR_cloudflare_api_token \
            -e R2_ACCESS_KEY_ID \
            -e R2_SECRET_ACCESS_KEY \
            -e TF_VAR_vultr_api_key \
            -e TAILSCALE_API_KEY \
            -e TAILSCALE_TAILNET \
            -e TF_VAR_tailscale_tailnet \
            -e TF_VAR_PD_CLIENT_ID \
            -e TF_VAR_PD_CLIENT_SECRET \
            -e TF_VAR_pd_subdomain \
            -e TF_VAR_pd_user_tok \
            -e TF_VAR_GC_ACCESS_TOK \
            -e TF_VAR_SOC_DEV_TERRAFORM_SA_TOK \
            -e TF_BACKEND_BUCKET \
            -e TF_VAR_admin_subnets \
            -e TF_VAR_admin_ip \
            -e TF_VAR_ssh_public_key \
            -e TF_VAR_cloudflare_zone_id \
            -e TF_VAR_infisical_client_id \
            -e TF_VAR_infisical_client_secret \
            -e TF_VAR_infisical_org_id \
            ghcr.io/noahwhite/ghost-stack-shell:latest \
            bash -c "git config --global --add safe.directory /home/devops/app && ./opentofu/scripts/tofu.sh dev apply tfplan"

      - name: Verify Ghost health after deployment
        if: steps.skip-check.outputs.skip != 'true' && env.PLANS_MATCH == 'true'
        env:
          HEALTH_CHECK_TOKEN: ${{ secrets.HEALTH_CHECK_TOKEN }}
        run: |
          echo "🏥 Starting health checks for Ghost deployment..."

          MAX_RETRIES=30
          RETRY_INTERVAL=10
          URL="https://separationofconcerns.dev"

          for i in $(seq 1 $MAX_RETRIES); do
            echo "Health check attempt $i of $MAX_RETRIES..."
            HTTP_CODE=$(curl -sSL --max-time 10 -o /dev/null -w "%{http_code}" \
              -H "X-Health-Check-Token: ${HEALTH_CHECK_TOKEN}" \
              "$URL" 2>&1 || echo "000")

            if [ "$HTTP_CODE" = "200" ]; then
              echo "✅ Ghost is healthy - received HTTP $HTTP_CODE"
              echo "HEALTH_CHECK_PASSED=true" >> $GITHUB_ENV
              exit 0
            fi

            echo "   Received HTTP $HTTP_CODE, waiting ${RETRY_INTERVAL}s before retry..."
            sleep $RETRY_INTERVAL
          done

          echo "❌ Health check failed after $MAX_RETRIES attempts ($(($MAX_RETRIES * $RETRY_INTERVAL))s timeout)"
          echo "   Last HTTP response code: $HTTP_CODE"
          exit 1

      - name: Deployment summary
        if: always()
        run: |
          echo "================================"
          echo "Deployment Summary"
          echo "================================"
          if [ "${{ steps.skip-check.outputs.skip }}" == "true" ]; then
            echo "⏭️ Deployment skipped - no infrastructure changes in PR"
            echo "   PR #${{ steps.pr.outputs.number }} merged without infrastructure deployment"
          elif [ "${{ env.PLANS_MATCH }}" == "true" ] && [ "${{ env.HEALTH_CHECK_PASSED }}" == "true" ]; then
            echo "✅ Infrastructure changes applied successfully"
            echo "✅ Ghost health check passed"
            echo "   PR #${{ steps.pr.outputs.number }} deployed to dev environment"
            echo "   URL: https://separationofconcerns.dev"
          elif [ "${{ env.PLANS_MATCH }}" == "true" ]; then
            echo "✅ Infrastructure changes applied"
            echo "❌ Ghost health check failed"
            echo "   PR #${{ steps.pr.outputs.number }} - deployment may need investigation"
          else
            echo "❌ Deployment failed - plans did not match"
            echo "   State drift detected since PR approval"
          fi