e2e-cloud.yml

name: E2E Cloud Tests

on:
  workflow_dispatch:
    inputs:
      scenario:
        description: "Scenario YAML to run (relative to test-framework/)"
        required: true
        default: "scenarios/face-discovery-self.yaml"
      keep_session:
        description: "Keep session dir after run (for debugging)"
        required: false
        type: boolean
        default: false
  schedule:
    # Nightly at 07:00 UTC (midnight PT)
    - cron: "0 7 * * *"

concurrency:
  # One E2E run at a time — prevents dev cloud state conflicts.
  group: e2e-cloud
  cancel-in-progress: false

jobs:
  e2e:
    name: E2E Test Framework (dev cloud)
    runs-on: ubuntu-latest
    # Bumped 30 → 90 on 2026-05-03: scheduled nightly runs were being silently
    # cancelled at exactly 30m18s for 8 consecutive days because org-sync-2node's
    # first sync_trigger after a fresh org join takes ~18 min in fold_db's
    # SyncEngine::sync (engine.rs:599). The 30-min wall fired before the test
    # could fail cleanly with logs, so we couldn't even tell which I/O call was
    # slow. 90 min lets the test complete (or fail with logs) so we can localize
    # the hang. Track the underlying perf regression separately.
    # See gbrain projects/e2e-cloud-nightly-cancellation-2026-04-26.
    timeout-minutes: 90

    # Only run on the main repo (not forks) to protect secrets
    if: github.repository == 'EdgeVector/fold_db_node'

    env:
      RUSTFLAGS: -C linker=clang -C link-arg=-fuse-ld=mold
      # FOLDDB_TEST_DEV_API / FOLDDB_TEST_DEV_SCHEMA are exported from
      # environments.json after checkout (see "Resolve dev URLs" step).

    steps:
      - name: Free disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /usr/local/share/powershell /usr/share/swift
          sudo apt-get clean
          df -h /

      - name: Checkout repository
        uses: actions/checkout@v4

      # Single source of truth: environments.json. Resolved here so a URL
      # change in environments.json propagates to CI without editing this
      # workflow. Lint enforces no hardcoded gateway hostnames anywhere.
      - name: Resolve dev URLs from environments.json
        run: |
          sudo apt-get update && sudo apt-get install -y jq
          echo "FOLDDB_TEST_DEV_API=$(./scripts/get-env-url.sh dev exemem_api)" >> "$GITHUB_ENV"
          echo "FOLDDB_TEST_DEV_SCHEMA=$(./scripts/get-env-url.sh dev schema_service)" >> "$GITHUB_ENV"

      # Strip the local-dev cargo patch that redirects schema_service
      # deps to a sibling workspace checkout — CI has no sibling, so
      # the git dep + Cargo.lock pin must resolve instead.
      - name: Remove local-dev cargo patch
        run: rm -f .cargo/config.toml

      - name: Install mold linker
        run: sudo apt-get update && sudo apt-get install -y mold

      - name: Install jq + yq + ajv (scenario validation)
        run: |
          sudo apt-get install -y jq
          sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
          sudo chmod +x /usr/local/bin/yq
          sudo npm install -g ajv-cli
          jq --version
          yq --version
          ajv help | head -1

      - name: Install Python + PyNaCl (for node factory key generation)
        run: |
          python3 -m pip install --quiet pynacl

      - name: Install Rust
        uses: dtolnay/rust-toolchain@stable

      - name: Configure git for private dependencies
        run: git config --global url."https://x-access-token:${{ secrets.GH_PAT }}@github.com/".insteadOf "https://github.com/"

      - name: Cache Rust artifacts
        uses: Swatinem/rust-cache@v2

      - name: Install Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "20"

      - name: Build React frontend
        run: |
          cd src/server/static-react
          npm ci
          npm run build

      - name: Build folddb_server (face-detection feature)
        run: cargo build --features face-detection --bin folddb_server

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_E2E_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_E2E_SECRET_ACCESS_KEY }}
          aws-region: us-west-2

      - name: Write test admin secret to disk
        run: |
          mkdir -p ~/.folddb
          echo -n "${{ secrets.FOLDDB_TEST_ADMIN_SECRET }}" > ~/.folddb/test-admin-secret-dev.txt
          chmod 600 ~/.folddb/test-admin-secret-dev.txt

      - name: Run scenario (manual trigger)
        if: github.event_name == 'workflow_dispatch'
        env:
          # folddb_server's ingestion service needs an Anthropic key to
          # classify/extract images; without it the /ingest endpoint
          # returns 503 and the scenario fails at the first ingest step.
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          # Route image → markdown through Anthropic vision. The runner
          # doesn't have Ollama installed, so the default (Ollama)
          # path would fail at ConnectionRefused @ localhost:11434.
          INGESTION_VISION_BACKEND: anthropic
        run: |
          cd test-framework
          SCENARIO="${{ github.event.inputs.scenario }}"
          KEEP="${{ github.event.inputs.keep_session }}"
          ARGS=""
          if [[ "$KEEP" == "true" ]]; then
            ARGS="--keep-session"
          fi
          bash run-scenario.sh "$SCENARIO" $ARGS

      - name: Run all nightly scenarios
        if: github.event_name == 'schedule'
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          INGESTION_VISION_BACKEND: anthropic
        run: |
          cd test-framework
          set +e
          FAIL=0
          for scenario in scenarios/face-discovery-self.yaml scenarios/face-discovery-3node.yaml scenarios/org-sync-2node.yaml; do
            echo ""
            echo "================================================================"
            echo "  Running: $scenario"
            echo "================================================================"
            bash run-scenario.sh "$scenario"
            rc=$?
            if [[ $rc -ne 0 ]]; then
              echo "[nightly] $scenario FAILED (rc=$rc)"
              FAIL=1
            else
              echo "[nightly] $scenario PASSED"
            fi
          done
          exit $FAIL

      - name: Upload session logs
        # `if: always()` (was `if: failure()`) so the artifact uploads on
        # cancellation too. The 30-min cancellation pattern (2026-04-26 →
        # 2026-05-03) produced zero artifacts because cancel skips `failure()`.
        # We need the logs from cancelled runs to localize the sync hang.
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-session-logs
          path: test-framework/logs/
          retention-days: 7

      - name: Summary
        # Per-scenario, per-assertion summary table + ::error:: annotations
        # for any failed assertion / failed step / incomplete run. Replaces
        # the old `find -name "*.failed"` summary, which never matched
        # anything (run-scenario.sh doesn't emit those marker files), so
        # every run was silently labelled ✅ PASS regardless of outcome.
        # Stdout → step summary, stderr → job log (where GHA parses
        # ::error:: lines into top-of-page annotations).
        if: always()
        run: |
          {
            echo "## E2E Cloud Test Results"
            echo ""
            bash test-framework/scripts/render-ci-summary.sh test-framework/logs
          } >> "$GITHUB_STEP_SUMMARY"