e2e.yml

name: 'E2E Tests'

on:
  # E2E is slow and currently flaky, so it is NOT in the merge queue (gating the
  # serial queue on it would stall every merge). It runs post-merge on `main`,
  # plus a nightly full regression and on-demand. Promote it to `merge_group` +
  # required once a stable subset is carved out.
  push:
    branches:
      - 'main'
      - 'feat/e2e/**'
  schedule:
    - cron: '0 4 * * *' # nightly full regression (~04:00 UTC), guaranteed signal
  workflow_dispatch:

concurrency:
  # Scope the group by event so pushes to `main` coalesce with each other without
  # ever touching the nightly schedule or a manual dispatch — they share the ref
  # but not the event. feat/e2e/** pushes coalesce within their own branch.
  group: |-
    ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.ref_name }}
  # Do NOT cancel in-progress runs on `main`: a full run takes ~40min while
  # merges land every ~18min (median), so cancelling on every merge starved the
  # suite — over 100 push runs, 67 were cancelled and only 25 ever reported.
  # With cancellation off, GitHub keeps at most ONE pending run per group and
  # cancels the previously pending one, so the queue collapses to the newest
  # tree on its own: the in-flight run always finishes, and each result covers
  # the batch of commits merged since the last one (bisect that range when it
  # goes red). Dev branches keep cancelling superseded runs — nobody bisects
  # those, and the latest push is the only tree of interest.
  cancel-in-progress: |-
    ${{ github.event_name == 'push' && github.ref_name != 'main' }}

jobs:
  e2e-test-linux:
    name: 'E2E Test (Linux) - ${{ matrix.sandbox }} - shard ${{ matrix.shard }}'
    runs-on: 'ubuntu-latest'
    # Skip on fork PRs: forks have no access to repository secrets
    # (OPENAI_*, DOCKERHUB_*), so the matrix would fail unconditionally
    # and show misleading red status. Same-repo PRs run normally.
    if: |-
      ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
    strategy:
      # One failing shard must not cancel the others: a partial matrix hides the
      # rest of the suite and reports an incomplete failure set.
      fail-fast: false
      matrix:
        sandbox:
          - 'sandbox:none'
          - 'sandbox:docker'
        # The suite is ~16min of wall clock on one runner, dominated by a long
        # tail of sdk-typescript files. vitest assigns files to shards by path
        # hash, so those spread out instead of clustering in one shard.
        shard:
          - '1/3'
          - '2/3'
          - '3/3'
        node-version:
          - '22.x'
    steps:
      - name: 'Checkout'
        uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3

      - name: 'Set up Node.js ${{ matrix.node-version }}'
        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0
        with:
          node-version: '${{ matrix.node-version }}'
          cache: 'npm'
          cache-dependency-path: 'package-lock.json'
          registry-url: 'https://registry.npmjs.org/'

      - name: 'Configure npm for rate limiting'
        run: |-
          npm config set fetch-retry-mintimeout 20000
          npm config set fetch-retry-maxtimeout 120000
          npm config set fetch-retries 5
          npm config set fetch-timeout 300000

      - name: 'Install dependencies'
        env:
          # `npm ci` runs the `prepare` script, which builds and bundles the
          # whole workspace — and the two steps below then do it a second time.
          # Skip it so the install only installs; `prepare` still generates
          # git-commit.ts, which the build needs. (The web-shell job below has
          # no build step of its own, so it keeps building during install.)
          QWEN_SKIP_PREPARE: '1'
        run: |-
          npm ci --prefer-offline --no-audit --progress=false

      - name: 'Build project'
        run: |-
          npm run build

      - name: 'Bundle CLI for E2E tests'
        run: |-
          npm run bundle

      - name: 'Set up Docker'
        if: |-
          ${{ matrix.sandbox == 'sandbox:docker' }}
        uses: 'docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5' # ratchet:docker/setup-buildx-action@v4

      - name: 'Set up Podman'
        if: |-
          ${{ matrix.sandbox == 'sandbox:podman' }}
        uses: 'redhat-actions/podman-login@4934294ad0449894bcd1e9f191899d7292469603' # ratchet:redhat-actions/podman-login@v1
        with:
          registry: 'docker.io'
          username: '${{ secrets.DOCKERHUB_USERNAME }}'
          password: '${{ secrets.DOCKERHUB_TOKEN }}'

      # The sandbox test script builds this image itself, but without `-s` it
      # first re-runs install + build + bundle + pack on the host — all of which
      # the steps above already did, and none of which the image consumes (the
      # Dockerfile rebuilds and packs inside its own builder stage).
      - name: 'Build the sandbox image'
        if: |-
          ${{ matrix.sandbox == 'sandbox:docker' }}
        env:
          # build_sandbox.js resolves the container command through
          # sandbox_command.js, which exits non-zero on Linux when this is unset
          # — the test script used to supply it.
          QWEN_SANDBOX: 'docker'
          # Without this, build_sandbox.js pipes docker build output to
          # /dev/null — the longest step in the docker leg becomes undiagnosable.
          VERBOSE: 'true'
        run: |-
          npm run build:sandbox -- -s

      - name: 'Run E2E tests'
        env:
          OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}'
          OPENAI_BASE_URL: '${{ secrets.OPENAI_BASE_URL }}'
          OPENAI_MODEL: '${{ secrets.OPENAI_MODEL }}'
          KEEP_OUTPUT: 'true'
          VERBOSE: 'true'
        run: |-
          # The docker leg runs vitest directly instead of through
          # test:integration:sandbox:docker: that script would rebuild the image
          # the step above just built.
          if [[ "${{ matrix.sandbox }}" == "sandbox:docker" ]]; then
            npx cross-env QWEN_SANDBOX=docker vitest run --root ./integration-tests --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}'
          else
            npm run test:integration:sandbox:none -- --exclude '**/interactive/cron-interactive.test.ts' --exclude '**/channel-plugin.test.ts' --shard='${{ matrix.shard }}'
          fi

  e2e-test-macos:
    name: 'E2E Test - macOS - shard ${{ matrix.shard }}'
    runs-on: 'macos-latest'
    # Skip on fork PRs (no secrets) — see e2e-test-linux above.
    if: |-
      ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
    strategy:
      fail-fast: false
      matrix:
        # Two shards, not three: the macOS runner is the slowest to install and
        # build, so a third shard would add more fixed cost than it removes.
        shard:
          - '1/2'
          - '2/2'
    steps:
      - name: 'Checkout'
        uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3

      - name: 'Set up Node.js'
        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'
          cache-dependency-path: 'package-lock.json'
          registry-url: 'https://registry.npmjs.org/'

      - name: 'Configure npm for rate limiting'
        run: |-
          npm config set fetch-retry-mintimeout 20000
          npm config set fetch-retry-maxtimeout 120000
          npm config set fetch-retries 5
          npm config set fetch-timeout 300000

      - name: 'Install dependencies'
        env:
          # See e2e-test-linux: the install must not build, the two steps
          # below do it explicitly.
          QWEN_SKIP_PREPARE: '1'
        run: |-
          npm ci --prefer-offline --no-audit --progress=false

      - name: 'Build project'
        run: |-
          npm run build

      - name: 'Bundle CLI for E2E tests'
        run: |-
          npm run bundle

      - name: 'Run E2E tests'
        env:
          OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}'
          OPENAI_BASE_URL: '${{ secrets.OPENAI_BASE_URL }}'
          OPENAI_MODEL: '${{ secrets.OPENAI_MODEL }}'
        run: 'npx cross-env VERBOSE=true KEEP_OUTPUT=true QWEN_SANDBOX=false vitest run --root ./integration-tests --exclude "**/interactive/cron-interactive.test.ts" --exclude "**/channel-plugin.test.ts" --shard="${{ matrix.shard }}"'

  isolated-nightly:
    name: '${{ matrix.label }} (nightly)'
    runs-on: 'ubuntu-latest'
    if: |-
      ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
    continue-on-error: true
    strategy:
      fail-fast: false
      matrix:
        include:
          - label: 'cron-interactive E2E'
            test_file: 'interactive/cron-interactive.test.ts'
          - label: 'channel-plugin E2E'
            test_file: 'channel-plugin.test.ts'
    steps:
      - name: 'Checkout'
        uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3

      - name: 'Set up Node.js'
        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'
          cache-dependency-path: 'package-lock.json'
          registry-url: 'https://registry.npmjs.org/'

      - name: 'Configure npm for rate limiting'
        run: |-
          npm config set fetch-retry-mintimeout 20000
          npm config set fetch-retry-maxtimeout 120000
          npm config set fetch-retries 5
          npm config set fetch-timeout 300000

      - name: 'Install dependencies'
        env:
          # See e2e-test-linux: the install must not build, the two steps
          # below do it explicitly.
          QWEN_SKIP_PREPARE: '1'
        run: |-
          npm ci --prefer-offline --no-audit --progress=false

      - name: 'Build project'
        run: |-
          npm run build

      - name: 'Bundle CLI for E2E tests'
        run: |-
          npm run bundle

      - name: 'Run ${{ matrix.label }} tests'
        env:
          OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}'
          OPENAI_BASE_URL: '${{ secrets.OPENAI_BASE_URL }}'
          OPENAI_MODEL: '${{ secrets.OPENAI_MODEL }}'
          KEEP_OUTPUT: 'true'
          VERBOSE: 'true'
        run: 'npx cross-env QWEN_SANDBOX=false vitest run --root ./integration-tests "${{ matrix.test_file }}"'

  web-shell-browser-regression:
    name: 'web-shell Browser Regression'
    runs-on: 'ubuntu-latest'
    if: |-
      ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
    steps:
      - name: 'Checkout'
        uses: 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd' # v6.0.2

      - name: 'Set up Node.js'
        uses: 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e' # v6.4.0
        with:
          node-version-file: '.nvmrc'
          cache: 'npm'
          cache-dependency-path: 'package-lock.json'
          registry-url: 'https://registry.npmjs.org/'

      - name: 'Configure npm for rate limiting'
        run: |-
          npm config set fetch-retry-mintimeout 20000
          npm config set fetch-retry-maxtimeout 120000
          npm config set fetch-retries 5
          npm config set fetch-timeout 300000

      - name: 'Install dependencies'
        run: |-
          npm ci --prefer-offline --no-audit --progress=false

      - name: 'Install Playwright Chromium'
        run: 'npx playwright install --with-deps chromium'

      - name: 'Run web-shell browser regression'
        run: 'npm run test:e2e --workspace=packages/web-shell'

      - name: 'Upload web-shell Playwright artifacts'
        if: '${{ always() }}'
        uses: 'actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' # v7.0.1
        with:
          name: 'web-shell-browser-regression'
          path: |-
            packages/web-shell/client/e2e/test-results
            packages/web-shell/client/e2e/playwright-report
          if-no-files-found: 'ignore'