[e2e] Add Python e2e Test

.changeset/python-workbench-app.md

@@ -0,0 +1,4 @@
+---
+---
+
+Gate the e2e suite for cross-language conformance — tests whose subject is the JS implementation are marked JS-only, and an app declares via `e2e-conformance.json` which `99_e2e` fixtures it implements plus any individual tests its runtime cannot yet support — and add `workbench/python` as the first non-JavaScript target. No-op for the existing workbench apps, which ship no declaration.

.github/scripts/aggregate-e2e-results.js

@@ -55,6 +55,11 @@ function findResultFiles(dir) {
     'e2e-failures-',
     'e2e-diagnostics-',
     'e2e-runtime-logs-',
+    // Not a report: the per-app cross-language conformance declaration
+    // (`workbench/*/e2e-conformance.json` and its `.example` sibling). The
+    // trailing dot matters — the Python lane's report is
+    // `e2e-conformance-python.json` and must keep matching.
+    'e2e-conformance.',
   ]);
 }
 
@@ -260,6 +265,7 @@ function parseJobInfo(filename) {
           'mongodb',
           'redis',
           'starter',
+          'python',
           'nest',
           'tanstack',
         ].some((app) => p.startsWith(app))
@@ -420,6 +426,7 @@ const categoryNames = {
   'local-prod': '📦 Local Production',
   'local-postgres': '🐘 Local Postgres',
   windows: '🪟 Windows',
+  conformance: '🌐 Cross-language Conformance',
   community: '🌍 Community Worlds',
   other: '📋 Other',
 };
@@ -431,6 +438,7 @@ const categoryOrder = [
   'local-prod',
   'local-postgres',
   'windows',
+  'conformance',
   'community',
   'other',
 ];

.github/scripts/aggregate-e2e-results.test.js

@@ -52,6 +52,23 @@ function renderAggregate(files) {
   }
 }
 
+// Same, but the per-job summary one CI job posts for itself (no `--mode`).
+function renderJob(files) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-agg-'));
+  for (const [name, contents] of Object.entries(files)) {
+    fs.writeFileSync(path.join(dir, name), contents);
+  }
+  try {
+    return execFileSync(
+      process.execPath,
+      [SCRIPT, dir, '--job-name', 'E2E Python Conformance'],
+      { encoding: 'utf8' }
+    );
+  } catch (error) {
+    return error.stdout;
+  }
+}
+
 test('few failures list inline above a collapsed summary section', () => {
   const body = renderAggregate({
     'e2e-vercel-prod-nextjs-turbopack.json': resultJson('a', 40, [
@@ -129,3 +146,25 @@ test('Details by Category has no nested collapsibles', () => {
   assert.match(block, /\*\*❌ ▲ Vercel Production\*\*/);
   assert.match(block, /\*\*✅ 💻 Local Development\*\*/);
 });
+
+test('conformance declarations are not mistaken for result files', () => {
+  // Per-job mode, because that is where a stray file shows up: it lists one row
+  // per report file it found.
+  const body = renderJob({
+    'e2e-conformance-python.json': resultJson('a', 2),
+    // Both of these live in the repo, match `e2e-*.json`, and are conformance
+    // declarations rather than vitest reports.
+    'e2e-conformance.json': JSON.stringify({
+      language: 'python',
+      fixtures: ['nullByteWorkflow'],
+    }),
+    'e2e-conformance.example.json': JSON.stringify({ fixtures: [] }),
+  });
+
+  // The one real report is the only one found, so there is no by-file table at
+  // all. With the strays counted there was, listing them as 0/0/0 rows.
+  assert.doesNotMatch(body, /Results by File/);
+  assert.match(body, /\| \*\*Total\*\* \| \*\*2\*\* \|/);
+  assert.doesNotMatch(body, /e2e-conformance\.json/);
+  assert.doesNotMatch(body, /e2e-conformance\.example\.json/);
+});

.github/workflows/tests.yml

@@ -377,6 +377,22 @@ jobs:
           - name: "tanstack-start"
             project-id: "prj_643jeVugTMq5ivsOFQHcbLG1qcnu"
             project-slug: "workbench-tanstack-start-workflow"
+          - name: "python"
+            project-id: "prj_MtdPMACTDJnukz8LzWM65Fob0IVU"
+            project-slug: "workbench-python-workflow"
+        exclude:
+          # `vm` selects the JS engine the workflow body runs on. There is no
+          # Python meaning to that axis — the Python app has one runtime — so
+          # only the `node` half of the pair is a real configuration.
+          #
+          # GitHub Actions matches an `exclude` entry by deep equality on every
+          # key it names, so the whole `app` object has to be repeated verbatim.
+          # Keep it in sync with the entry above.
+          - vm: quickjs
+            app:
+              name: "python"
+              project-id: "prj_MtdPMACTDJnukz8LzWM65Fob0IVU"
+              project-slug: "workbench-python-workflow"
     env:
       TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
       TURBO_TEAM: ${{ vars.TURBO_TEAM }}
@@ -1094,6 +1110,116 @@ jobs:
           retention-days: 7
           if-no-files-found: ignore
 
+  # Cross-language conformance: the same TypeScript e2e suite, driven against the
+  # Python workbench app instead of a JavaScript one. Deliberately its own job
+  # rather than a `getTestMatrix` row — every step in the two local lanes is JS
+  # toolchain (`pnpm dev` / `pnpm start`, the canary `next` patch, `dev.test.ts`
+  # and `local-build.test.ts`, and the per-framework `DEV_TEST_CONFIG` of
+  # compiler-generated paths), and none of it has a Python counterpart. The
+  # `e2e-vercel-prod` lane, whose rows only need a URL, does carry `python` as an
+  # ordinary matrix row.
+  #
+  # What this lane protects is the gate, not the fixtures it happens to run: a
+  # test that stops skipping correctly, or a fixture declared in
+  # `e2e-conformance.json` that the app stops registering, fails here and nowhere
+  # else.
+  e2e-python:
+    name: E2E Python Conformance
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    if: ${{ needs.ci-scope.outputs.fast-path != 'true' && !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }}
+    needs: [ci-scope, e2e-package-build]
+
+    env:
+      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+      TURBO_TEAM: ${{ vars.TURBO_TEAM }}
+      WORKFLOW_PUBLIC_MANIFEST: '1'
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Setup environment
+        uses: ./.github/actions/setup-workflow-dev
+        with:
+          build-packages: 'false'
+
+      # The Python app needs nothing from `packages/`, but the driver does: the
+      # vitest suite imports `@workflow/core` and shells out to the `workflow`
+      # CLI to inspect runs.
+      - name: Download shared package builds
+        uses: actions/download-artifact@v4
+        with:
+          name: e2e-package-build-artifacts
+          path: packages
+
+      # Pinned rather than floating to latest: `uv sync --locked` below is only
+      # a meaningful gate if the tool reading the lock is a known quantity.
+      # Pinning here rather than via `[tool.uv] required-version` is deliberate
+      # — that key would also bind the uv that `@vercel/python` bundles on the
+      # build image, which we do not control.
+      - name: Setup uv
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+        with:
+          version: '0.12.2'
+          enable-cache: true
+          cache-dependency-glob: workbench/python/uv.lock
+
+      # `--locked` rather than `--frozen`: a `pyproject.toml` edit that was never
+      # relocked should fail the job, not be silently ignored.
+      - name: Install Python dependencies
+        working-directory: workbench/python
+        run: uv sync --locked
+
+      - name: Run E2E Tests
+        run: |
+          APP_LOG="$GITHUB_WORKSPACE/python-app.log"
+          rm -f "$APP_LOG"
+          (cd workbench/python && uv run uvicorn app:app --port 3000 > "$APP_LOG" 2>&1) &
+
+          for _ in $(seq 1 30); do
+            if curl -sf http://localhost:3000/.well-known/workflow/v1/manifest.json -o /dev/null; then
+              echo "app is up"
+              break
+            fi
+            sleep 1
+          done
+          curl -sf http://localhost:3000/.well-known/workflow/v1/manifest.json | head -c 2000
+          echo
+
+          # Only `e2e.test.ts` — `e2e-agent.test.ts` (which `pnpm test:e2e` also
+          # runs) has no conformance gate and needs routes this app does not serve.
+          pnpm vitest run packages/core/e2e/e2e.test.ts \
+            --reporter=verbose --reporter=json \
+            --reporter=./packages/core/e2e/github-reporter.ts \
+            --outputFile=e2e-conformance-python.json
+        env:
+          NODE_OPTIONS: "--enable-source-maps"
+          APP_NAME: "python"
+          WORKBENCH_APP_PATH: ${{ github.workspace }}/workbench/python
+          DEPLOYMENT_URL: "http://localhost:3000"
+
+      - name: Print Python app logs
+        if: always()
+        run: cat "$GITHUB_WORKSPACE/python-app.log" || true
+
+      - name: Generate E2E summary
+        if: always()
+        run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Python Conformance" >> $GITHUB_STEP_SUMMARY || true
+
+      - name: Upload E2E results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-results-conformance-python
+          path: |
+            e2e-conformance-python.json
+            e2e-diagnostics-python-local.json
+          retention-days: 7
+          if-no-files-found: ignore
+
   e2e-windows:
     name: E2E Windows Tests (${{ matrix.vm }})
     runs-on: windows-latest
@@ -1308,7 +1434,7 @@ jobs:
   summary:
     name: E2E Summary
     runs-on: ubuntu-latest
-    needs: [ci-scope, e2e-vercel-prod, e2e-vercel-ws-transport, e2e-local-dev, e2e-local-prod, e2e-local-postgres, e2e-windows]
+    needs: [ci-scope, e2e-vercel-prod, e2e-vercel-ws-transport, e2e-local-dev, e2e-local-prod, e2e-local-postgres, e2e-python, e2e-windows]
     if: always() && !cancelled() && needs.ci-scope.outputs.fast-path != 'true'
     timeout-minutes: 10
 
@@ -1344,7 +1470,7 @@ jobs:
   e2e-required-check:
     name: E2E Required Check
     runs-on: ubuntu-latest
-    needs: [ci-scope, unit, e2e-package-build, e2e-vercel-prod, e2e-vercel-ws-transport, e2e-local-dev, e2e-local-prod, e2e-local-postgres, e2e-windows]
+    needs: [ci-scope, unit, e2e-package-build, e2e-vercel-prod, e2e-vercel-ws-transport, e2e-local-dev, e2e-local-prod, e2e-local-postgres, e2e-python, e2e-windows]
     if: always()
     timeout-minutes: 5
 
@@ -1358,6 +1484,7 @@ jobs:
           LOCAL_DEV_STATUS: ${{ needs.e2e-local-dev.result }}
           LOCAL_PROD_STATUS: ${{ needs.e2e-local-prod.result }}
           POSTGRES_STATUS: ${{ needs.e2e-local-postgres.result }}
+          PYTHON_STATUS: ${{ needs.e2e-python.result }}
           WINDOWS_STATUS: ${{ needs.e2e-windows.result }}
           FAST_PATH: ${{ needs.ci-scope.outputs.fast-path }}
           VALIDATION_FAST_PATH: ${{ needs.ci-scope.outputs.validation-fast-path }}
@@ -1394,6 +1521,7 @@ jobs:
             [[ "$LOCAL_DEV_STATUS" == "skipped" ]] || echo "Warning: e2e-local-dev was not skipped ($LOCAL_DEV_STATUS)"
             [[ "$LOCAL_PROD_STATUS" == "skipped" ]] || echo "Warning: e2e-local-prod was not skipped ($LOCAL_PROD_STATUS)"
             [[ "$POSTGRES_STATUS" == "skipped" ]] || echo "Warning: e2e-local-postgres was not skipped ($POSTGRES_STATUS)"
+            [[ "$PYTHON_STATUS" == "skipped" ]] || echo "Warning: e2e-python was not skipped ($PYTHON_STATUS)"
             [[ "$WINDOWS_STATUS" == "skipped" ]] || echo "Warning: e2e-windows was not skipped ($WINDOWS_STATUS)"
           else
             echo "Standard PR - checking all jobs"
@@ -1408,6 +1536,7 @@ jobs:
             [[ "$LOCAL_DEV_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-local-dev ($LOCAL_DEV_STATUS)")
             [[ "$LOCAL_PROD_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-local-prod ($LOCAL_PROD_STATUS)")
             [[ "$POSTGRES_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-local-postgres ($POSTGRES_STATUS)")
+            [[ "$PYTHON_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-python ($PYTHON_STATUS)")
             [[ "$WINDOWS_STATUS" == "success" ]] || FAILED_JOBS+=("e2e-windows ($WINDOWS_STATUS)")
           fi
 

package.json

@@ -7,6 +7,7 @@
     "@changesets/changelog-github": "0.5.2",
     "@changesets/cli": "^2.29.8",
     "@vitest/coverage-v8": "catalog:",
+    "@vitest/runner": "catalog:",
     "esbuild": "catalog:",
     "husky": "^9.1.7",
     "lint-staged": "^16.2.7",

packages/core/e2e/e2e-conformance.example.json

@@ -0,0 +1,29 @@
+{
+  "$comment": [
+    "Reference for `e2e-conformance.json`. Copy this into a workbench app's",
+    "directory to run the e2e suite against an app that is not written in",
+    "JavaScript. `ConformanceConfig` in ./utils.ts documents each field and how",
+    "it ratchets; the short version is that every key below is a claim the suite",
+    "will fail on if it stops being true.",
+    "",
+    "\"language\": anything other than \"javascript\" skips the tests marked",
+    "`testJsOnly` / `describeJsOnly` in e2e.test.ts.",
+    "",
+    "\"fixtures\": the workflows/99_e2e fixtures this app implements.",
+    "",
+    "\"unsupported\" (optional): tests to skip by exact name, mapped to the reason,",
+    "for behaviour the app's runtime does not implement even though the fixture",
+    "is ported."
+  ],
+  "language": "python",
+  "fixtures": [
+    "addTenWorkflow",
+    "sleepingWorkflow",
+    "promiseAllWorkflow",
+    "nullByteWorkflow",
+    "hookWorkflow"
+  ],
+  "unsupported": {
+    "the exact name of a test in e2e.test.ts": "why this app's runtime cannot pass it yet"
+  }
+}