From d034d1515c9687c20336f1540ecaf164ac081507 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:24:52 +0000 Subject: [PATCH] test: add opt-in e2e tier scaffolding for the SDK test suites --- .changeset/plenty-pugs-organize.md | 7 +++ .github/workflows/cli_tests.yml | 16 ++++++ .github/workflows/js_sdk_tests.yml | 39 +++++++++---- .github/workflows/python_sdk_tests.yml | 15 +++++ .github/workflows/sdk_e2e_tests.yml | 56 +++++++++++++++++++ .github/workflows/sdk_tests.yml | 6 ++ CONTRIBUTING.md | 20 +++++++ packages/cli/package.json | 1 + packages/cli/tests/README.md | 35 ++++++++++++ .../sandbox/backend_integration.test.ts | 53 ++++++------------ .../tests/commands/sandbox/exec_pipe.test.ts | 40 +++---------- packages/cli/tests/setup.ts | 45 +++++++++++++++ packages/cli/vitest.e2e.config.mts | 17 ++++++ packages/js-sdk/package.json | 4 +- packages/js-sdk/tests/README.md | 55 ++++++++++++++++++ packages/js-sdk/tests/e2eFiles.mts | 22 ++++++++ .../runtimes/cloudflare/vitest.config.mts | 6 +- packages/js-sdk/tests/setup.ts | 41 +++++++++++++- packages/js-sdk/vitest.config.mts | 31 ++++++++++ packages/python-sdk/pytest.ini | 6 +- packages/python-sdk/tests/README.md | 50 +++++++++++++++++ packages/python-sdk/tests/conftest.py | 30 ++++++++++ packages/python-sdk/tests/envd_versions.py | 13 +++++ 23 files changed, 524 insertions(+), 84 deletions(-) create mode 100644 .changeset/plenty-pugs-organize.md create mode 100644 .github/workflows/sdk_e2e_tests.yml create mode 100644 packages/cli/tests/README.md create mode 100644 packages/cli/vitest.e2e.config.mts create mode 100644 packages/js-sdk/tests/README.md create mode 100644 packages/js-sdk/tests/e2eFiles.mts create mode 100644 packages/python-sdk/tests/README.md create mode 100644 packages/python-sdk/tests/envd_versions.py diff --git a/.changeset/plenty-pugs-organize.md b/.changeset/plenty-pugs-organize.md new file mode 100644 index 0000000000..33864d450d --- /dev/null +++ b/.changeset/plenty-pugs-organize.md @@ -0,0 +1,7 @@ +--- +'@e2b/python-sdk': patch +'@e2b/cli': patch +'e2b': patch +--- + +Split the test suites into a fully mocked default tier and an opt-in `E2B_E2E=1` end-to-end tier (tests only, no runtime changes) diff --git a/.github/workflows/cli_tests.yml b/.github/workflows/cli_tests.yml index a5c8bb5fc1..13b49e9896 100644 --- a/.github/workflows/cli_tests.yml +++ b/.github/workflows/cli_tests.yml @@ -7,6 +7,11 @@ on: required: false type: string default: '' + e2e: + description: 'Run the e2e tier (drives the CLI against real sandboxes) instead of the mocked unit tier' + required: false + type: boolean + default: false secrets: E2B_API_KEY: required: true @@ -64,9 +69,20 @@ jobs: run: pnpm build working-directory: ./packages/cli + # The default tier: fully mocked, no sandboxes, no credentials needed. - name: Run tests + if: ${{ !inputs.e2e }} run: pnpm test working-directory: ./packages/cli env: E2B_API_KEY: ${{ secrets.E2B_API_KEY }} E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }} + + # The opt-in tier: drives the built CLI against real sandboxes. + - name: Run e2e tests + if: ${{ inputs.e2e }} + run: pnpm test:e2e + working-directory: ./packages/cli + env: + E2B_API_KEY: ${{ secrets.E2B_API_KEY }} + E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }} diff --git a/.github/workflows/js_sdk_tests.yml b/.github/workflows/js_sdk_tests.yml index 3a08f0dcb8..c7c9937c08 100644 --- a/.github/workflows/js_sdk_tests.yml +++ b/.github/workflows/js_sdk_tests.yml @@ -12,6 +12,11 @@ on: required: false type: boolean default: false + e2e: + description: 'Run the e2e tier (provisions sandboxes and builds templates) instead of the mocked unit tier' + required: false + type: boolean + default: false secrets: E2B_API_KEY: required: true @@ -83,21 +88,22 @@ jobs: pnpm install --frozen-lockfile # Only the Node runtime runs the vitest `browser` project, which drives - # Chromium through Playwright. + # Chromium through Playwright — and that project is e2e (it provisions a + # sandbox from a browser bundle). - name: Get Playwright version - if: matrix.runtime == 'node' + if: matrix.runtime == 'node' && inputs.e2e id: playwright-version run: echo "version=$(node -p "require('playwright/package.json').version")" >> "$GITHUB_OUTPUT" - name: Cache Playwright browsers - if: matrix.runtime == 'node' + if: matrix.runtime == 'node' && inputs.e2e uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ${{ matrix.os == 'windows-latest' && '~/AppData/Local/ms-playwright' || '~/.cache/ms-playwright' }} key: playwright-${{ runner.os }}-${{ steps.playwright-version.outputs.version }} - name: Install Playwright Chromium - if: matrix.runtime == 'node' + if: matrix.runtime == 'node' && inputs.e2e run: pnpm run playwright:install # The unit bundle test and the Cloudflare deploy config fail in CI when @@ -105,32 +111,43 @@ jobs: - name: Test build run: pnpm build + # The default tier: fully mocked, no sandboxes, no credentials needed. - name: Run Node tests - if: matrix.runtime == 'node' + if: matrix.runtime == 'node' && !inputs.e2e run: pnpm test env: E2B_API_KEY: ${{ secrets.E2B_API_KEY }} E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }} + # The opt-in tier: real sandboxes, envd round-trips and template builds. + - name: Run Node e2e tests + if: matrix.runtime == 'node' && inputs.e2e + run: | + pnpm test:e2e + pnpm test:browser + env: + E2B_API_KEY: ${{ secrets.E2B_API_KEY }} + E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }} + - name: Install Bun - if: matrix.runtime == 'bun' + if: matrix.runtime == 'bun' && !inputs.e2e uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 - name: Run test suite under Bun - if: matrix.runtime == 'bun' + if: matrix.runtime == 'bun' && !inputs.e2e run: pnpm test:bun env: E2B_API_KEY: ${{ secrets.E2B_API_KEY }} E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }} - name: Install Deno - if: matrix.runtime == 'deno' + if: matrix.runtime == 'deno' && !inputs.e2e uses: denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed # v2.0.5 with: deno-version: v${{ env.TOOL_VERSION_DENO }} - name: Run test suite under Deno - if: matrix.runtime == 'deno' + if: matrix.runtime == 'deno' && !inputs.e2e run: pnpm test:deno env: E2B_API_KEY: ${{ secrets.E2B_API_KEY }} @@ -138,7 +155,7 @@ jobs: # Full unit + connectionConfig suite inside workerd (vitest-pool-workers). - name: Run test suite under Cloudflare workerd - if: matrix.runtime == 'cloudflare' + if: matrix.runtime == 'cloudflare' && !inputs.e2e run: pnpm test:cf env: E2B_API_KEY: ${{ secrets.E2B_API_KEY }} @@ -148,7 +165,7 @@ jobs: # global setup (wrangler deploy --temporary, no Cloudflare credentials # needed) and deletes the worker in teardown. - name: Run Cloudflare Workers deploy tests - if: matrix.runtime == 'cloudflare-deploy' + if: matrix.runtime == 'cloudflare-deploy' && !inputs.e2e run: pnpm test:cf:deploy env: E2B_API_KEY: ${{ secrets.E2B_API_KEY }} diff --git a/.github/workflows/python_sdk_tests.yml b/.github/workflows/python_sdk_tests.yml index 741d24246b..7491c111c1 100644 --- a/.github/workflows/python_sdk_tests.yml +++ b/.github/workflows/python_sdk_tests.yml @@ -7,6 +7,11 @@ on: required: false type: string default: '' + e2e: + description: 'Run the e2e tier (provisions sandboxes and builds templates) instead of the mocked unit tier' + required: false + type: boolean + default: false secrets: E2B_API_KEY: required: true @@ -50,8 +55,18 @@ jobs: - name: Test build run: uv build + # The default tier: fully mocked, no sandboxes, no credentials needed. - name: Run tests + if: ${{ !inputs.e2e }} run: uv run pytest --verbose --numprocesses=4 env: E2B_API_KEY: ${{ secrets.E2B_API_KEY }} E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }} + + # The opt-in tier: real sandboxes, envd round-trips and template builds. + - name: Run e2e tests + if: ${{ inputs.e2e }} + run: uv run pytest -m e2e --verbose --numprocesses=4 + env: + E2B_API_KEY: ${{ secrets.E2B_API_KEY }} + E2B_DOMAIN: ${{ inputs.E2B_DOMAIN }} diff --git a/.github/workflows/sdk_e2e_tests.yml b/.github/workflows/sdk_e2e_tests.yml new file mode 100644 index 0000000000..8bc4ffa587 --- /dev/null +++ b/.github/workflows/sdk_e2e_tests.yml @@ -0,0 +1,56 @@ +name: SDK E2E Tests + +# The opt-in tier. These jobs provision sandboxes, talk to envd and build +# templates against live infrastructure, so they are not part of the required +# PR checks (see sdk_tests.yml, which runs the fully mocked default tier). +# +# Run them manually from the Actions tab, or by adding the `e2e` label to a PR. +on: + workflow_dispatch: + inputs: + staging: + description: 'Run against staging instead of production' + required: false + type: boolean + default: false + pull_request: + branches: + - main + types: [opened, synchronize, reopened, labeled] + +permissions: + contents: read + +jobs: + js-e2e: + name: E2E / JS SDK + if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'e2e') }} + uses: ./.github/workflows/js_sdk_tests.yml + with: + e2e: true + # The e2e tier only runs under Node — the Bun/Deno/Cloudflare legs cover + # the mocked tier in the default workflow. + node-only: true + E2B_DOMAIN: ${{ inputs.staging && vars.E2B_DOMAIN_STAGING || '' }} + secrets: + E2B_API_KEY: ${{ inputs.staging && secrets.E2B_API_KEY_STAGING || secrets.E2B_API_KEY }} + + python-e2e: + name: E2E / Python SDK + if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'e2e') }} + uses: ./.github/workflows/python_sdk_tests.yml + with: + e2e: true + E2B_DOMAIN: ${{ inputs.staging && vars.E2B_DOMAIN_STAGING || '' }} + secrets: + E2B_API_KEY: ${{ inputs.staging && secrets.E2B_API_KEY_STAGING || secrets.E2B_API_KEY }} + + cli-e2e: + name: E2E / CLI + if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'e2e') }} + uses: ./.github/workflows/cli_tests.yml + with: + e2e: true + E2B_DOMAIN: ${{ inputs.staging && vars.E2B_DOMAIN_STAGING || '' }} + secrets: + E2B_API_KEY: ${{ inputs.staging && secrets.E2B_API_KEY_STAGING || secrets.E2B_API_KEY }} diff --git a/.github/workflows/sdk_tests.yml b/.github/workflows/sdk_tests.yml index 8e488c25c9..5b77fcb12a 100644 --- a/.github/workflows/sdk_tests.yml +++ b/.github/workflows/sdk_tests.yml @@ -1,5 +1,8 @@ name: SDK Tests +# The default tier: fully mocked, deterministic, no sandboxes and no template +# builds. The behavioral tests that need live infrastructure live in the opt-in +# sdk_e2e_tests.yml workflow (`e2e` PR label or manual dispatch). on: pull_request: branches: @@ -76,6 +79,9 @@ jobs: secrets: E2B_API_KEY: ${{ secrets.E2B_API_KEY }} + # The staging legs re-run the same mocked tier against the staging domain, so + # they only catch environment-specific breakage now; backend compatibility is + # verified by the e2e workflow (`workflow_dispatch` with `staging: true`). js-tests-staging: name: Staging / JS SDK Tests needs: changes diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ff3e758ff9..223e32d21a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,2 +1,22 @@ # Contributing If you want to contribute, open a PR, issue, or start a discussion on our [Discord](https://discord.gg/dSBY3ms2Qr). + +## Tests + +Every package splits its tests into two tiers: + +- **unit (default)** — fully mocked, deterministic, no sandboxes and no + credentials. +- **e2e (opt-in)** — real sandboxes, envd round-trips and template builds; + requires `E2B_E2E=1` and an API key. + +| Package | Unit | E2E | +| --- | --- | --- | +| `packages/js-sdk` | `pnpm test` | `pnpm test:e2e`, `pnpm test:browser` | +| `packages/python-sdk` | `uv run pytest` | `uv run pytest -m e2e` | +| `packages/cli` | `pnpm test` | `pnpm test:e2e` | + +Details, and where a new test belongs, are in each package's +`tests/README.md`. On CI the `SDK Tests` workflow runs the unit tier for every +PR; the e2e tier runs in the opt-in `SDK E2E Tests` workflow (add the `e2e` +label to a PR or dispatch it manually). diff --git a/packages/cli/package.json b/packages/cli/package.json index b4f0c7c4ff..4ac9a7aebe 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -43,6 +43,7 @@ "format": "prettier --write src", "test:interactive": "pnpm build && ./dist/index.js", "test": "vitest run", + "test:e2e": "vitest run --config vitest.e2e.config.mts", "test:watch": "vitest watch", "test:coverage": "vitest run --coverage", "check-deps": "knip" diff --git a/packages/cli/tests/README.md b/packages/cli/tests/README.md new file mode 100644 index 0000000000..27b6001988 --- /dev/null +++ b/packages/cli/tests/README.md @@ -0,0 +1,35 @@ +# CLI tests + +The suite has two tiers. + +## Unit tier (default) + +```bash +pnpm test +``` + +Fully mocked (`vi.mock` over `e2b` and the CLI's API modules) or driving the +built CLI against local input only — deterministic, no sandboxes, no +credentials. It asserts on argument parsing, validation, output formatting and +the calls the CLI makes into the SDK. + +## E2E tier (opt-in) + +```bash +E2B_API_KEY=e2b_... pnpm test:e2e +``` + +Tests that drive the built CLI against a real sandbox (`exec` piping, +`backend_integration`). They need `E2B_E2E=1` (set by the script above) plus +credentials, from `E2B_API_KEY` or `~/.e2b/config.json`; without both they are +skipped. + +Use `e2eTest` (or `skipE2E` in a `beforeAll`) from [`setup.ts`](./setup.ts), +which also resolves the shared `e2eApiKey`/`e2eDomain`. `E2B_DEBUG` is a +separate axis and disables the e2e tier because it points the CLI at a local +stack. + +## CI + +`SDK Tests` runs the unit tier on every PR. The e2e tier runs in the opt-in +`SDK E2E Tests` workflow — add the `e2e` label to a PR or dispatch it manually. diff --git a/packages/cli/tests/commands/sandbox/backend_integration.test.ts b/packages/cli/tests/commands/sandbox/backend_integration.test.ts index 4af72a4312..2c268c68a8 100644 --- a/packages/cli/tests/commands/sandbox/backend_integration.test.ts +++ b/packages/cli/tests/commands/sandbox/backend_integration.test.ts @@ -1,28 +1,16 @@ -import { afterAll, beforeAll, describe, expect, test } from 'vitest' +import { afterAll, beforeAll, describe, expect } from 'vitest' import { Sandbox } from 'e2b' -import { getUserConfig } from 'src/user' import { bufferToText, - isDebug, + e2eApiKey, + e2eDomain, + e2eTest, parseEnvInt, runCli, runCliWithPipedStdin, + skipE2E, } from '../../setup' -type UserConfigWithDomain = NonNullable> & { - domain?: string - E2B_DOMAIN?: string -} - -const userConfig = safeGetUserConfig() as UserConfigWithDomain | null -const domain = - process.env.E2B_DOMAIN || - userConfig?.E2B_DOMAIN || - userConfig?.domain || - 'e2b.app' -const apiKey = process.env.E2B_API_KEY || userConfig?.projectApiKey -const shouldSkip = !apiKey || isDebug -const integrationTest = test.skipIf(shouldSkip) const templateId = process.env.E2B_CLI_BACKEND_TEMPLATE_ID || process.env.E2B_TEMPLATE_ID || @@ -35,8 +23,8 @@ const perTestTimeoutMs = parseEnvInt('E2B_CLI_BACKEND_TEST_TIMEOUT_MS', 30_000) const spawnTimeoutMs = perTestTimeoutMs const cliEnv: NodeJS.ProcessEnv = { ...process.env, - E2B_DOMAIN: domain, - E2B_API_KEY: apiKey, + E2B_DOMAIN: e2eDomain, + E2B_API_KEY: e2eApiKey, } delete cliEnv.E2B_DEBUG @@ -50,11 +38,11 @@ describe('sandbox cli backend integration', () => { let sandbox: Sandbox beforeAll(async () => { - if (shouldSkip) return + if (skipE2E) return sandbox = await Sandbox.create(templateId, { - apiKey, - domain, + apiKey: e2eApiKey, + domain: e2eDomain, timeoutMs: sandboxTimeoutMs, }) }, 30_000) @@ -71,7 +59,7 @@ describe('sandbox cli backend integration', () => { } }, 15_000) - integrationTest( + e2eTest( 'list shows the sandbox', { timeout: perTestTimeoutMs }, async () => { @@ -81,7 +69,7 @@ describe('sandbox cli backend integration', () => { } ) - integrationTest( + e2eTest( 'info shows the sandbox details', { timeout: perTestTimeoutMs }, async () => { @@ -104,7 +92,7 @@ describe('sandbox cli backend integration', () => { } ) - integrationTest( + e2eTest( 'exec runs a command without piped stdin', { timeout: perTestTimeoutMs }, async () => { @@ -122,7 +110,7 @@ describe('sandbox cli backend integration', () => { } ) - integrationTest( + e2eTest( 'exec runs a command with piped stdin', { timeout: perTestTimeoutMs }, async () => { @@ -138,7 +126,7 @@ describe('sandbox cli backend integration', () => { } ) - integrationTest( + e2eTest( 'metrics returns successfully', { timeout: perTestTimeoutMs }, async () => { @@ -153,7 +141,7 @@ describe('sandbox cli backend integration', () => { } ) - integrationTest( + e2eTest( 'kill removes the sandbox', { timeout: perTestTimeoutMs }, async () => { @@ -196,12 +184,3 @@ function sandboxExistsInList( const parsed = JSON.parse(text) as Array<{ sandboxId?: string }> return parsed.some((item) => item.sandboxId === sandboxId) } - -function safeGetUserConfig(): ReturnType | null { - try { - return getUserConfig() - } catch (err) { - console.warn(`Failed to read ~/.e2b/config.json: ${String(err)}`) - return null - } -} diff --git a/packages/cli/tests/commands/sandbox/exec_pipe.test.ts b/packages/cli/tests/commands/sandbox/exec_pipe.test.ts index e57e3ce9e6..2ffccfa4ee 100644 --- a/packages/cli/tests/commands/sandbox/exec_pipe.test.ts +++ b/packages/cli/tests/commands/sandbox/exec_pipe.test.ts @@ -1,11 +1,12 @@ import { randomBytes } from 'node:crypto' -import { describe, expect, test } from 'vitest' +import { describe, expect } from 'vitest' import { Sandbox } from 'e2b' -import { getUserConfig } from 'src/user' import { type CliRunResult, bufferToText, - isDebug, + e2eApiKey, + e2eDomain, + e2eTest, parseEnvInt, runCliWithPipedStdin, } from '../../setup' @@ -17,20 +18,6 @@ type PipeCase = { timeoutMs?: number } -type UserConfigWithDomain = NonNullable> & { - domain?: string - E2B_DOMAIN?: string -} - -const userConfig = safeGetUserConfig() as UserConfigWithDomain | null -const domain = - process.env.E2B_DOMAIN || - userConfig?.E2B_DOMAIN || - userConfig?.domain || - 'e2b.app' -const apiKey = process.env.E2B_API_KEY || userConfig?.projectApiKey -const shouldSkip = !apiKey || isDebug -const integrationTest = test.skipIf(shouldSkip) const templateId = process.env.E2B_PIPE_TEMPLATE_ID || process.env.E2B_TEMPLATE_ID || @@ -47,8 +34,8 @@ const defaultCmdTimeoutMs = parseEnvInt( ) const cliEnv: NodeJS.ProcessEnv = { ...process.env, - E2B_DOMAIN: domain, - E2B_API_KEY: apiKey, + E2B_DOMAIN: e2eDomain, + E2B_API_KEY: e2eApiKey, } delete cliEnv.E2B_DEBUG @@ -100,13 +87,13 @@ const largeBinaryCases: PipeCase[] = [ ] describe('sandbox exec stdin piping (integration)', () => { - integrationTest( + e2eTest( 'pipes stdin to remote command', { timeout: testTimeoutMs }, async () => { const sandbox = await Sandbox.create(templateId, { - apiKey, - domain, + apiKey: e2eApiKey, + domain: e2eDomain, timeoutMs: sandboxTimeoutMs, }) @@ -177,12 +164,3 @@ function assertExecSucceeded( throw new Error(`${name} failed with rc=${result.status} stderr=${stderr}`) } } - -function safeGetUserConfig(): ReturnType | null { - try { - return getUserConfig() - } catch (err) { - console.warn(`Failed to read ~/.e2b/config.json: ${String(err)}`) - return null - } -} diff --git a/packages/cli/tests/setup.ts b/packages/cli/tests/setup.ts index 32c463340d..ed6455e6a7 100644 --- a/packages/cli/tests/setup.ts +++ b/packages/cli/tests/setup.ts @@ -1,8 +1,53 @@ import { execSync, spawn, spawnSync } from 'node:child_process' import path from 'node:path' +import { test } from 'vitest' + +import { getUserConfig } from 'src/user' + export const isDebug = process.env.E2B_DEBUG !== undefined +/** + * Opt-in flag for the e2e tier: tests that drive the CLI against live + * infrastructure. The default `pnpm test` run stays fully mocked. + */ +export const isE2E = process.env.E2B_E2E !== undefined + +type UserConfigWithDomain = NonNullable> & { + domain?: string + E2B_DOMAIN?: string +} + +function safeGetUserConfig(): UserConfigWithDomain | null { + try { + return getUserConfig() as UserConfigWithDomain | null + } catch (err) { + console.warn(`Failed to read ~/.e2b/config.json: ${String(err)}`) + return null + } +} + +const userConfig = safeGetUserConfig() + +const DEFAULT_E2E_DOMAIN = 'e2b.dev' + +export const e2eDomain = + process.env.E2B_DOMAIN || + userConfig?.E2B_DOMAIN || + userConfig?.domain || + DEFAULT_E2E_DOMAIN + +export const e2eApiKey = process.env.E2B_API_KEY || userConfig?.projectApiKey + +/** + * True when the e2e tier can't run: it needs the explicit opt-in and + * credentials, and debug mode points the CLI at a local stack instead. + */ +export const skipE2E = !isE2E || !e2eApiKey || isDebug + +/** `test` for the e2e tier — skipped unless `E2B_E2E=1` and credentials are set. */ +export const e2eTest = test.skipIf(skipE2E) + type CliRunOptions = { timeoutMs: number env?: NodeJS.ProcessEnv diff --git a/packages/cli/vitest.e2e.config.mts b/packages/cli/vitest.e2e.config.mts new file mode 100644 index 0000000000..fd01f141cc --- /dev/null +++ b/packages/cli/vitest.e2e.config.mts @@ -0,0 +1,17 @@ +import { defineConfig, mergeConfig } from 'vitest/config' + +import base from './vitest.config' + +// Opt-in tier: `pnpm test:e2e`. The flag the tests gate on is set here rather +// than in the package script, which would need POSIX-only `VAR=value` syntax +// and break on Windows. +export default mergeConfig( + base, + defineConfig({ + test: { + env: { + E2B_E2E: '1', + }, + }, + }) +) diff --git a/packages/js-sdk/package.json b/packages/js-sdk/package.json index 9abad4bf7e..3ea0166d65 100644 --- a/packages/js-sdk/package.json +++ b/packages/js-sdk/package.json @@ -27,7 +27,9 @@ "build": "tsc --noEmit && tsdown", "dev": "tsdown --watch", "example": "tsx example.mts", - "test": "vitest run", + "test": "vitest run --project unit --project template --project connectionConfig", + "test:e2e": "vitest run --project e2e", + "test:browser": "vitest run --project browser", "generate": "npm-run-all generate:* && pnpm run format", "generate:api": "redocly bundle js-sdk --config ../../redocly.yaml -o ../../spec/openapi_generated.js-sdk.yml && openapi-typescript ../../spec/openapi_generated.js-sdk.yml -x api_key --array-length --alphabetize --default-non-nullable false --output src/api/schema.gen.ts", "generate:envd": "cd ../../spec/envd && buf generate --template buf-js.gen.yaml\n", diff --git a/packages/js-sdk/tests/README.md b/packages/js-sdk/tests/README.md new file mode 100644 index 0000000000..45401758a1 --- /dev/null +++ b/packages/js-sdk/tests/README.md @@ -0,0 +1,55 @@ +# JS SDK tests + +The suite has two tiers. + +## Unit tier (default) + +```bash +pnpm test +``` + +Fully mocked (msw over the API and envd endpoints), deterministic, no sandboxes, +no credentials, seconds to run. It asserts on client-side logic: request payload +shaping, config propagation, version gating, response parsing and format +switching, RPC/API error mapping, pagination, URL construction and pure +utilities. + +Vitest projects: `unit`, `template`, `connectionConfig`. The other runtimes run +the same tier: `pnpm test:bun`, `pnpm test:deno`, `pnpm test:cf`. + +## E2E tier (opt-in) + +```bash +E2B_API_KEY=e2b_... pnpm test:e2e +E2B_API_KEY=e2b_... pnpm test:browser +``` + +Everything whose assertions depend on real behavior across the RPC boundary — +process execution, filesystem round-trips, PTY semantics, git inside the VM, +sandbox lifecycle against live infrastructure and server-side template builds. +It provisions sandboxes and builds templates, so it needs `E2B_E2E=1` (set by +the scripts above) and an API key. Without the opt-in these tests are skipped. + +The file list lives in [`e2eFiles.mts`](./e2eFiles.mts) and drives both the +`e2e` project and the exclusions of the default projects, so a new behavioral +test only needs to be added there. Use `e2eTest`, `e2eBuildTemplateTest` or the +`sandboxTest` fixture from [`setup.ts`](./setup.ts) — all three skip unless +`E2B_E2E` is set — and their `hostedTest`/`hostedSandboxTest` variants when a +local envd can't stand in for the real thing (control plane, traffic proxy, +snapshots), which additionally skip under `E2B_DEBUG`. + +`E2B_DEBUG` is a separate axis: it points the SDK at a local envd instead of a +provisioned sandbox and does not enable or disable either tier. + +## Where a module's tests land + +Volume and Secret are entirely request-shaping, error mapping and pagination, so +they sit in the unit tier over an in-memory mock of their APIs. The one +exception is real mount content: `volume/mount.test.ts` writes through a mounted +volume in one sandbox and reads it back in another, which only a live mount can +exercise. + +## CI + +`SDK Tests` runs the unit tier on every PR. The e2e tier runs in the opt-in +`SDK E2E Tests` workflow — add the `e2e` label to a PR or dispatch it manually. diff --git a/packages/js-sdk/tests/e2eFiles.mts b/packages/js-sdk/tests/e2eFiles.mts new file mode 100644 index 0000000000..a7cd4fbd6e --- /dev/null +++ b/packages/js-sdk/tests/e2eFiles.mts @@ -0,0 +1,22 @@ +/** + * The e2e tier: files whose assertions depend on real behavior across the RPC + * boundary in envd or the control plane — process execution, filesystem + * round-trips, PTY semantics, git inside the VM, sandbox lifecycle against live + * infrastructure and server-side template builds. They provision sandboxes, so + * they only run with `E2B_E2E=1` and credentials (`pnpm test:e2e`). + * + * Everything else is fully mocked and runs by default. The directories below + * hold behavioral tests only — the client-side logic that used to live next to + * them (commandHandle, entryInfo, watchHandle, gitValidation) sits one level up. + */ +export const e2eFiles = [ + 'tests/api/{info,kill,list,snapshot}.test.ts', + 'tests/sandbox/commands/**/*.test.ts', + 'tests/sandbox/files/**/*.test.ts', + 'tests/sandbox/git/**/*.test.ts', + 'tests/sandbox/pty/**/*.test.ts', + 'tests/sandbox/{connect,create,fork,host,internetAccess,kill,lifecycleBehavior,metrics,network,secure,snapshot,snapshot-api,timeout}.test.ts', + 'tests/template/{backgroundBuild,build,exists,tagsBuild}.test.ts', + 'tests/volume/mount.test.ts', + 'tests/template/methods/{makeSymlink,runCmd}.test.ts', +] diff --git a/packages/js-sdk/tests/runtimes/cloudflare/vitest.config.mts b/packages/js-sdk/tests/runtimes/cloudflare/vitest.config.mts index 85cae1af53..d633c0f9c8 100644 --- a/packages/js-sdk/tests/runtimes/cloudflare/vitest.config.mts +++ b/packages/js-sdk/tests/runtimes/cloudflare/vitest.config.mts @@ -2,6 +2,8 @@ import { cloudflareTest } from '@cloudflare/vitest-pool-workers' import { config } from 'dotenv' import { defineConfig } from 'vitest/config' +import { e2eFiles } from '../../e2eFiles.mjs' + const env = config() // Error names thrown by src/errors.ts (plus CommandExitError) — the shapes @@ -56,6 +58,8 @@ export default defineConfig({ // virtual filesystem can never see (and throws in CI when the file is // "missing"); the Node unit project keeps running it. 'tests/bundle/**', + // The e2e tier provisions sandboxes; workerd only runs the mocked tier. + ...e2eFiles, ], globals: false, testTimeout: 30_000, @@ -82,7 +86,7 @@ export default defineConfig({ // workerd's teardown error for in-flight streams when a test kills // the sandbox mid-request. message === 'Network connection lost.' || - // Stub rejection from tests/sandbox/git/validation.test.ts. + // Stub rejection from tests/sandbox/gitValidation.test.ts. message === 'commands.run should not be called') if (expectedRejection) return false }, diff --git a/packages/js-sdk/tests/setup.ts b/packages/js-sdk/tests/setup.ts index 91d427c3cc..b4a138fb46 100644 --- a/packages/js-sdk/tests/setup.ts +++ b/packages/js-sdk/tests/setup.ts @@ -70,8 +70,10 @@ async function buildTemplate( export const sandboxTest = base.extend({ template, sandboxTestId: [ - // eslint-disable-next-line no-empty-pattern - async ({}, use) => { + async ({ skip }, use) => { + // Every sandboxTest provisions a real sandbox, so the whole fixture is + // opt-in — see the e2e tier in tests/README.md. + skip(!isE2E, E2E_SKIP_REASON) const id = `test-${generateRandomString()}` await use(id) }, @@ -143,11 +145,46 @@ export const volumeTest = base.extend({ ], }) +/** Runs against a local envd instead of a provisioned sandbox. */ export const isDebug = process.env.E2B_DEBUG !== undefined +/** Opt-in for the e2e tier: tests that need real infrastructure. */ +export const isE2E = process.env.E2B_E2E !== undefined + +const E2E_SKIP_REASON = 'set E2B_E2E=1 to run the e2e tier' + +/** A test that needs real infrastructure — skipped unless E2B_E2E is set. */ +export const e2eTest = base.skipIf(!isE2E) + +/** + * A test that needs hosted infrastructure — the control plane, the traffic + * proxy, snapshots — which a local envd can't stand in for, so it stays + * skipped under E2B_DEBUG on top of the e2e opt-in. + */ +export const hostedTest = e2eTest.skipIf(isDebug) + +/** {@link sandboxTest} for a test that needs hosted infrastructure. */ +export const hostedSandboxTest = sandboxTest.skipIf(isDebug) + +/** + * A template build against real infrastructure — skipped unless E2B_E2E is + * set. Builds always run server-side, so E2B_DEBUG skips them too. + */ +export const e2eBuildTemplateTest = buildTemplateTest.skipIf(!isE2E || isDebug) + /** Placeholder API key with a valid format for tests that don't hit the API. */ export const TEST_API_KEY = `e2b_${'0'.repeat(40)}` +/** + * The highest envd version below one of the `ENVD_*` thresholds, for + * exercising the reject branch of a version gate without hardcoding a version + * that stops being below the threshold when it moves. A prerelease of a + * version sorts below the version itself. + */ +export function belowEnvdVersion(version: string): string { + return `${version}-0` +} + function generateRandomString(length: number = 8): string { return Math.random() .toString(36) diff --git a/packages/js-sdk/vitest.config.mts b/packages/js-sdk/vitest.config.mts index 63439b0112..b1c9781301 100644 --- a/packages/js-sdk/vitest.config.mts +++ b/packages/js-sdk/vitest.config.mts @@ -2,7 +2,10 @@ import { defineConfig } from 'vitest/config' import { playwright } from '@vitest/browser-playwright' import { config } from 'dotenv' +import { e2eFiles } from './tests/e2eFiles.mjs' + const env = config() + export default defineConfig({ test: { projects: [ @@ -14,6 +17,7 @@ export default defineConfig({ 'tests/runtimes/**', 'tests/template/**', 'tests/connectionConfig.test.ts', + ...e2eFiles, ], // Isolation is required: several suites patch global fetch via msw // and rely on module mocks (vi.doMock / vi.resetModules). Under @@ -38,6 +42,8 @@ export default defineConfig({ }, { test: { + // Provisions a real sandbox from a browser bundle, so it belongs to + // the e2e tier: run with `pnpm test:browser`. name: 'browser', include: ['tests/runtimes/browser/**/*.{test,spec}.tsx'], browser: { @@ -57,10 +63,35 @@ export default defineConfig({ test: { name: 'template', include: ['tests/template/**/*.test.ts'], + exclude: e2eFiles, + globals: false, + testTimeout: 180_000, + environment: 'node', + setupFiles: ['tests/globalFetchFallback.setup.ts'], + }, + }, + { + test: { + // Opt-in tier: run with `pnpm test:e2e` (needs E2B_E2E=1 and + // credentials). Excluded from the default `pnpm test` run. + name: 'e2e', + include: e2eFiles, + isolate: true, globals: false, testTimeout: 180_000, environment: 'node', setupFiles: ['tests/globalFetchFallback.setup.ts'], + deps: { + interopDefault: true, + }, + env: { + ...(process.env as Record), + ...env.parsed, + // Selecting this project is the opt-in, so the flag the tests gate + // on is set here instead of in the package script, which would + // need POSIX-only `VAR=value` syntax and break on Windows. + E2B_E2E: '1', + }, }, }, { diff --git a/packages/python-sdk/pytest.ini b/packages/python-sdk/pytest.ini index bb043ba0ad..2ef13ac126 100644 --- a/packages/python-sdk/pytest.ini +++ b/packages/python-sdk/pytest.ini @@ -2,11 +2,15 @@ [pytest] markers = skip_debug: skip test if E2B_DEBUG is set. + e2e: test needs live infrastructure (sandboxes, template builds) and credentials; excluded by default, run with `-m e2e`. + mocked: test mocks the API calls its fixtures would make, so it stays in the default tier even when it requests an e2e fixture. asyncio_mode=auto asyncio_default_fixture_loop_scope=session asyncio_default_test_loop_scope=session -addopts = "--import-mode=importlib" +# The default tier is fully mocked: the e2e marker (see tests/conftest.py) is +# excluded unless the run asks for it with `-m e2e`, which overrides this. +addopts = --import-mode=importlib -m "not e2e" # Makes shared test helpers (e.g. envd_frame_server) importable under importlib mode. pythonpath = tests timeout = 30 diff --git a/packages/python-sdk/tests/README.md b/packages/python-sdk/tests/README.md new file mode 100644 index 0000000000..8aeb7c12fd --- /dev/null +++ b/packages/python-sdk/tests/README.md @@ -0,0 +1,50 @@ +# Python SDK tests + +The suite has two tiers. + +## Unit tier (default) + +```bash +uv run pytest +``` + +Fully mocked (`httpx.MockTransport` and monkeypatched generated API modules), +deterministic, no sandboxes, no credentials, seconds to run. It asserts on +client-side logic: request payload shaping, config propagation, version gating, +response parsing and format switching, RPC/API error mapping, pagination, URL +construction and pure utilities. + +`pytest.ini` sets `addopts = -m "not e2e"`, so the e2e tier is excluded unless +you ask for it. + +## E2E tier (opt-in) + +```bash +E2B_API_KEY=e2b_... uv run pytest -m e2e +``` + +Everything whose assertions depend on real behavior across the RPC boundary — +process execution, filesystem round-trips, PTY semantics, git inside the VM, +sandbox lifecycle against live infrastructure and server-side template builds. +It provisions sandboxes and builds templates, so it needs an API key. + +Tests land in this tier automatically when they use one of the live fixtures +(`sandbox`, `sandbox_factory`, `async_sandbox`, `async_sandbox_factory`, `build`, +`async_build`) — see `pytest_collection_modifyitems` in +[`conftest.py`](./conftest.py). A test that calls live APIs without such a +fixture needs an explicit `@pytest.mark.e2e`; conversely, a test that mocks the +API calls its fixture would make (e.g. the template stacktrace tests) opts back +into the default tier with `@pytest.mark.mocked`. + +Serialization and hashing are shared synchronous logic, so +`tests/sync/template_sync/test_serialization.py` has no async mirror; the same +goes for the request-shaping tests under `tests/shared/`, which cover both +clients in one file. + +`skip_debug`/`E2B_DEBUG` is a separate axis: it points the SDK at a local envd +instead of a provisioned sandbox and does not enable or disable either tier. + +## CI + +`SDK Tests` runs the unit tier on every PR. The e2e tier runs in the opt-in +`SDK E2E Tests` workflow — add the `e2e` label to a PR or dispatch it manually. diff --git a/packages/python-sdk/tests/conftest.py b/packages/python-sdk/tests/conftest.py index fa8f4a0949..884f09fea6 100644 --- a/packages/python-sdk/tests/conftest.py +++ b/packages/python-sdk/tests/conftest.py @@ -40,6 +40,36 @@ def test_api_key() -> str: return "e2b_" + "0" * 40 +# Fixtures that provision live infrastructure: a sandbox on real compute or a +# server-side template build. Any test requesting one belongs to the e2e tier, +# which `pytest.ini` excludes by default (`-m "not e2e"`); run it with +# `pytest -m e2e` and credentials. Tests that reach the control plane without +# these fixtures carry an explicit `@pytest.mark.e2e`, and tests that mock the +# fixture's API calls opt back out with `@pytest.mark.mocked`. +E2E_FIXTURES = frozenset( + { + "sandbox", + "sandbox_factory", + "async_sandbox", + "async_sandbox_factory", + "build", + "async_build", + } +) + + +def pytest_collection_modifyitems(items): + for item in items: + if not isinstance(item, pytest.Function): + continue + # `pytest.mark.mocked` opts out: the test replaces the API calls the + # fixture would make, so nothing is provisioned. + if item.get_closest_marker("mocked"): + continue + if not E2E_FIXTURES.isdisjoint(item.fixturenames): + item.add_marker(pytest.mark.e2e) + + @pytest.hookimpl(tryfirst=True, hookwrapper=True) def pytest_runtest_makereport(item, call): outcome = yield diff --git a/packages/python-sdk/tests/envd_versions.py b/packages/python-sdk/tests/envd_versions.py new file mode 100644 index 0000000000..6404ed6d02 --- /dev/null +++ b/packages/python-sdk/tests/envd_versions.py @@ -0,0 +1,13 @@ +"""Helpers for testing the SDK's envd version gates.""" + +from packaging.version import Version + + +def below_envd_version(version: Version) -> str: + """The highest envd version below one of the `ENVD_*` thresholds. + + Lets a gate's reject branch be exercised without hardcoding a version that + stops being below the threshold when it moves — a release candidate of a + version sorts below the version itself. + """ + return f"{version}rc1"