From 93453e0b05c80bcb257f35a5ceae7c55e02f6ee2 Mon Sep 17 00:00:00 2001 From: Ian Flores Siaca <18703558+ian-flores@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:21:55 -0800 Subject: [PATCH 1/5] ci(workbench): experiment: run the smoke suite with xdist (-n auto) --- .github/workflows/workbench-smoke.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workbench-smoke.yml b/.github/workflows/workbench-smoke.yml index 547cfbed..d01b6374 100644 --- a/.github/workflows/workbench-smoke.yml +++ b/.github/workflows/workbench-smoke.yml @@ -317,7 +317,7 @@ jobs: fi # shellcheck disable=SC2086 # word splitting is how the file list is passed uv run pytest $files \ - -v -rs -n 0 \ + -v -rs \ --vip-config=vip.toml \ --junitxml=smoke-results.xml env: From 4dcb70059955ddd93cbe8834a1d477f4c43a9454 Mon Sep 17 00:00:00 2001 From: Ian Flores Siaca <18703558+ian-flores@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:37:52 -0800 Subject: [PATCH 2/5] ci(workbench): run the smoke suite in parallel with one user per worker Workbench refuses a second sign-in by the same user within auth-sign-in-throttle-seconds (5 s by default) and reports it as "Temporary server error", which is why parallel workers sharing one account failed. The smoke workflow now creates -gw per xdist worker and sets VIP_TEST_USERNAME_PER_WORKER=1 so each worker signs in as its own account. The login retry now waits 6 s so a retry clears the throttle window. Closes #644 --- .github/workflows/workbench-smoke.yml | 30 +++++++++++++++++--- AGENTS.md | 4 ++- selftests/test_workbench_cleanup.py | 15 ++++++++++ selftests/test_workbench_parallel.py | 41 +++++++++++++++++++++++++++ src/vip_tests/workbench/cleanup.py | 4 +-- src/vip_tests/workbench/conftest.py | 8 ++++++ src/vip_tests/workbench/login.py | 6 ++-- src/vip_tests/workbench/naming.py | 18 ++++++++++++ 8 files changed, 117 insertions(+), 9 deletions(-) diff --git a/.github/workflows/workbench-smoke.yml b/.github/workflows/workbench-smoke.yml index d01b6374..eb8bc6fa 100644 --- a/.github/workflows/workbench-smoke.yml +++ b/.github/workflows/workbench-smoke.yml @@ -247,11 +247,31 @@ jobs: env: SUITE: ${{ inputs.suite || (github.event_name == 'schedule' && 'full' || 'gate') }} + # Workbench refuses a second sign-in by the same user within + # auth-sign-in-throttle-seconds (5 s by default) and answers it with + # "Temporary server error", so xdist workers sharing one account trip it. + # Give each worker its own account, named the way VIP derives it under + # VIP_TEST_USERNAME_PER_WORKER (-gw), with the same password. The + # password goes in on stdin so it never appears in a process list. + - name: Create per-worker test users + id: workers + env: + CID: ${{ steps.workbench.outputs.CONTAINER_ID }} + WB_USER: ${{ steps.workbench.outputs.WORKBENCH_USER }} + WB_PASSWORD: ${{ steps.workbench.outputs.WORKBENCH_PASSWORD }} + run: | + WORKERS=$(nproc) + for i in $(seq 0 $((WORKERS - 1))); do + docker exec "${CID}" useradd -m -s /bin/bash "${WB_USER}-gw${i}" + printf '%s:%s\n' "${WB_USER}-gw${i}" "${WB_PASSWORD}" | docker exec -i "${CID}" chpasswd + done + echo "Created ${WORKERS} per-worker users (${WB_USER}-gw0..$((WORKERS - 1)))" + echo "count=${WORKERS}" >> "$GITHUB_OUTPUT" + # Run the subset of tests that work against Docker Workbench - name: Run Workbench smoke tests - # -n 0 disables xdist (overriding -n auto from pyproject.toml) because - # the Docker Workbench returns "Temporary server error" when multiple - # workers attempt concurrent logins. + # -n matches the per-worker users created above, so every worker has an + # account to sign in as. # # No `-k` filter: the plugin already deselects scenarios for products # that are not enabled in vip.toml, so the previous `-k "workbench"` @@ -317,7 +337,7 @@ jobs: fi # shellcheck disable=SC2086 # word splitting is how the file list is passed uv run pytest $files \ - -v -rs \ + -v -rs -n "${WORKERS}" \ --vip-config=vip.toml \ --junitxml=smoke-results.xml env: @@ -325,6 +345,8 @@ jobs: # slow suites are still exercised daily without taxing the merge path. SUITE: ${{ inputs.suite || (github.event_name == 'schedule' && 'full' || 'gate') }} VIP_TEST_PASSWORD: ${{ steps.workbench.outputs.WORKBENCH_PASSWORD }} + VIP_TEST_USERNAME_PER_WORKER: "1" + WORKERS: ${{ steps.workers.outputs.count }} # Tolerant, unretried upload -- see connect-smoke.yml's "Upload test # results" step for the full rationale, including why no safe retry diff --git a/AGENTS.md b/AGENTS.md index ef5b62af..3a0e63d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -208,7 +208,7 @@ Key principles: | `src/vip/install/plan.py` | Pure `build_install_plan` / `build_uninstall_plan` builders | | `src/vip/install/runner.py` | Plan executor: dry-run formatting + execute (system packages, Playwright, manifest writes) | | `src/vip_tests/conftest.py` | Directory-scoped warning filter (kept out of the global plugin deliberately) plus the three autouse Connect content-cleanup fixtures — see that file's docstring for why those stay directory-scoped instead of moving to `src/vip/fixtures.py` | -| `src/vip_tests/workbench/naming.py` | Worker-scoped Workbench session names: `current_worker_id`, `vip_session_prefix`, `capacity_session_prefix`, `k8s_session_prefix`, `unique_session_name` — see "Workbench session ownership" below | +| `src/vip_tests/workbench/naming.py` | Worker-scoped Workbench session names: `current_worker_id`, `vip_session_prefix`, `capacity_session_prefix`, `k8s_session_prefix`, `unique_session_name` — see "Workbench session ownership" below; also `worker_username`, the per-worker account behind `VIP_TEST_USERNAME_PER_WORKER` | | `src/vip_tests/workbench/capacity.py` | Resource-profile detection and capping shared by `test_session_capacity.py` and `test_session_capacity_k8s.py` (`ResourceProfileDisabledError`) | | `src/vip_tests/workbench/cleanup.py` | Cookie-based Workbench session cleanup, the API-token sweep used when there are no cookies (`--api-auth`), and the autouse cleanup fixtures. `@api_auth` scenarios get the token sweep and never request `page` | | `src/vip_tests/workbench/sessions.py` | Session-state waits, failure/timeout messages, and the unproven-skip helpers for Workbench scenarios, plus `join_session` — the one way to click a session's join link (waits for Active, retries the click once with a warning, fails if the browser never enters the session). Don't hand-roll `a[title='join ...']` clicks in step files | @@ -289,6 +289,7 @@ Configuration is in `vip.toml` (see `vip.toml.example` for the template). Secret - `VIP_WORKBENCH_API_KEY` — Workbench API token (`Authorization: Bearer`). Under `--api-auth` it launches IDE sessions through the Workbench API and lets cleanup run without a browser; use a super-admin token (see `vip.toml.example`) - `VIP_TEST_USERNAME` - `VIP_TEST_PASSWORD` +- `VIP_TEST_USERNAME_PER_WORKER` — set to `1` to give each xdist worker its own Workbench account, `-gw`, sharing the configured password. Workbench refuses a repeat sign-in by one user within `auth-sign-in-throttle-seconds` (5 s by default), which parallel workers sharing an account hit as "Temporary server error". The accounts must already exist; `workbench-smoke.yml` creates them. Serial runs ignore it - `VIP_TEST_TOTP_SECRET` — optional base32 TOTP seed used by `--headless-auth` to auto-fill MFA codes for a dedicated test service account. **Equivalent to bypassing 2FA — never use a personal account's seed.** The plugin loads config via `--vip-config` or defaults to `./vip.toml`. If no config file exists, all product tests are skipped. @@ -339,6 +340,7 @@ Every render also produces `_output/vip-report.pdf` from `report/vip-report.qmd` - **Smoke workflows** (`connect-smoke.yml`, `workbench-smoke.yml`, `packagemanager-smoke.yml`, `mock-idp-e2e.yml`) -- run the product suites against real containers. On PR/push each tests a single latest version (change-gated via a `changes` paths-filter job); on `schedule` (nightly, staggered hourly) and `workflow_dispatch` a `set-matrix` job fans each out across the product version support window (current + 2 back). There are no pinned tags to bump: every `set-matrix` job calls `recent-production-versions.yml`, which resolves versions from the product's own GHCR tag list at run time. Each workflow's `*-status` aggregation job is scope-aware: it passes when the suite was legitimately out of scope (the PR's paths didn't match) but **fails** when the suite was in scope (`changes.relevant == 'true'`) yet did not succeed -- so a path-gated skip, an excluded actor, or a missing license secret cannot report a green required check without the suite having run. The Connect, Workbench, and Package Manager `*-status` jobs are required merge checks. `connect-smoke.yml` also starts a Mailpit container and points Connect's SMTP settings at it (through with-connect's `env:` input, using `CONNECT_
_` overrides), gives the bootstrap admin an email address, and enables `[email]`, so `test_email` performs a real send instead of skipping. A follow-up step fails the job if Mailpit received nothing. `mock-idp-e2e.yml` is structured the same way so `Mock-IdP E2E Status` can be promoted to a required check via a separate admin action. It seeds a Keycloak realm and has no Entra lane: Entra ID is a Microsoft-hosted login app that can't be stood up in a container, so the `entra` strategy in `src/vip/idp.py` is validated against a real tenant instead of mock-IdP CI. That test account must also be excluded from Conditional Access device-compliance policies, since headless auth runs in an unmanaged browser Entra's CA would otherwise block. + `workbench-smoke.yml` runs the suite in parallel with one Workbench account per xdist worker: a step creates `-gw0..N-1` in the container (N = the runner's `nproc`, also passed as `-n`) and the test step sets `VIP_TEST_USERNAME_PER_WORKER=1`. One shared account fails under xdist because Workbench throttles repeat sign-ins by the same user (`auth-sign-in-throttle-seconds`, 5 s by default). `workbench-smoke.yml` additionally splits into two tiers via a `suite` value: PR/push runs `gate` (the fast subset), the nightly schedule runs `full` (every Workbench file), and `workflow_dispatch` can pick either. The split is based on a measured run rather than taste — `full` buys 8 more real passes for ~523s more test time, which is worth a nightly but not a merge gate. Skip reasons do **not** appear in the log even with `-rs`, because VIP's plugin owns the terminal reporter; read them from `` in the uploaded `smoke-results.xml`. `workbench-smoke.yml` also has an `API-auth smoke test` job that runs `vip verify --api-auth` over `workbench/test_ide_launch_api.py` against the same kind of container with no browser installed. It appends `workbench-api-enabled=1` and `workbench-api-super-admin-enabled=1` to `rserver.conf`, restarts the container with `docker restart` (the image's supervisor stops the container if rserver alone exits), mints a super-admin token with `rstudio-server generate-api-token ... --token-env`, and hands it to VIP as `VIP_WORKBENCH_API_KEY` step env, never in `vip.toml`. It then reruns the RStudio scenario with a `user` token owned by the test user, the other documented launch path. A final step fails the job unless the RStudio scenario passed, rather than skipped, under both tokens. `Workbench Smoke Tests Status` aggregates it with the main job. The three cross-cutting suites (`cross_product/test_resources`, `security/test_auth_policy`, `config_hygiene/test_secrets`) run in all three product workflows. `test_secrets` asserts no plaintext `api_key`/`password` in the generated `vip.toml`, so credentials must be passed to pytest as step `env` (`VIP_CONNECT_API_KEY`, `VIP_TEST_PASSWORD`) and never written into the config file. diff --git a/selftests/test_workbench_cleanup.py b/selftests/test_workbench_cleanup.py index 58bc642b..bbd58464 100644 --- a/selftests/test_workbench_cleanup.py +++ b/selftests/test_workbench_cleanup.py @@ -1201,3 +1201,18 @@ def test_token_sweep_logs_instead_of_raising_when_the_api_refuses(caplog): with caplog.at_level(logging.WARNING, logger="vip_tests.workbench.cleanup"): assert cleanup._quit_vip_sessions_via_token(client, _token_config(), owner="main") == 0 assert "could not sweep VIP Workbench sessions" in caplog.text + + +def test_token_sweep_targets_this_workers_own_account(monkeypatch): + from types import SimpleNamespace + + from vip_tests.workbench import cleanup + + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw1") + monkeypatch.setenv("VIP_TEST_USERNAME_PER_WORKER", "1") + users: list[str] = [] + client = SimpleNamespace( + quit_vip_sessions_via_api=lambda user, *, owner: users.append(user) or 0 + ) + cleanup._quit_vip_sessions_via_token(client, _token_config(), owner="gw1") + assert users == ["alice-gw1"] diff --git a/selftests/test_workbench_parallel.py b/selftests/test_workbench_parallel.py index d0009ee5..d60db3e4 100644 --- a/selftests/test_workbench_parallel.py +++ b/selftests/test_workbench_parallel.py @@ -548,3 +548,44 @@ def test_regroup_wins_via_get_closest_marker_and_leaves_no_duplicate(self, pytes # And no leftover duplicate, or xdist would glue both names into one composite group. groups = [m.args[0] for m in item.iter_markers("xdist_group")] assert groups == ["workbench_packages"] + + +class TestPerWorkerUsername: + """Workbench throttles repeat sign-ins by one user, so parallel CI gives each worker its own.""" + + def test_opt_in_suffixes_the_worker_id(self, monkeypatch): + from vip_tests.workbench.naming import worker_username + + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw2") + monkeypatch.setenv("VIP_TEST_USERNAME_PER_WORKER", "1") + assert worker_username("testuser") == "testuser-gw2" + + @pytest.mark.parametrize( + ("worker", "flag", "base"), + [ + ("gw2", None, "testuser"), # not opted in: customers keep their one account + ("gw2", "0", "testuser"), + (None, "1", "testuser"), # serial run: no worker to suffix + ("gw2", "1", ""), # no configured user: nothing to derive from + ], + ids=["no-flag", "flag-off", "serial", "no-base"], + ) + def test_base_username_is_kept_otherwise(self, monkeypatch, worker, flag, base): + from vip_tests.workbench.naming import worker_username + + for name, value in ( + ("PYTEST_XDIST_WORKER", worker), + ("VIP_TEST_USERNAME_PER_WORKER", flag), + ): + if value is None: + monkeypatch.delenv(name, raising=False) + else: + monkeypatch.setenv(name, value) + assert worker_username(base) == base + + def test_login_retry_waits_out_the_sign_in_throttle(self): + import inspect + + # Workbench's auth-sign-in-throttle-seconds defaults to 5; a shorter wait retries into it. + default = inspect.signature(login.workbench_login).parameters["retry_delay"].default + assert default > 5 diff --git a/src/vip_tests/workbench/cleanup.py b/src/vip_tests/workbench/cleanup.py index cd0b2934..f12fd2d1 100644 --- a/src/vip_tests/workbench/cleanup.py +++ b/src/vip_tests/workbench/cleanup.py @@ -16,7 +16,7 @@ from vip.workbench_ui import ( quit_vip_sessions_via_ui as _quit_vip_sessions_via_ui, ) -from vip_tests.workbench.naming import current_worker_id +from vip_tests.workbench.naming import current_worker_id, worker_username logger = logging.getLogger(__name__) @@ -128,7 +128,7 @@ def _quit_vip_sessions_via_token( """ if not vip_config.workbench.api_key: return 0 - username = vip_config.auth.username + username = worker_username(vip_config.auth.username) if not username: return 0 try: diff --git a/src/vip_tests/workbench/conftest.py b/src/vip_tests/workbench/conftest.py index 52e8c7ee..e9819c8d 100644 --- a/src/vip_tests/workbench/conftest.py +++ b/src/vip_tests/workbench/conftest.py @@ -21,6 +21,7 @@ from playwright.sync_api import Page from pytest_bdd import given +from vip.config import VIPConfig from vip.plugin import _auth_session_key # Re-exported for selftests/test_workbench_cleanup.py, which imports it from @@ -59,6 +60,7 @@ k8s_session_prefix, unique_session_name, vip_session_prefix, + worker_username, ) from vip_tests.workbench.sessions import ( _navigated_into_session, @@ -290,6 +292,12 @@ def extract_repo_urls(output: str) -> list[str]: return re.findall(r"https?://[^\s<>\"']+", output, re.IGNORECASE) +@pytest.fixture(scope="session") +def test_username(vip_config: VIPConfig) -> str: + """The Workbench test account, one per xdist worker under ``VIP_TEST_USERNAME_PER_WORKER``.""" + return worker_username(vip_config.auth.username) + + @pytest.fixture def wb_login( page: Page, diff --git a/src/vip_tests/workbench/login.py b/src/vip_tests/workbench/login.py index 3cf2a64a..7b36e255 100644 --- a/src/vip_tests/workbench/login.py +++ b/src/vip_tests/workbench/login.py @@ -224,7 +224,7 @@ def workbench_login( auth_mode: str = "none", workbench_auth_error: str | None = None, max_retries: int = 3, - retry_delay: float = 2.0, + retry_delay: float = 6.0, ) -> None: """Navigate to Workbench homepage, logging in only if required. @@ -248,7 +248,9 @@ def workbench_login( establish a Workbench session, if known. Quoted in the skip message so users see the real cause instead of a guess. max_retries: Max login attempts on transient errors (default 3) - retry_delay: Seconds to wait between retries (default 2.0) + retry_delay: Seconds to wait between retries (default 6.0). Longer than + Workbench's per-user sign-in throttle (5 s by default), which answers a + too-quick repeat sign-in with "Temporary server error". Raises: pytest.skip: For non-password auth without a pre-loaded auth session, diff --git a/src/vip_tests/workbench/naming.py b/src/vip_tests/workbench/naming.py index 666a4846..0e31badc 100644 --- a/src/vip_tests/workbench/naming.py +++ b/src/vip_tests/workbench/naming.py @@ -20,6 +20,24 @@ def current_worker_id() -> str: return os.environ.get("PYTEST_XDIST_WORKER", "main") +# Opt-in switch for one Workbench account per xdist worker; the deployment must already have them. +PER_WORKER_USERS_ENV = "VIP_TEST_USERNAME_PER_WORKER" + + +def worker_username(base: str) -> str: + """Return the Workbench test account this worker signs in as. + + Workbench refuses a second sign-in by the same user within + ``auth-sign-in-throttle-seconds`` (default 5 s), so parallel workers sharing one + account fail with "Temporary server error". With ``VIP_TEST_USERNAME_PER_WORKER=1`` + each worker uses ``-`` (``testuser-gw0``); serial runs keep *base*. + """ + worker = os.environ.get("PYTEST_XDIST_WORKER") + if not base or not worker or os.environ.get(PER_WORKER_USERS_ENV) != "1": + return base + return f"{base}-{worker}" + + def vip_session_prefix(kind: str) -> str: """Build a ``_vip____`` session-name prefix. From 02147b38183ee06b6719a01d62ea5e311ce04c87 Mon Sep 17 00:00:00 2001 From: Ian Flores Siaca <18703558+ian-flores@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:46:24 -0800 Subject: [PATCH 3/5] fix(workbench): clone into a per-worker directory Every IDE scenario cloned into /tmp/ and removed it first, so parallel workers removed or cloned over each other's checkout; as different users the rm failed with Permission denied. Clone into /tmp/vip_git_/ instead. --- selftests/test_workbench_git_ops.py | 25 +++++++++++++++++++++++-- src/vip_tests/workbench/test_git_ops.py | 14 +++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/selftests/test_workbench_git_ops.py b/selftests/test_workbench_git_ops.py index 619d0880..9abd9ccc 100644 --- a/selftests/test_workbench_git_ops.py +++ b/selftests/test_workbench_git_ops.py @@ -40,20 +40,41 @@ def fake_terminal_run(page, cmd, timeout=30_000, *, readback_lang="r"): return "" monkeypatch.setattr(git_ops, "terminal_run", fake_terminal_run) + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw3") git_cfg = _FakeGitCfg("https://github.com/example/jumpstart_examples.git") git_session_ctx: dict = {} clone_dir = git_ops._do_clone(object(), git_session_ctx, git_cfg, readback_lang="r") - assert clone_dir == "/tmp/jumpstart_examples" + assert clone_dir == "/tmp/vip_git_gw3/jumpstart_examples" assert git_session_ctx["clone_dir"] == clone_dir - rm_index = captured["cmd"].find("rm -rf /tmp/jumpstart_examples") + rm_index = captured["cmd"].find("rm -rf /tmp/vip_git_gw3/jumpstart_examples") clone_index = captured["cmd"].find("git clone") assert rm_index != -1, f"expected rm -rf of clone dir in command: {captured['cmd']!r}" assert rm_index < clone_index, "rm -rf must run before git clone" +def test_do_clone_gives_each_worker_its_own_directory(monkeypatch): + # Parallel workers (often different users) must not delete or clone over each other's checkout. + cmds: list[str] = [] + monkeypatch.setattr( + git_ops, + "terminal_run", + lambda page, cmd, timeout=30_000, *, readback_lang="r": cmds.append(cmd), + ) + git_cfg = _FakeGitCfg("https://github.com/example/repo.git") + + dirs = [] + for worker in ("gw0", "gw1"): + monkeypatch.setenv("PYTEST_XDIST_WORKER", worker) + dirs.append(git_ops._do_clone(object(), {}, git_cfg, readback_lang="r")) + + assert dirs == ["/tmp/vip_git_gw0/repo", "/tmp/vip_git_gw1/repo"] + # The per-worker parent does not exist on a fresh server, so it is created before cd. + assert cmds[0].index("mkdir -p /tmp/vip_git_gw0") < cmds[0].index("cd /tmp/vip_git_gw0") + + def test_do_clone_uses_custom_workdir(monkeypatch): captured: dict = {} diff --git a/src/vip_tests/workbench/test_git_ops.py b/src/vip_tests/workbench/test_git_ops.py index 183c6ce2..6ba7139b 100644 --- a/src/vip_tests/workbench/test_git_ops.py +++ b/src/vip_tests/workbench/test_git_ops.py @@ -36,6 +36,7 @@ workbench_login, ) from vip_tests.workbench.exec import ExecError, ensure_positron_console, file_exists, terminal_run +from vip_tests.workbench.naming import current_worker_id from vip_tests.workbench.pages import ( Homepage, NewSessionDialog, @@ -377,16 +378,23 @@ def _do_clone( git_cfg, readback_lang: str, *, - workdir: str = "/tmp", + workdir: str | None = None, ) -> str: - """Run ``git clone`` in the IDE terminal and return the cloned directory path.""" + """Run ``git clone`` in the IDE terminal and return the cloned directory path. + + Defaults to ``/tmp/vip_git_`` so parallel workers, which may be different + users, never remove or clone over each other's checkout. + """ + if workdir is None: + workdir = f"/tmp/vip_git_{current_worker_id()}" auth_url = _inject_token_into_url(git_cfg.clone_url, git_cfg.token) repo_dir = _repo_dir_from_url(git_cfg.clone_url) clone_dir = f"{workdir}/{repo_dir}" + q_workdir = shlex.quote(workdir) terminal_run( page, - f"rm -rf {shlex.quote(clone_dir)} && cd {shlex.quote(workdir)} && " + f"rm -rf {shlex.quote(clone_dir)} && mkdir -p {q_workdir} && cd {q_workdir} && " f"GIT_TERMINAL_PROMPT=0 git clone {shlex.quote(auth_url)}", timeout=_TIMEOUT_GIT_NETWORK, readback_lang=readback_lang, From f09f4fa4396057c66fb42c70c30df70ef950194e Mon Sep 17 00:00:00 2001 From: Ian Flores Siaca <18703558+ian-flores@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:12:12 -0800 Subject: [PATCH 4/5] fix(plugin): apply xdist groups before xdist reads them pytest-xdist reads xdist_group markers in its own collection hook, which ran before VIP's, so no group VIP assigned ever took effect and --dist loadgroup spread every test freely. Run VIP's hooks tryfirst, keep groups only for product directories, and group Workbench scenarios by IDE and module when VIP_TEST_USERNAME_PER_WORKER is set. --- AGENTS.md | 6 +++- selftests/test_workbench_parallel.py | 44 ++++++++++++++++++++++++++-- src/vip/plugin/selection.py | 17 +++++++---- src/vip_tests/workbench/conftest.py | 26 ++++++++-------- src/vip_tests/workbench/naming.py | 7 ++++- 5 files changed, 75 insertions(+), 25 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3a0e63d3..5a54b017 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -340,7 +340,7 @@ Every render also produces `_output/vip-report.pdf` from `report/vip-report.qmd` - **Smoke workflows** (`connect-smoke.yml`, `workbench-smoke.yml`, `packagemanager-smoke.yml`, `mock-idp-e2e.yml`) -- run the product suites against real containers. On PR/push each tests a single latest version (change-gated via a `changes` paths-filter job); on `schedule` (nightly, staggered hourly) and `workflow_dispatch` a `set-matrix` job fans each out across the product version support window (current + 2 back). There are no pinned tags to bump: every `set-matrix` job calls `recent-production-versions.yml`, which resolves versions from the product's own GHCR tag list at run time. Each workflow's `*-status` aggregation job is scope-aware: it passes when the suite was legitimately out of scope (the PR's paths didn't match) but **fails** when the suite was in scope (`changes.relevant == 'true'`) yet did not succeed -- so a path-gated skip, an excluded actor, or a missing license secret cannot report a green required check without the suite having run. The Connect, Workbench, and Package Manager `*-status` jobs are required merge checks. `connect-smoke.yml` also starts a Mailpit container and points Connect's SMTP settings at it (through with-connect's `env:` input, using `CONNECT_
_` overrides), gives the bootstrap admin an email address, and enables `[email]`, so `test_email` performs a real send instead of skipping. A follow-up step fails the job if Mailpit received nothing. `mock-idp-e2e.yml` is structured the same way so `Mock-IdP E2E Status` can be promoted to a required check via a separate admin action. It seeds a Keycloak realm and has no Entra lane: Entra ID is a Microsoft-hosted login app that can't be stood up in a container, so the `entra` strategy in `src/vip/idp.py` is validated against a real tenant instead of mock-IdP CI. That test account must also be excluded from Conditional Access device-compliance policies, since headless auth runs in an unmanaged browser Entra's CA would otherwise block. - `workbench-smoke.yml` runs the suite in parallel with one Workbench account per xdist worker: a step creates `-gw0..N-1` in the container (N = the runner's `nproc`, also passed as `-n`) and the test step sets `VIP_TEST_USERNAME_PER_WORKER=1`. One shared account fails under xdist because Workbench throttles repeat sign-ins by the same user (`auth-sign-in-throttle-seconds`, 5 s by default). + `workbench-smoke.yml` runs the suite in parallel with one Workbench account per xdist worker: a step creates `-gw0..N-1` in the container (N = the runner's `nproc`, also passed as `-n`) and the test step sets `VIP_TEST_USERNAME_PER_WORKER=1`. One shared account fails under xdist because Workbench throttles repeat sign-ins by the same user (`auth-sign-in-throttle-seconds`, 5 s by default). With the flag set, the Workbench conftest groups scenarios by IDE and by module, as it does under a shared auth session; without it, every Workbench scenario stays in one `workbench` xdist group, so a single account signs in from one worker. `workbench-smoke.yml` additionally splits into two tiers via a `suite` value: PR/push runs `gate` (the fast subset), the nightly schedule runs `full` (every Workbench file), and `workflow_dispatch` can pick either. The split is based on a measured run rather than taste — `full` buys 8 more real passes for ~523s more test time, which is worth a nightly but not a merge gate. Skip reasons do **not** appear in the log even with `-rs`, because VIP's plugin owns the terminal reporter; read them from `` in the uploaded `smoke-results.xml`. `workbench-smoke.yml` also has an `API-auth smoke test` job that runs `vip verify --api-auth` over `workbench/test_ide_launch_api.py` against the same kind of container with no browser installed. It appends `workbench-api-enabled=1` and `workbench-api-super-admin-enabled=1` to `rserver.conf`, restarts the container with `docker restart` (the image's supervisor stops the container if rserver alone exits), mints a super-admin token with `rstudio-server generate-api-token ... --token-env`, and hands it to VIP as `VIP_WORKBENCH_API_KEY` step env, never in `vip.toml`. It then reruns the RStudio scenario with a `user` token owned by the test user, the other documented launch path. A final step fails the job unless the RStudio scenario passed, rather than skipped, under both tokens. `Workbench Smoke Tests Status` aggregates it with the main job. The three cross-cutting suites (`cross_product/test_resources`, `security/test_auth_policy`, `config_hygiene/test_secrets`) run in all three product workflows. `test_secrets` asserts no plaintext `api_key`/`password` in the generated `vip.toml`, so credentials must be passed to pytest as step `env` (`VIP_CONNECT_API_KEY`, `VIP_TEST_PASSWORD`) and never written into the config file. @@ -428,6 +428,10 @@ refactor(connect)!: rename client constructor parameters - Missing the colon and space after the type (e.g. `feat add feature` — must be `feat: add feature`). - Using a PR title that is not conventional when the branch will be squash-merged. +## xdist groups + +`--dist loadgroup` (the default in `pyproject.toml`) keeps every test with the same `xdist_group` on one worker. `plugin/selection.py` groups tests in a `connect/`, `workbench/` or `package_manager/` directory by that name and leaves everything else ungrouped; the Workbench conftest refines its group by IDE and module when concurrent logins are safe. pytest-xdist reads the groups in its own `pytest_collection_modifyitems` on each worker, so every hook that adds a group must be `@pytest.hookimpl(tryfirst=True)` -- a group added after xdist's hook is silently ignored. `selftests/test_workbench_parallel.py::TestXdistSeesVipGroups` runs real workers to guard this. + ## Pytest warning filters Register warning filters in `src/vip/plugin/configure.py::pytest_configure` (via `config.addinivalue_line("filterwarnings", ...)`), not in `pyproject.toml`'s `[tool.pytest.ini_options]`. Filters in `pyproject.toml` only apply when pytest runs from this repo's rootdir -- users who install vip into another project pick up the plugin but not the config, so the warnings reappear there. Keeping the filters in the plugin means they travel with the installed package. diff --git a/selftests/test_workbench_parallel.py b/selftests/test_workbench_parallel.py index d60db3e4..8a3a1f1b 100644 --- a/selftests/test_workbench_parallel.py +++ b/selftests/test_workbench_parallel.py @@ -221,11 +221,25 @@ class TestCollectionHook: def _wb_dir(self): return Path(wb.__file__).parent - def test_no_auth_session_is_a_noop(self): + def test_no_auth_session_is_a_noop(self, monkeypatch): + monkeypatch.delenv("VIP_TEST_USERNAME_PER_WORKER", raising=False) item = _FakeItem(self._wb_dir() / "test_packages.py", []) wb.pytest_collection_modifyitems(_FakeConfig(None), [item]) assert item.added == [] # untouched under password / no-auth + def test_per_worker_users_group_without_an_auth_session(self, monkeypatch): + # Password auth with one account per worker can log in concurrently, so it spreads too. + monkeypatch.setenv("VIP_TEST_USERNAME_PER_WORKER", "1") + launch = _FakeItem(self._wb_dir() / "test_ide_launch.py", ["workbench", "vscode"]) + extensions = _FakeItem(self._wb_dir() / "test_ide_extensions.py", ["workbench", "vscode"]) + packages = _FakeItem(self._wb_dir() / "test_packages.py", ["workbench"]) + wb.pytest_collection_modifyitems(_FakeConfig(None), [launch, extensions, packages]) + assert launch.added == [("xdist_group", ("workbench_ide_vscode",))] + assert ( + extensions.added == launch.added + ) # an IDE's launch and extension tests share a worker + assert packages.added == [("xdist_group", ("workbench_packages",))] + def test_ide_item_grouped_by_ide(self): item = _FakeItem(self._wb_dir() / "test_ide_launch.py", ["workbench", "rstudio"]) wb.pytest_collection_modifyitems(_FakeConfig(object()), [item]) @@ -532,8 +546,8 @@ class TestRealMarkerMechanics: """ def test_regroup_wins_via_get_closest_marker_and_leaves_no_duplicate(self, pytester): - # Disable the vip plugin for the nested collection: its own _assign_xdist_group - # would inject a "general" group and obscure the mechanic under test. + # Disable the vip plugin for the nested collection so its own _assign_xdist_group + # cannot add a group and obscure the mechanic under test. modcol = pytester.getmodulecol("def test_x(): pass", configargs=["-p", "no:vip"]) (item,) = pytester.genitems([modcol]) # Simulate a pre-existing group, then apply exactly the hook's two operations. @@ -589,3 +603,27 @@ def test_login_retry_waits_out_the_sign_in_throttle(self): # Workbench's auth-sign-in-throttle-seconds defaults to 5; a shorter wait retries into it. default = inspect.signature(login.workbench_login).parameters["retry_delay"].default assert default > 5 + + +class TestXdistSeesVipGroups: + """xdist reads ``xdist_group`` in its own collection hook, so VIP's must run first.""" + + def test_directory_group_keeps_its_tests_on_one_worker(self, pytester): + pytester.makefile(".toml", vip='[general]\ndeployment_name = "Selftest"') + out = pytester.path / "workers.txt" + pytester.mkdir("workbench") + pytester.path.joinpath("workbench", "test_grouped.py").write_text( + "import os, pytest\n" + "@pytest.mark.parametrize('i', range(6))\n" + "def test_where(i, request):\n" + f" with open({str(out)!r}, 'a', encoding='utf-8') as f:\n" + " worker = os.environ['PYTEST_XDIST_WORKER']\n" + " f.write(worker + ' ' + request.node.nodeid + '\\n')\n", + encoding="utf-8", + ) + result = pytester.runpytest("--vip-config=vip.toml", "-n", "3", "--dist", "loadgroup") + result.assert_outcomes(passed=6) + + rows = [line.split(" ", 1) for line in out.read_text(encoding="utf-8").splitlines()] + assert all(nodeid.endswith("@workbench") for _, nodeid in rows), rows + assert len({worker for worker, _ in rows}) == 1, rows diff --git a/src/vip/plugin/selection.py b/src/vip/plugin/selection.py index 514c0031..38092676 100644 --- a/src/vip/plugin/selection.py +++ b/src/vip/plugin/selection.py @@ -20,12 +20,16 @@ from vip.version import ProductVersion +@pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems( config: pytest.Config, items: list[pytest.Item], ) -> None: """Deselect tests whose product is not configured, skip tests whose version requirement is not met, and ensure prerequisites run first. + + ``tryfirst`` so the ``xdist_group`` markers exist before pytest-xdist's own collection + hook reads them on each worker; after it, ``--dist loadgroup`` never sees a group. """ vip_cfg: VIPConfig = config.stash[_vip_config_key] no_auth = config.getoption("--no-auth", default=False) @@ -72,17 +76,18 @@ def _assign_xdist_group(item: pytest.Item) -> None: """Assign an ``xdist_group`` marker so each product runs on its own worker. Respects any existing ``xdist_group`` marker (set via conftest.py - ``pytestmark`` or per-test decorators). Otherwise, tests under - ``tests/connect/``, ``tests/workbench/``, or ``tests/package_manager/`` - are grouped by directory name, and everything else (prerequisites, - cross_product, performance, security) lands in a shared ``general`` group. + ``pytestmark`` or per-test decorators). Otherwise, tests under a + ``connect/``, ``workbench/``, or ``package_manager/`` directory are grouped + by directory name. Everything else (prerequisites, cross_product, + security, extensions) stays ungrouped, so ``--dist loadgroup`` spreads it + across workers. """ if item.get_closest_marker("xdist_group") is not None: return fspath = getattr(item, "path", None) or Path() dir_name = fspath.parent.name - group = dir_name if dir_name in _PRODUCT_DIRS else "general" - item.add_marker(pytest.mark.xdist_group(group)) + if dir_name in _PRODUCT_DIRS: + item.add_marker(pytest.mark.xdist_group(dir_name)) # Every marker that names a product *item* needs: ``@connect`` and ``@requires_connect`` alike. diff --git a/src/vip_tests/workbench/conftest.py b/src/vip_tests/workbench/conftest.py index e9819c8d..f6957984 100644 --- a/src/vip_tests/workbench/conftest.py +++ b/src/vip_tests/workbench/conftest.py @@ -58,6 +58,7 @@ capacity_session_prefix, current_worker_id, k8s_session_prefix, + per_worker_users_enabled, unique_session_name, vip_session_prefix, worker_username, @@ -182,27 +183,24 @@ def _workbench_group_name(ide_markers: set[str], module_stem: str) -> str: return f"workbench_{stem}" +@pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems( config: pytest.Config, items: list[pytest.Item], ) -> None: - """Group Workbench tests for parallel execution when a shared auth session is active. + """Group Workbench tests for parallel execution when the run can log in concurrently. - Under --interactive-auth / --headless-auth all Workbench tests authenticate as the same - shared account. Rather than pin them all to one worker (the old serial workaround), we - group them so LoadGroupScheduling spreads them across workers: IDE-launch scenarios by + That is a shared auth session (--interactive-auth / --headless-auth, whose concurrent SSO + round-trips :func:`oidc_login_lock` serializes) or one account per worker + (``VIP_TEST_USERNAME_PER_WORKER``, which sidesteps Workbench's per-user sign-in + throttle). LoadGroupScheduling then spreads the groups across workers: IDE scenarios by IDE (``workbench_ide_``), everything else by feature module (``workbench_``). - The simultaneous-login storm this used to cause is prevented by the cross-worker login - lock in :func:`workbench_login` (see :func:`oidc_login_lock`), not by serialization. - - Password / no-auth runs are left untouched: they hit the early return below, so their - Workbench items keep the default ``workbench`` group that ``plugin/selection.py``'s - :func:`_assign_xdist_group` directory fallback assigns (the module-level ``pytestmark`` - at the top of this file does not propagate to sibling test modules, so it assigns - nothing here). + + Any other password / no-auth run keeps the single ``workbench`` group that + ``plugin/selection.py``'s :func:`_assign_xdist_group` assigns, so one worker signs in as + the one account. ``tryfirst`` because xdist reads the groups in its own collection hook. """ - if config.stash.get(_auth_session_key, None) is None: - # No shared auth session — password auth or no auth. Keep default parallel behavior. + if config.stash.get(_auth_session_key, None) is None and not per_worker_users_enabled(): return workbench_dir = Path(__file__).parent diff --git a/src/vip_tests/workbench/naming.py b/src/vip_tests/workbench/naming.py index 0e31badc..5b1e1b49 100644 --- a/src/vip_tests/workbench/naming.py +++ b/src/vip_tests/workbench/naming.py @@ -24,6 +24,11 @@ def current_worker_id() -> str: PER_WORKER_USERS_ENV = "VIP_TEST_USERNAME_PER_WORKER" +def per_worker_users_enabled() -> bool: + """True when this run gives each xdist worker its own Workbench account.""" + return os.environ.get(PER_WORKER_USERS_ENV) == "1" + + def worker_username(base: str) -> str: """Return the Workbench test account this worker signs in as. @@ -33,7 +38,7 @@ def worker_username(base: str) -> str: each worker uses ``-`` (``testuser-gw0``); serial runs keep *base*. """ worker = os.environ.get("PYTEST_XDIST_WORKER") - if not base or not worker or os.environ.get(PER_WORKER_USERS_ENV) != "1": + if not base or not worker or not per_worker_users_enabled(): return base return f"{base}-{worker}" From 6481b79d83bb42ee1d4f9e68264277bda6b27663 Mon Sep 17 00:00:00 2001 From: Ian Flores Siaca <18703558+ian-flores@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:20:05 -0800 Subject: [PATCH 5/5] build(deps): hold pytest at 9.1.1 until pytest-xdist supports 9.2 pytest 9.2 reads node ids from a field pytest-xdist 3.8.0 does not write its loadgroup suffix to, so every xdist_group was ignored (pytest-dev/pytest-xdist#1407). Pin 9.1.1 and keep Dependabot off 9.2 until a newer xdist release. --- .github/dependabot.yml | 6 ++++++ AGENTS.md | 2 +- pyproject.toml | 2 +- uv.lock | 10 +++++----- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 742dd935..69d78ccf 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,6 +14,12 @@ updates: - dependency-name: pytest-bdd update-types: - version-update:semver-major + # pytest 9.2 stores node ids where pytest-xdist 3.8.0 does not write the loadgroup suffix, so + # --dist loadgroup silently ignores every group (pytest-dev/pytest-xdist#1407). Drop this once a + # pytest-xdist release after 3.8.0 is pinned; TestXdistSeesVipGroups fails if it comes off early. + - dependency-name: pytest + versions: + - ">=9.2" package-ecosystem: uv schedule: interval: weekly diff --git a/AGENTS.md b/AGENTS.md index 5a54b017..4db9a259 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -430,7 +430,7 @@ refactor(connect)!: rename client constructor parameters ## xdist groups -`--dist loadgroup` (the default in `pyproject.toml`) keeps every test with the same `xdist_group` on one worker. `plugin/selection.py` groups tests in a `connect/`, `workbench/` or `package_manager/` directory by that name and leaves everything else ungrouped; the Workbench conftest refines its group by IDE and module when concurrent logins are safe. pytest-xdist reads the groups in its own `pytest_collection_modifyitems` on each worker, so every hook that adds a group must be `@pytest.hookimpl(tryfirst=True)` -- a group added after xdist's hook is silently ignored. `selftests/test_workbench_parallel.py::TestXdistSeesVipGroups` runs real workers to guard this. +`--dist loadgroup` (the default in `pyproject.toml`) keeps every test with the same `xdist_group` on one worker. `plugin/selection.py` groups tests in a `connect/`, `workbench/` or `package_manager/` directory by that name and leaves everything else ungrouped; the Workbench conftest refines its group by IDE and module when concurrent logins are safe. pytest-xdist reads the groups in its own `pytest_collection_modifyitems` on each worker, so every hook that adds a group must be `@pytest.hookimpl(tryfirst=True)` -- a group added after xdist's hook is silently ignored. `selftests/test_workbench_parallel.py::TestXdistSeesVipGroups` runs real workers to guard this. pytest stays below 9.2 (pin in `pyproject.toml`, ignore in `.github/dependabot.yml`) until a pytest-xdist release after 3.8.0: pytest 9.2 moved node ids where xdist 3.8.0 writes the group, so every group is ignored (pytest-dev/pytest-xdist#1407). ## Pytest warning filters diff --git a/pyproject.toml b/pyproject.toml index 20feb57e..32a182df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "httpx>=0.27,<1", "playwright==1.65.0", # exact-pinned: shapes vip run output; see docs/development.md "pyotp~=2.9", - "pytest==9.2.0", # exact-pinned; at/above CVE-2025-71176 floor (9.0.3) + "pytest==9.1.1", # exact-pinned; at/above CVE-2025-71176 floor (9.0.3); <9.2 until pytest-xdist > 3.8.0 (pytest-dev/pytest-xdist#1407) "pytest-bdd==9.0.1", # exact-pinned "pytest-order==1.5.0", # exact-pinned "pytest-playwright==0.10.0", # exact-pinned diff --git a/uv.lock b/uv.lock index 2d85961e..0ae09c3c 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.15'", @@ -1875,7 +1875,7 @@ requires-dist = [ { name = "playwright", specifier = "==1.65.0" }, { name = "pygments", specifier = ">=2.20.0,<3" }, { name = "pyotp", specifier = "~=2.9" }, - { name = "pytest", specifier = "==9.2.0" }, + { name = "pytest", specifier = "==9.1.1" }, { name = "pytest-bdd", specifier = "==9.0.1" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.0" }, { name = "pytest-order", specifier = "==1.5.0" }, @@ -1998,7 +1998,7 @@ wheels = [ [[package]] name = "pytest" -version = "9.2.0" +version = "9.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -2007,9 +2007,9 @@ dependencies = [ { name = "pluggy" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/79/42/bb4641652e58dedeac2ec51abc1f8cae4fee4ec63899a0999016229a0f2a/pytest-9.2.0.tar.gz", hash = "sha256:97c4f3b1f3c7414aa36465e09b8220f1d03627993bd51b1282e193753c80557b", size = 1797555, upload-time = "2026-10-27T21:30:20.728Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/51/39/1f5d8c8877a0fa327db959c19341d4be61660de1ab6b7391f9eb805bcd67/pytest-9.2.0-py3-none-any.whl", hash = "sha256:07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588", size = 411776, upload-time = "2026-10-27T21:30:19.131Z" }, + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] [[package]]