From 205b7dce24fbc8d1b0d047dab16c5bbc7e47163b Mon Sep 17 00:00:00 2001 From: Ian Flores Siaca <18703558+ian-flores@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:45:15 -0800 Subject: [PATCH 1/4] ci(workbench): reapply parallel smoke with one user per worker Reapplies #840 (reverted in #841 after an early merge). --- .github/dependabot.yml | 6 ++ .github/workflows/workbench-smoke.yml | 30 +++++++-- AGENTS.md | 8 ++- pyproject.toml | 2 +- selftests/test_workbench_cleanup.py | 15 +++++ selftests/test_workbench_git_ops.py | 25 +++++++- selftests/test_workbench_parallel.py | 85 ++++++++++++++++++++++++- src/vip/plugin/selection.py | 17 +++-- src/vip_tests/workbench/cleanup.py | 4 +- src/vip_tests/workbench/conftest.py | 34 ++++++---- src/vip_tests/workbench/login.py | 6 +- src/vip_tests/workbench/naming.py | 23 +++++++ src/vip_tests/workbench/test_git_ops.py | 14 +++- uv.lock | 10 +-- 14 files changed, 236 insertions(+), 43 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 742dd935c..69d78ccff 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,6 +14,12 @@ updates: - dependency-name: pytest-bdd update-types: - version-update:semver-major + # pytest 9.2 stores node ids where pytest-xdist 3.8.0 does not write the loadgroup suffix, so + # --dist loadgroup silently ignores every group (pytest-dev/pytest-xdist#1407). Drop this once a + # pytest-xdist release after 3.8.0 is pinned; TestXdistSeesVipGroups fails if it comes off early. + - dependency-name: pytest + versions: + - ">=9.2" package-ecosystem: uv schedule: interval: weekly diff --git a/.github/workflows/workbench-smoke.yml b/.github/workflows/workbench-smoke.yml index 547cfbedf..eb8bc6fab 100644 --- a/.github/workflows/workbench-smoke.yml +++ b/.github/workflows/workbench-smoke.yml @@ -247,11 +247,31 @@ jobs: env: SUITE: ${{ inputs.suite || (github.event_name == 'schedule' && 'full' || 'gate') }} + # Workbench refuses a second sign-in by the same user within + # auth-sign-in-throttle-seconds (5 s by default) and answers it with + # "Temporary server error", so xdist workers sharing one account trip it. + # Give each worker its own account, named the way VIP derives it under + # VIP_TEST_USERNAME_PER_WORKER (-gw), with the same password. The + # password goes in on stdin so it never appears in a process list. + - name: Create per-worker test users + id: workers + env: + CID: ${{ steps.workbench.outputs.CONTAINER_ID }} + WB_USER: ${{ steps.workbench.outputs.WORKBENCH_USER }} + WB_PASSWORD: ${{ steps.workbench.outputs.WORKBENCH_PASSWORD }} + run: | + WORKERS=$(nproc) + for i in $(seq 0 $((WORKERS - 1))); do + docker exec "${CID}" useradd -m -s /bin/bash "${WB_USER}-gw${i}" + printf '%s:%s\n' "${WB_USER}-gw${i}" "${WB_PASSWORD}" | docker exec -i "${CID}" chpasswd + done + echo "Created ${WORKERS} per-worker users (${WB_USER}-gw0..$((WORKERS - 1)))" + echo "count=${WORKERS}" >> "$GITHUB_OUTPUT" + # Run the subset of tests that work against Docker Workbench - name: Run Workbench smoke tests - # -n 0 disables xdist (overriding -n auto from pyproject.toml) because - # the Docker Workbench returns "Temporary server error" when multiple - # workers attempt concurrent logins. + # -n matches the per-worker users created above, so every worker has an + # account to sign in as. # # No `-k` filter: the plugin already deselects scenarios for products # that are not enabled in vip.toml, so the previous `-k "workbench"` @@ -317,7 +337,7 @@ jobs: fi # shellcheck disable=SC2086 # word splitting is how the file list is passed uv run pytest $files \ - -v -rs -n 0 \ + -v -rs -n "${WORKERS}" \ --vip-config=vip.toml \ --junitxml=smoke-results.xml env: @@ -325,6 +345,8 @@ jobs: # slow suites are still exercised daily without taxing the merge path. SUITE: ${{ inputs.suite || (github.event_name == 'schedule' && 'full' || 'gate') }} VIP_TEST_PASSWORD: ${{ steps.workbench.outputs.WORKBENCH_PASSWORD }} + VIP_TEST_USERNAME_PER_WORKER: "1" + WORKERS: ${{ steps.workers.outputs.count }} # Tolerant, unretried upload -- see connect-smoke.yml's "Upload test # results" step for the full rationale, including why no safe retry diff --git a/AGENTS.md b/AGENTS.md index 730bd1650..4b86455aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -208,7 +208,7 @@ Key principles: | `src/vip/install/plan.py` | Pure `build_install_plan` / `build_uninstall_plan` builders | | `src/vip/install/runner.py` | Plan executor: dry-run formatting + execute (system packages, Playwright, manifest writes) | | `src/vip_tests/conftest.py` | Directory-scoped warning filter (kept out of the global plugin deliberately) plus the three autouse Connect content-cleanup fixtures — see that file's docstring for why those stay directory-scoped instead of moving to `src/vip/fixtures.py` | -| `src/vip_tests/workbench/naming.py` | Worker-scoped Workbench session names: `current_worker_id`, `vip_session_prefix`, `capacity_session_prefix`, `k8s_session_prefix`, `unique_session_name` — see "Workbench session ownership" below | +| `src/vip_tests/workbench/naming.py` | Worker-scoped Workbench session names: `current_worker_id`, `vip_session_prefix`, `capacity_session_prefix`, `k8s_session_prefix`, `unique_session_name` — see "Workbench session ownership" below; also `worker_username`, the per-worker account behind `VIP_TEST_USERNAME_PER_WORKER` | | `src/vip_tests/workbench/capacity.py` | Resource-profile detection and capping shared by `test_session_capacity.py` and `test_session_capacity_k8s.py` (`ResourceProfileDisabledError`) | | `src/vip_tests/workbench/cleanup.py` | Cookie-based Workbench session cleanup, the API-token sweep used when there are no cookies (`--api-auth`), and the autouse cleanup fixtures. `@api_auth` scenarios get the token sweep and never request `page` | | `src/vip_tests/workbench/sessions.py` | Session-state waits, failure/timeout messages, and the unproven-skip helpers for Workbench scenarios, plus `join_session` — the one way to click a session's join link (waits for Active, retries the click once with a warning, fails if the browser never enters the session). Don't hand-roll `a[title='join ...']` clicks in step files | @@ -289,6 +289,7 @@ Configuration is in `vip.toml` (see `vip.toml.example` for the template). Secret - `VIP_WORKBENCH_API_KEY` — Workbench API token (`Authorization: Bearer`). Under `--api-auth` it launches IDE sessions through the Workbench API and lets cleanup run without a browser; use a super-admin token (see `vip.toml.example`) - `VIP_TEST_USERNAME` - `VIP_TEST_PASSWORD` +- `VIP_TEST_USERNAME_PER_WORKER` — set to `1` to give each xdist worker its own Workbench account, `-gw`, sharing the configured password. Workbench refuses a repeat sign-in by one user within `auth-sign-in-throttle-seconds` (5 s by default), which parallel workers sharing an account hit as "Temporary server error". The accounts must already exist; `workbench-smoke.yml` creates them. Serial runs ignore it - `VIP_TEST_TOTP_SECRET` — optional base32 TOTP seed used by `--headless-auth` to auto-fill MFA codes for a dedicated test service account. **Equivalent to bypassing 2FA — never use a personal account's seed.** The plugin loads config via `--vip-config` or defaults to `./vip.toml`. If no config file exists, all product tests are skipped. @@ -339,6 +340,7 @@ Every render also produces `_output/vip-report.pdf` from `report/vip-report.qmd` - **Smoke workflows** (`connect-smoke.yml`, `workbench-smoke.yml`, `packagemanager-smoke.yml`, `mock-idp-e2e.yml`) -- run the product suites against real containers. On PR/push each tests a single latest version (change-gated via a `changes` paths-filter job); on `schedule` (nightly, staggered hourly) and `workflow_dispatch` a `set-matrix` job fans each out across the product version support window (current + 2 back). There are no pinned tags to bump: every `set-matrix` job calls `recent-production-versions.yml`, which resolves versions from the product's own GHCR tag list at run time. Each workflow's `*-status` aggregation job is scope-aware: it passes when the suite was legitimately out of scope (the PR's paths didn't match) but **fails** when the suite was in scope (`changes.relevant == 'true'`) yet did not succeed -- so a path-gated skip, an excluded actor, or a missing license secret cannot report a green required check without the suite having run. The Connect, Workbench, and Package Manager `*-status` jobs are required merge checks. `connect-smoke.yml` also starts a Mailpit container and points Connect's SMTP settings at it (through with-connect's `env:` input, using `CONNECT_
_` overrides), gives the bootstrap admin an email address, and enables `[email]`, so `test_email` performs a real send instead of skipping. A follow-up step fails the job if Mailpit received nothing. `mock-idp-e2e.yml` is structured the same way so `Mock-IdP E2E Status` can be promoted to a required check via a separate admin action. It seeds a Keycloak realm and has no Entra lane: Entra ID is a Microsoft-hosted login app that can't be stood up in a container, so the `entra` strategy in `src/vip/idp.py` is validated against a real tenant instead of mock-IdP CI. That test account must also be excluded from Conditional Access device-compliance policies, since headless auth runs in an unmanaged browser Entra's CA would otherwise block. + `workbench-smoke.yml` runs the suite in parallel with one Workbench account per xdist worker: a step creates `-gw0..N-1` in the container (N = the runner's `nproc`, also passed as `-n`) and the test step sets `VIP_TEST_USERNAME_PER_WORKER=1`. One shared account fails under xdist because Workbench throttles repeat sign-ins by the same user (`auth-sign-in-throttle-seconds`, 5 s by default). With the flag set, the Workbench conftest groups scenarios by IDE and by module, as it does under a shared auth session; without it, every Workbench scenario stays in one `workbench` xdist group, so a single account signs in from one worker. `workbench-smoke.yml` additionally splits into two tiers via a `suite` value: PR/push runs `gate` (the fast subset), the nightly schedule runs `full` (every Workbench file), and `workflow_dispatch` can pick either. The split is based on a measured run rather than taste — `full` buys 8 more real passes for ~523s more test time, which is worth a nightly but not a merge gate. Skip reasons do **not** appear in the log even with `-rs`, because VIP's plugin owns the terminal reporter; read them from `` in the uploaded `smoke-results.xml`. `workbench-smoke.yml` also has an `API-auth smoke test` job that runs `vip verify --api-auth` over `workbench/test_ide_launch_api.py` against the same kind of container with no browser installed. It appends `workbench-api-enabled=1` and `workbench-api-super-admin-enabled=1` to `rserver.conf`, restarts the container with `docker restart` (the image's supervisor stops the container if rserver alone exits), mints a super-admin token with `rstudio-server generate-api-token ... --token-env`, and hands it to VIP as `VIP_WORKBENCH_API_KEY` step env, never in `vip.toml`. It then reruns the RStudio scenario with a `user` token owned by the test user, the other documented launch path. A final step fails the job unless the RStudio scenario passed, rather than skipped, under both tokens. `Workbench Smoke Tests Status` aggregates it with the main job. The three cross-cutting suites (`cross_product/test_resources`, `security/test_auth_policy`, `config_hygiene/test_secrets`) run in all three product workflows. `test_secrets` asserts no plaintext `api_key`/`password` in the generated `vip.toml`, so credentials must be passed to pytest as step `env` (`VIP_CONNECT_API_KEY`, `VIP_TEST_PASSWORD`) and never written into the config file. @@ -426,6 +428,10 @@ refactor(connect)!: rename client constructor parameters - Missing the colon and space after the type (e.g. `feat add feature` — must be `feat: add feature`). - Using a PR title that is not conventional when the branch will be squash-merged. +## xdist groups + +`--dist loadgroup` (the default in `pyproject.toml`) keeps every test with the same `xdist_group` on one worker. `plugin/selection.py` groups tests in a `connect/`, `workbench/` or `package_manager/` directory by that name and leaves everything else ungrouped; the Workbench conftest refines its group by IDE and module when concurrent logins are safe. pytest-xdist reads the groups in its own `pytest_collection_modifyitems` on each worker, so every hook that adds a group must be `@pytest.hookimpl(tryfirst=True)` -- a group added after xdist's hook is silently ignored. `selftests/test_workbench_parallel.py::TestXdistSeesVipGroups` runs real workers to guard this. pytest stays below 9.2 (pin in `pyproject.toml`, ignore in `.github/dependabot.yml`) until a pytest-xdist release after 3.8.0: pytest 9.2 moved node ids where xdist 3.8.0 writes the group, so every group is ignored (pytest-dev/pytest-xdist#1407). + ## Pytest warning filters Register warning filters in `src/vip/plugin/configure.py::pytest_configure` (via `config.addinivalue_line("filterwarnings", ...)`), not in `pyproject.toml`'s `[tool.pytest.ini_options]`. Filters in `pyproject.toml` only apply when pytest runs from this repo's rootdir -- users who install vip into another project pick up the plugin but not the config, so the warnings reappear there. Keeping the filters in the plugin means they travel with the installed package. diff --git a/pyproject.toml b/pyproject.toml index 20feb57eb..32a182df2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "httpx>=0.27,<1", "playwright==1.65.0", # exact-pinned: shapes vip run output; see docs/development.md "pyotp~=2.9", - "pytest==9.2.0", # exact-pinned; at/above CVE-2025-71176 floor (9.0.3) + "pytest==9.1.1", # exact-pinned; at/above CVE-2025-71176 floor (9.0.3); <9.2 until pytest-xdist > 3.8.0 (pytest-dev/pytest-xdist#1407) "pytest-bdd==9.0.1", # exact-pinned "pytest-order==1.5.0", # exact-pinned "pytest-playwright==0.10.0", # exact-pinned diff --git a/selftests/test_workbench_cleanup.py b/selftests/test_workbench_cleanup.py index 58bc642b0..bbd584647 100644 --- a/selftests/test_workbench_cleanup.py +++ b/selftests/test_workbench_cleanup.py @@ -1201,3 +1201,18 @@ def test_token_sweep_logs_instead_of_raising_when_the_api_refuses(caplog): with caplog.at_level(logging.WARNING, logger="vip_tests.workbench.cleanup"): assert cleanup._quit_vip_sessions_via_token(client, _token_config(), owner="main") == 0 assert "could not sweep VIP Workbench sessions" in caplog.text + + +def test_token_sweep_targets_this_workers_own_account(monkeypatch): + from types import SimpleNamespace + + from vip_tests.workbench import cleanup + + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw1") + monkeypatch.setenv("VIP_TEST_USERNAME_PER_WORKER", "1") + users: list[str] = [] + client = SimpleNamespace( + quit_vip_sessions_via_api=lambda user, *, owner: users.append(user) or 0 + ) + cleanup._quit_vip_sessions_via_token(client, _token_config(), owner="gw1") + assert users == ["alice-gw1"] diff --git a/selftests/test_workbench_git_ops.py b/selftests/test_workbench_git_ops.py index 619d0880f..9abd9ccca 100644 --- a/selftests/test_workbench_git_ops.py +++ b/selftests/test_workbench_git_ops.py @@ -40,20 +40,41 @@ def fake_terminal_run(page, cmd, timeout=30_000, *, readback_lang="r"): return "" monkeypatch.setattr(git_ops, "terminal_run", fake_terminal_run) + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw3") git_cfg = _FakeGitCfg("https://github.com/example/jumpstart_examples.git") git_session_ctx: dict = {} clone_dir = git_ops._do_clone(object(), git_session_ctx, git_cfg, readback_lang="r") - assert clone_dir == "/tmp/jumpstart_examples" + assert clone_dir == "/tmp/vip_git_gw3/jumpstart_examples" assert git_session_ctx["clone_dir"] == clone_dir - rm_index = captured["cmd"].find("rm -rf /tmp/jumpstart_examples") + rm_index = captured["cmd"].find("rm -rf /tmp/vip_git_gw3/jumpstart_examples") clone_index = captured["cmd"].find("git clone") assert rm_index != -1, f"expected rm -rf of clone dir in command: {captured['cmd']!r}" assert rm_index < clone_index, "rm -rf must run before git clone" +def test_do_clone_gives_each_worker_its_own_directory(monkeypatch): + # Parallel workers (often different users) must not delete or clone over each other's checkout. + cmds: list[str] = [] + monkeypatch.setattr( + git_ops, + "terminal_run", + lambda page, cmd, timeout=30_000, *, readback_lang="r": cmds.append(cmd), + ) + git_cfg = _FakeGitCfg("https://github.com/example/repo.git") + + dirs = [] + for worker in ("gw0", "gw1"): + monkeypatch.setenv("PYTEST_XDIST_WORKER", worker) + dirs.append(git_ops._do_clone(object(), {}, git_cfg, readback_lang="r")) + + assert dirs == ["/tmp/vip_git_gw0/repo", "/tmp/vip_git_gw1/repo"] + # The per-worker parent does not exist on a fresh server, so it is created before cd. + assert cmds[0].index("mkdir -p /tmp/vip_git_gw0") < cmds[0].index("cd /tmp/vip_git_gw0") + + def test_do_clone_uses_custom_workdir(monkeypatch): captured: dict = {} diff --git a/selftests/test_workbench_parallel.py b/selftests/test_workbench_parallel.py index d0009ee5e..8a3a1f1be 100644 --- a/selftests/test_workbench_parallel.py +++ b/selftests/test_workbench_parallel.py @@ -221,11 +221,25 @@ class TestCollectionHook: def _wb_dir(self): return Path(wb.__file__).parent - def test_no_auth_session_is_a_noop(self): + def test_no_auth_session_is_a_noop(self, monkeypatch): + monkeypatch.delenv("VIP_TEST_USERNAME_PER_WORKER", raising=False) item = _FakeItem(self._wb_dir() / "test_packages.py", []) wb.pytest_collection_modifyitems(_FakeConfig(None), [item]) assert item.added == [] # untouched under password / no-auth + def test_per_worker_users_group_without_an_auth_session(self, monkeypatch): + # Password auth with one account per worker can log in concurrently, so it spreads too. + monkeypatch.setenv("VIP_TEST_USERNAME_PER_WORKER", "1") + launch = _FakeItem(self._wb_dir() / "test_ide_launch.py", ["workbench", "vscode"]) + extensions = _FakeItem(self._wb_dir() / "test_ide_extensions.py", ["workbench", "vscode"]) + packages = _FakeItem(self._wb_dir() / "test_packages.py", ["workbench"]) + wb.pytest_collection_modifyitems(_FakeConfig(None), [launch, extensions, packages]) + assert launch.added == [("xdist_group", ("workbench_ide_vscode",))] + assert ( + extensions.added == launch.added + ) # an IDE's launch and extension tests share a worker + assert packages.added == [("xdist_group", ("workbench_packages",))] + def test_ide_item_grouped_by_ide(self): item = _FakeItem(self._wb_dir() / "test_ide_launch.py", ["workbench", "rstudio"]) wb.pytest_collection_modifyitems(_FakeConfig(object()), [item]) @@ -532,8 +546,8 @@ class TestRealMarkerMechanics: """ def test_regroup_wins_via_get_closest_marker_and_leaves_no_duplicate(self, pytester): - # Disable the vip plugin for the nested collection: its own _assign_xdist_group - # would inject a "general" group and obscure the mechanic under test. + # Disable the vip plugin for the nested collection so its own _assign_xdist_group + # cannot add a group and obscure the mechanic under test. modcol = pytester.getmodulecol("def test_x(): pass", configargs=["-p", "no:vip"]) (item,) = pytester.genitems([modcol]) # Simulate a pre-existing group, then apply exactly the hook's two operations. @@ -548,3 +562,68 @@ def test_regroup_wins_via_get_closest_marker_and_leaves_no_duplicate(self, pytes # And no leftover duplicate, or xdist would glue both names into one composite group. groups = [m.args[0] for m in item.iter_markers("xdist_group")] assert groups == ["workbench_packages"] + + +class TestPerWorkerUsername: + """Workbench throttles repeat sign-ins by one user, so parallel CI gives each worker its own.""" + + def test_opt_in_suffixes_the_worker_id(self, monkeypatch): + from vip_tests.workbench.naming import worker_username + + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw2") + monkeypatch.setenv("VIP_TEST_USERNAME_PER_WORKER", "1") + assert worker_username("testuser") == "testuser-gw2" + + @pytest.mark.parametrize( + ("worker", "flag", "base"), + [ + ("gw2", None, "testuser"), # not opted in: customers keep their one account + ("gw2", "0", "testuser"), + (None, "1", "testuser"), # serial run: no worker to suffix + ("gw2", "1", ""), # no configured user: nothing to derive from + ], + ids=["no-flag", "flag-off", "serial", "no-base"], + ) + def test_base_username_is_kept_otherwise(self, monkeypatch, worker, flag, base): + from vip_tests.workbench.naming import worker_username + + for name, value in ( + ("PYTEST_XDIST_WORKER", worker), + ("VIP_TEST_USERNAME_PER_WORKER", flag), + ): + if value is None: + monkeypatch.delenv(name, raising=False) + else: + monkeypatch.setenv(name, value) + assert worker_username(base) == base + + def test_login_retry_waits_out_the_sign_in_throttle(self): + import inspect + + # Workbench's auth-sign-in-throttle-seconds defaults to 5; a shorter wait retries into it. + default = inspect.signature(login.workbench_login).parameters["retry_delay"].default + assert default > 5 + + +class TestXdistSeesVipGroups: + """xdist reads ``xdist_group`` in its own collection hook, so VIP's must run first.""" + + def test_directory_group_keeps_its_tests_on_one_worker(self, pytester): + pytester.makefile(".toml", vip='[general]\ndeployment_name = "Selftest"') + out = pytester.path / "workers.txt" + pytester.mkdir("workbench") + pytester.path.joinpath("workbench", "test_grouped.py").write_text( + "import os, pytest\n" + "@pytest.mark.parametrize('i', range(6))\n" + "def test_where(i, request):\n" + f" with open({str(out)!r}, 'a', encoding='utf-8') as f:\n" + " worker = os.environ['PYTEST_XDIST_WORKER']\n" + " f.write(worker + ' ' + request.node.nodeid + '\\n')\n", + encoding="utf-8", + ) + result = pytester.runpytest("--vip-config=vip.toml", "-n", "3", "--dist", "loadgroup") + result.assert_outcomes(passed=6) + + rows = [line.split(" ", 1) for line in out.read_text(encoding="utf-8").splitlines()] + assert all(nodeid.endswith("@workbench") for _, nodeid in rows), rows + assert len({worker for worker, _ in rows}) == 1, rows diff --git a/src/vip/plugin/selection.py b/src/vip/plugin/selection.py index 514c0031c..38092676d 100644 --- a/src/vip/plugin/selection.py +++ b/src/vip/plugin/selection.py @@ -20,12 +20,16 @@ from vip.version import ProductVersion +@pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems( config: pytest.Config, items: list[pytest.Item], ) -> None: """Deselect tests whose product is not configured, skip tests whose version requirement is not met, and ensure prerequisites run first. + + ``tryfirst`` so the ``xdist_group`` markers exist before pytest-xdist's own collection + hook reads them on each worker; after it, ``--dist loadgroup`` never sees a group. """ vip_cfg: VIPConfig = config.stash[_vip_config_key] no_auth = config.getoption("--no-auth", default=False) @@ -72,17 +76,18 @@ def _assign_xdist_group(item: pytest.Item) -> None: """Assign an ``xdist_group`` marker so each product runs on its own worker. Respects any existing ``xdist_group`` marker (set via conftest.py - ``pytestmark`` or per-test decorators). Otherwise, tests under - ``tests/connect/``, ``tests/workbench/``, or ``tests/package_manager/`` - are grouped by directory name, and everything else (prerequisites, - cross_product, performance, security) lands in a shared ``general`` group. + ``pytestmark`` or per-test decorators). Otherwise, tests under a + ``connect/``, ``workbench/``, or ``package_manager/`` directory are grouped + by directory name. Everything else (prerequisites, cross_product, + security, extensions) stays ungrouped, so ``--dist loadgroup`` spreads it + across workers. """ if item.get_closest_marker("xdist_group") is not None: return fspath = getattr(item, "path", None) or Path() dir_name = fspath.parent.name - group = dir_name if dir_name in _PRODUCT_DIRS else "general" - item.add_marker(pytest.mark.xdist_group(group)) + if dir_name in _PRODUCT_DIRS: + item.add_marker(pytest.mark.xdist_group(dir_name)) # Every marker that names a product *item* needs: ``@connect`` and ``@requires_connect`` alike. diff --git a/src/vip_tests/workbench/cleanup.py b/src/vip_tests/workbench/cleanup.py index cd0b2934a..f12fd2d1a 100644 --- a/src/vip_tests/workbench/cleanup.py +++ b/src/vip_tests/workbench/cleanup.py @@ -16,7 +16,7 @@ from vip.workbench_ui import ( quit_vip_sessions_via_ui as _quit_vip_sessions_via_ui, ) -from vip_tests.workbench.naming import current_worker_id +from vip_tests.workbench.naming import current_worker_id, worker_username logger = logging.getLogger(__name__) @@ -128,7 +128,7 @@ def _quit_vip_sessions_via_token( """ if not vip_config.workbench.api_key: return 0 - username = vip_config.auth.username + username = worker_username(vip_config.auth.username) if not username: return 0 try: diff --git a/src/vip_tests/workbench/conftest.py b/src/vip_tests/workbench/conftest.py index 52e8c7ee7..f69579848 100644 --- a/src/vip_tests/workbench/conftest.py +++ b/src/vip_tests/workbench/conftest.py @@ -21,6 +21,7 @@ from playwright.sync_api import Page from pytest_bdd import given +from vip.config import VIPConfig from vip.plugin import _auth_session_key # Re-exported for selftests/test_workbench_cleanup.py, which imports it from @@ -57,8 +58,10 @@ capacity_session_prefix, current_worker_id, k8s_session_prefix, + per_worker_users_enabled, unique_session_name, vip_session_prefix, + worker_username, ) from vip_tests.workbench.sessions import ( _navigated_into_session, @@ -180,27 +183,24 @@ def _workbench_group_name(ide_markers: set[str], module_stem: str) -> str: return f"workbench_{stem}" +@pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems( config: pytest.Config, items: list[pytest.Item], ) -> None: - """Group Workbench tests for parallel execution when a shared auth session is active. + """Group Workbench tests for parallel execution when the run can log in concurrently. - Under --interactive-auth / --headless-auth all Workbench tests authenticate as the same - shared account. Rather than pin them all to one worker (the old serial workaround), we - group them so LoadGroupScheduling spreads them across workers: IDE-launch scenarios by + That is a shared auth session (--interactive-auth / --headless-auth, whose concurrent SSO + round-trips :func:`oidc_login_lock` serializes) or one account per worker + (``VIP_TEST_USERNAME_PER_WORKER``, which sidesteps Workbench's per-user sign-in + throttle). LoadGroupScheduling then spreads the groups across workers: IDE scenarios by IDE (``workbench_ide_``), everything else by feature module (``workbench_``). - The simultaneous-login storm this used to cause is prevented by the cross-worker login - lock in :func:`workbench_login` (see :func:`oidc_login_lock`), not by serialization. - - Password / no-auth runs are left untouched: they hit the early return below, so their - Workbench items keep the default ``workbench`` group that ``plugin/selection.py``'s - :func:`_assign_xdist_group` directory fallback assigns (the module-level ``pytestmark`` - at the top of this file does not propagate to sibling test modules, so it assigns - nothing here). + + Any other password / no-auth run keeps the single ``workbench`` group that + ``plugin/selection.py``'s :func:`_assign_xdist_group` assigns, so one worker signs in as + the one account. ``tryfirst`` because xdist reads the groups in its own collection hook. """ - if config.stash.get(_auth_session_key, None) is None: - # No shared auth session — password auth or no auth. Keep default parallel behavior. + if config.stash.get(_auth_session_key, None) is None and not per_worker_users_enabled(): return workbench_dir = Path(__file__).parent @@ -290,6 +290,12 @@ def extract_repo_urls(output: str) -> list[str]: return re.findall(r"https?://[^\s<>\"']+", output, re.IGNORECASE) +@pytest.fixture(scope="session") +def test_username(vip_config: VIPConfig) -> str: + """The Workbench test account, one per xdist worker under ``VIP_TEST_USERNAME_PER_WORKER``.""" + return worker_username(vip_config.auth.username) + + @pytest.fixture def wb_login( page: Page, diff --git a/src/vip_tests/workbench/login.py b/src/vip_tests/workbench/login.py index 3cf2a64a7..7b36e2559 100644 --- a/src/vip_tests/workbench/login.py +++ b/src/vip_tests/workbench/login.py @@ -224,7 +224,7 @@ def workbench_login( auth_mode: str = "none", workbench_auth_error: str | None = None, max_retries: int = 3, - retry_delay: float = 2.0, + retry_delay: float = 6.0, ) -> None: """Navigate to Workbench homepage, logging in only if required. @@ -248,7 +248,9 @@ def workbench_login( establish a Workbench session, if known. Quoted in the skip message so users see the real cause instead of a guess. max_retries: Max login attempts on transient errors (default 3) - retry_delay: Seconds to wait between retries (default 2.0) + retry_delay: Seconds to wait between retries (default 6.0). Longer than + Workbench's per-user sign-in throttle (5 s by default), which answers a + too-quick repeat sign-in with "Temporary server error". Raises: pytest.skip: For non-password auth without a pre-loaded auth session, diff --git a/src/vip_tests/workbench/naming.py b/src/vip_tests/workbench/naming.py index 666a4846f..5b1e1b498 100644 --- a/src/vip_tests/workbench/naming.py +++ b/src/vip_tests/workbench/naming.py @@ -20,6 +20,29 @@ def current_worker_id() -> str: return os.environ.get("PYTEST_XDIST_WORKER", "main") +# Opt-in switch for one Workbench account per xdist worker; the deployment must already have them. +PER_WORKER_USERS_ENV = "VIP_TEST_USERNAME_PER_WORKER" + + +def per_worker_users_enabled() -> bool: + """True when this run gives each xdist worker its own Workbench account.""" + return os.environ.get(PER_WORKER_USERS_ENV) == "1" + + +def worker_username(base: str) -> str: + """Return the Workbench test account this worker signs in as. + + Workbench refuses a second sign-in by the same user within + ``auth-sign-in-throttle-seconds`` (default 5 s), so parallel workers sharing one + account fail with "Temporary server error". With ``VIP_TEST_USERNAME_PER_WORKER=1`` + each worker uses ``-`` (``testuser-gw0``); serial runs keep *base*. + """ + worker = os.environ.get("PYTEST_XDIST_WORKER") + if not base or not worker or not per_worker_users_enabled(): + return base + return f"{base}-{worker}" + + def vip_session_prefix(kind: str) -> str: """Build a ``_vip____`` session-name prefix. diff --git a/src/vip_tests/workbench/test_git_ops.py b/src/vip_tests/workbench/test_git_ops.py index 183c6ce28..6ba7139bd 100644 --- a/src/vip_tests/workbench/test_git_ops.py +++ b/src/vip_tests/workbench/test_git_ops.py @@ -36,6 +36,7 @@ workbench_login, ) from vip_tests.workbench.exec import ExecError, ensure_positron_console, file_exists, terminal_run +from vip_tests.workbench.naming import current_worker_id from vip_tests.workbench.pages import ( Homepage, NewSessionDialog, @@ -377,16 +378,23 @@ def _do_clone( git_cfg, readback_lang: str, *, - workdir: str = "/tmp", + workdir: str | None = None, ) -> str: - """Run ``git clone`` in the IDE terminal and return the cloned directory path.""" + """Run ``git clone`` in the IDE terminal and return the cloned directory path. + + Defaults to ``/tmp/vip_git_`` so parallel workers, which may be different + users, never remove or clone over each other's checkout. + """ + if workdir is None: + workdir = f"/tmp/vip_git_{current_worker_id()}" auth_url = _inject_token_into_url(git_cfg.clone_url, git_cfg.token) repo_dir = _repo_dir_from_url(git_cfg.clone_url) clone_dir = f"{workdir}/{repo_dir}" + q_workdir = shlex.quote(workdir) terminal_run( page, - f"rm -rf {shlex.quote(clone_dir)} && cd {shlex.quote(workdir)} && " + f"rm -rf {shlex.quote(clone_dir)} && mkdir -p {q_workdir} && cd {q_workdir} && " f"GIT_TERMINAL_PROMPT=0 git clone {shlex.quote(auth_url)}", timeout=_TIMEOUT_GIT_NETWORK, readback_lang=readback_lang, diff --git a/uv.lock b/uv.lock index 2d85961e1..0ae09c3c2 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.15'", @@ -1875,7 +1875,7 @@ requires-dist = [ { name = "playwright", specifier = "==1.65.0" }, { name = "pygments", specifier = ">=2.20.0,<3" }, { name = "pyotp", specifier = "~=2.9" }, - { name = "pytest", specifier = "==9.2.0" }, + { name = "pytest", specifier = "==9.1.1" }, { name = "pytest-bdd", specifier = "==9.0.1" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.0" }, { name = "pytest-order", specifier = "==1.5.0" }, @@ -1998,7 +1998,7 @@ wheels = [ [[package]] name = "pytest" -version = "9.2.0" +version = "9.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -2007,9 +2007,9 @@ dependencies = [ { name = "pluggy" }, { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/79/42/bb4641652e58dedeac2ec51abc1f8cae4fee4ec63899a0999016229a0f2a/pytest-9.2.0.tar.gz", hash = "sha256:97c4f3b1f3c7414aa36465e09b8220f1d03627993bd51b1282e193753c80557b", size = 1797555, upload-time = "2026-10-27T21:30:20.728Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/51/39/1f5d8c8877a0fa327db959c19341d4be61660de1ab6b7391f9eb805bcd67/pytest-9.2.0-py3-none-any.whl", hash = "sha256:07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588", size = 411776, upload-time = "2026-10-27T21:30:19.131Z" }, + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] [[package]] From 123c82222921d9215502a8775587e1868ae01468 Mon Sep 17 00:00:00 2001 From: Ian Flores Siaca <18703558+ian-flores@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:39:12 -0800 Subject: [PATCH 2/4] fix(workbench): queue the sign-out scenario last under xdist xdist dispatches whole groups, queued by each group's first test, so sign-out sharing workbench_auth with the early login scenario ran at the start of a shared-auth run and ended the session other workers were using. Give it a group of its own so its order(95) queues it last. --- AGENTS.md | 2 +- selftests/test_workbench_ordering.py | 68 ++++++++++++++++++++++++++++ src/vip_tests/workbench/conftest.py | 9 +++- 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4b86455aa..a3feb75a7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -442,7 +442,7 @@ Register warning filters in `src/vip/plugin/configure.py::pytest_configure` (via - Adding a ruff version anywhere other than the `dev` extra in `pyproject.toml` -- `ci.yml` and `.pre-commit-config.yaml` must keep reading the pin, not carrying their own copy. - Adding product SDK imports (use httpx directly). - Writing tests that modify or delete existing customer content. -- Adding a Workbench scenario that ends the shared auth session (sign-out, session revocation, password change) without ordering it last *and* restoring the session afterwards. Under `--interactive-auth` / `--headless-auth` every Workbench scenario shares one account, so ending that session breaks every scenario still running on other xdist workers, plus the cached auth session on disk. `test_workbench_signout` is the worked example. +- Adding a Workbench scenario that ends the shared auth session (sign-out, session revocation, password change) without ordering it last *and* restoring the session afterwards. Under `--interactive-auth` / `--headless-auth` every Workbench scenario shares one account, so ending that session breaks every scenario still running on other xdist workers, plus the cached auth session on disk. `test_workbench_signout` is the worked example; it also needs its own xdist group (`_OWN_GROUP_SCENARIOS` in the Workbench conftest), because xdist dispatches whole groups and a group that starts early would run it early. - Creating `.py` step files without a matching `.feature` file (or vice versa). - Forgetting the `@connect`/`@workbench`/`@package_manager` tag in feature files (breaks auto-skip). - Reading pytest-bdd internals anywhere other than `_scenario_for` and `_scenario_given_steps` in `src/vip/plugin/results.py`. pytest-bdd 9 has no public collection-time scenario API, so those two helpers are the only places allowed to touch its registry, its scenario templates or its `_pytest_bdd_example` parameter. diff --git a/selftests/test_workbench_ordering.py b/selftests/test_workbench_ordering.py index 1a2a3429d..2ba4a2156 100644 --- a/selftests/test_workbench_ordering.py +++ b/selftests/test_workbench_ordering.py @@ -20,6 +20,7 @@ from __future__ import annotations +import os import re import subprocess import sys @@ -188,6 +189,73 @@ def test_login_collects_before_signout(tmp_path: Path) -> None: assert login_idx < signout_idx, f"login must collect before sign-out: {nodeids}" +_GROUP_DUMP_PLUGIN = """ +def pytest_collection_finish(session): + for item in session.items: + groups = [m.args[0] for m in item.iter_markers("xdist_group")] + print("VIPGROUP", item.nodeid, "_".join(sorted(groups))) +""" + + +def _xdist_group_queue(tmp_path: Path) -> list[tuple[str, list[str]]]: + """Return Workbench groups in the order ``--dist loadgroup`` queues them, with their tests. + + xdist keys each group on its first test in collection order and hands groups out in that + order, so this is the dispatch order. Per-worker users switch the IDE/module grouping on. + """ + (tmp_path / "vip_group_dump.py").write_text(_GROUP_DUMP_PLUGIN, encoding="utf-8") + config_path = tmp_path / "vip.toml" + config_path.write_text( + '[workbench]\nurl = "https://workbench.example.com"\n\n' + '[connect]\nurl = "https://connect.example.com"\n', + encoding="utf-8", + ) + env = { + **os.environ, + "VIP_TEST_USERNAME_PER_WORKER": "1", + "PYTHONPATH": os.pathsep.join(filter(None, [str(tmp_path), os.environ.get("PYTHONPATH")])), + } + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + str(_WORKBENCH_DIR), + "--collect-only", + "-q", + "-p", + "vip_group_dump", + f"--vip-config={config_path}", + ], + cwd=_REPO_ROOT, + env=env, + capture_output=True, + encoding="utf-8", + errors="replace", + check=False, + ) + assert result.returncode == 0, f"collection failed:\n{result.stdout}\n{result.stderr}" + queue: dict[str, list[str]] = {} + for line in result.stdout.splitlines(): + if line.startswith("VIPGROUP "): + _, nodeid, group = line.split(" ", 2) + queue.setdefault(group, []).append(_function_of(nodeid)) + assert queue, f"no groups parsed from collection output:\n{result.stdout}" + return list(queue.items()) + + +def test_signout_is_the_last_xdist_group_and_runs_alone(tmp_path: Path) -> None: + """Sign-out must not ride in an earlier group, or xdist hands it out at the start of the run. + + Under --interactive-auth / --headless-auth it would then end the shared session while + other workers are mid-test, which ``order(95)`` alone cannot prevent. + """ + queue = _xdist_group_queue(tmp_path) + _, last_tests = queue[-1] + assert last_tests == ["test_workbench_signout"], f"sign-out must be queued last, alone: {queue}" + assert all("test_workbench_signout" not in tests for _, tests in queue[:-1]), queue + + def test_ide_marker_selects_matching_launch_and_extensions_scenarios(tmp_path: Path) -> None: """#592: each IDE marker must select exactly its launch scenario plus its (if any) extensions scenario -- nothing else. diff --git a/src/vip_tests/workbench/conftest.py b/src/vip_tests/workbench/conftest.py index f69579848..993c05cad 100644 --- a/src/vip_tests/workbench/conftest.py +++ b/src/vip_tests/workbench/conftest.py @@ -183,6 +183,12 @@ def _workbench_group_name(ide_markers: set[str], module_stem: str) -> str: return f"workbench_{stem}" +# Scenarios that end the session other workers share run in a group of their own. xdist queues +# groups by their first test in collection order, so sharing a group with an early scenario would +# hand them out at the start of the run; alone, sign-out's order(95) queues it last. +_OWN_GROUP_SCENARIOS = {"test_workbench_signout": "workbench_signout"} + + @pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems( config: pytest.Config, @@ -208,8 +214,9 @@ def pytest_collection_modifyitems( item_path = getattr(item, "path", None) if item_path is None or not item_path.is_relative_to(workbench_dir): continue + own_group = _OWN_GROUP_SCENARIOS.get(getattr(item, "originalname", "")) ide_markers = {m.name for m in item.iter_markers()} & set(_IDE_MARKERS) - group = _workbench_group_name(ide_markers, item_path.stem) + group = own_group or _workbench_group_name(ide_markers, item_path.stem) # Strip any pre-existing xdist_group marker before adding the hybrid group. No # per-test xdist_group marker exists today, so this is a defensive guard: xdist # concatenates *all* xdist_group marks on an item (via iter_markers), it does not From 2e5e5e0ac4e6ced5d37b35eea38f8a1862ea0d88 Mon Sep 17 00:00:00 2001 From: Ian Flores Siaca <18703558+ian-flores@users.noreply.github.com> Date: Thu, 19 Nov 2026 09:02:32 -0800 Subject: [PATCH 3/4] fix(workbench): hold sign-out until other xdist workers finish Workbench sign-out revokes every cookie the account holds, and xdist runs the last group while other workers finish theirs. Each worker now drops a marker at session end; under shared auth the sign-out step waits for them (unproven on timeout). A real -n 2 selftest fails without the wait. --- AGENTS.md | 3 +- selftests/test_workbench_ordering.py | 3 +- selftests/test_workbench_parallel.py | 63 +++++++++++++++++++++++++++- src/vip_tests/workbench/barrier.py | 63 ++++++++++++++++++++++++++++ src/vip_tests/workbench/conftest.py | 10 ++++- src/vip_tests/workbench/test_auth.py | 22 +++++++++- 6 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 src/vip_tests/workbench/barrier.py diff --git a/AGENTS.md b/AGENTS.md index a3feb75a7..02bed226b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -212,6 +212,7 @@ Key principles: | `src/vip_tests/workbench/capacity.py` | Resource-profile detection and capping shared by `test_session_capacity.py` and `test_session_capacity_k8s.py` (`ResourceProfileDisabledError`) | | `src/vip_tests/workbench/cleanup.py` | Cookie-based Workbench session cleanup, the API-token sweep used when there are no cookies (`--api-auth`), and the autouse cleanup fixtures. `@api_auth` scenarios get the token sweep and never request `page` | | `src/vip_tests/workbench/sessions.py` | Session-state waits, failure/timeout messages, and the unproven-skip helpers for Workbench scenarios, plus `join_session` — the one way to click a session's join link (waits for Active, retries the click once with a warning, fails if the browser never enters the session). Don't hand-roll `a[title='join ...']` clicks in step files | +| `src/vip_tests/workbench/barrier.py` | Cross-worker barrier: each xdist worker drops a done-marker at session end, and `wait_for_other_workers` holds the sign-out scenario until the others are done | | `src/vip_tests/workbench/login.py` | Workbench login: the cross-worker OIDC login lock, silent SSO, and `workbench_login` | | `src/vip_tests/workbench/exec.py` | In-session execution primitives for Workbench IDE tests, using marker-bracketed capture to extract console output | | `src/vip_tests/workbench/chronicle_probe.py` | Builds the in-session Chronicle raw-chunk probe expression from `chronicle_probe.R`, kept separate so it's unit-testable without a browser | @@ -442,7 +443,7 @@ Register warning filters in `src/vip/plugin/configure.py::pytest_configure` (via - Adding a ruff version anywhere other than the `dev` extra in `pyproject.toml` -- `ci.yml` and `.pre-commit-config.yaml` must keep reading the pin, not carrying their own copy. - Adding product SDK imports (use httpx directly). - Writing tests that modify or delete existing customer content. -- Adding a Workbench scenario that ends the shared auth session (sign-out, session revocation, password change) without ordering it last *and* restoring the session afterwards. Under `--interactive-auth` / `--headless-auth` every Workbench scenario shares one account, so ending that session breaks every scenario still running on other xdist workers, plus the cached auth session on disk. `test_workbench_signout` is the worked example; it also needs its own xdist group (`_OWN_GROUP_SCENARIOS` in the Workbench conftest), because xdist dispatches whole groups and a group that starts early would run it early. +- Adding a Workbench scenario that ends the shared auth session (sign-out, session revocation, password change) without ordering it last *and* restoring the session afterwards. Under `--interactive-auth` / `--headless-auth` every Workbench scenario shares one account, so ending that session breaks every scenario still running on other xdist workers, plus the cached auth session on disk. `test_workbench_signout` is the worked example; it also needs its own xdist group (`_OWN_GROUP_SCENARIOS` in the Workbench conftest), because xdist dispatches whole groups and a group that starts early would run it early. Queued last is still not finished last: xdist hands the last group to the first free worker while the others finish theirs, so the step also requests the `_other_workers_finished` fixture, which waits on `src/vip_tests/workbench/barrier.py` until every other worker's session has ended (`VIP_SIGNOUT_BARRIER_TIMEOUT`, 1800 s by default, then `unproven`). Workbench sign-out revokes every cookie the account holds, so signing out from a separate login does not avoid this. - Creating `.py` step files without a matching `.feature` file (or vice versa). - Forgetting the `@connect`/`@workbench`/`@package_manager` tag in feature files (breaks auto-skip). - Reading pytest-bdd internals anywhere other than `_scenario_for` and `_scenario_given_steps` in `src/vip/plugin/results.py`. pytest-bdd 9 has no public collection-time scenario API, so those two helpers are the only places allowed to touch its registry, its scenario templates or its `_pytest_bdd_example` parameter. diff --git a/selftests/test_workbench_ordering.py b/selftests/test_workbench_ordering.py index 2ba4a2156..bb831ef32 100644 --- a/selftests/test_workbench_ordering.py +++ b/selftests/test_workbench_ordering.py @@ -248,7 +248,8 @@ def test_signout_is_the_last_xdist_group_and_runs_alone(tmp_path: Path) -> None: """Sign-out must not ride in an earlier group, or xdist hands it out at the start of the run. Under --interactive-auth / --headless-auth it would then end the shared session while - other workers are mid-test, which ``order(95)`` alone cannot prevent. + other workers are mid-test, which ``order(95)`` alone cannot prevent. Queued last still + overlaps other workers' final groups; ``TestSignoutBarrier`` covers that part. """ queue = _xdist_group_queue(tmp_path) _, last_tests = queue[-1] diff --git a/selftests/test_workbench_parallel.py b/selftests/test_workbench_parallel.py index 8a3a1f1be..b9c0f5016 100644 --- a/selftests/test_workbench_parallel.py +++ b/selftests/test_workbench_parallel.py @@ -9,8 +9,8 @@ from filelock import FileLock from playwright.sync_api import TimeoutError as PlaywrightTimeoutError +from vip_tests.workbench import barrier, login from vip_tests.workbench import conftest as wb -from vip_tests.workbench import login class TestOidcLoginLock: @@ -627,3 +627,64 @@ def test_directory_group_keeps_its_tests_on_one_worker(self, pytester): rows = [line.split(" ", 1) for line in out.read_text(encoding="utf-8").splitlines()] assert all(nodeid.endswith("@workbench") for _, nodeid in rows), rows assert len({worker for worker, _ in rows}) == 1, rows + + +_BARRIER_CONFTEST = """ +import pytest +from vip_tests.workbench.barrier import mark_worker_done + +@pytest.hookimpl(trylast=True) +def pytest_sessionfinish(session): + mark_worker_done(session.config) +""" + + +class TestSignoutBarrier: + """Sign-out must start only after every other worker is done, not merely be queued last.""" + + def test_last_group_waits_for_the_other_worker(self, pytester): + pytester.makefile(".toml", vip='[general]\ndeployment_name = "Selftest"') + pytester.makeconftest(_BARRIER_CONFTEST) + busy_end = pytester.path / "busy_end.txt" + signout_start = pytester.path / "signout_start.txt" + pytester.makepyfile( + test_barrier=( + "import time, pytest\n" + "from vip_tests.workbench.barrier import wait_for_other_workers\n" + "@pytest.mark.xdist_group('busy')\n" + "def test_busy():\n" + " time.sleep(3)\n" + f" open({str(busy_end)!r}, 'w', encoding='utf-8').write(str(time.time()))\n" + "@pytest.mark.xdist_group('signout')\n" + "def test_signout(request):\n" + " assert wait_for_other_workers(request.config, 60, poll_interval=0.1) == []\n" + f" open({str(signout_start)!r}, 'w', encoding='utf-8').write(str(time.time()))\n" + ) + ) + result = pytester.runpytest("--vip-config=vip.toml", "-n", "2", "--dist", "loadgroup") + result.assert_outcomes(passed=2) + + # loadgroup hands each worker one group at the start, so without the barrier sign-out + # would begin while the busy group is still sleeping. + started = float(signout_start.read_text(encoding="utf-8")) + assert started >= float(busy_end.read_text(encoding="utf-8")) + + @pytest.fixture + def worker_gw0(self, tmp_path, monkeypatch): + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw0") + monkeypatch.setenv("PYTEST_XDIST_WORKER_COUNT", "2") + option = type("Option", (), {"basetemp": str(tmp_path / "popen-gw0")})() + return type("Config", (), {"option": option})() + + def test_returns_workers_still_running_at_timeout(self, worker_gw0): + assert barrier.wait_for_other_workers(worker_gw0, 0, poll_interval=0) == ["gw1"] + + def test_returns_empty_once_the_others_report_done(self, worker_gw0, monkeypatch): + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw1") + barrier.mark_worker_done(worker_gw0) + monkeypatch.setenv("PYTEST_XDIST_WORKER", "gw0") + assert barrier.wait_for_other_workers(worker_gw0, 0, poll_interval=0) == [] + + def test_serial_run_never_waits(self, worker_gw0, monkeypatch): + monkeypatch.delenv("PYTEST_XDIST_WORKER") + assert barrier.wait_for_other_workers(worker_gw0, 0, poll_interval=0) == [] diff --git a/src/vip_tests/workbench/barrier.py b/src/vip_tests/workbench/barrier.py new file mode 100644 index 000000000..1be688cc7 --- /dev/null +++ b/src/vip_tests/workbench/barrier.py @@ -0,0 +1,63 @@ +"""Hold a scenario until every other xdist worker in the run has finished. + +Workbench sign-out revokes every auth cookie the account holds, so under a shared auth session +it must not run while another worker still uses that session. A separate xdist group only queues +it last: xdist hands it to the first free worker while the others finish their own groups. Each +worker therefore drops a marker when its session ends, and the sign-out scenario waits for them. +""" + +from __future__ import annotations + +import os +import time +from pathlib import Path + +import pytest + +SIGNOUT_BARRIER_TIMEOUT = float(os.environ.get("VIP_SIGNOUT_BARRIER_TIMEOUT", "1800")) + +_DONE_DIR_NAME = "vip-workers-done" + + +def _done_dir(config: pytest.Config) -> Path | None: + """The run-wide marker directory, or None outside an xdist worker. + + xdist gives each worker ``basetemp = /popen-gw``, so the parent is + shared by this run's workers only, and pytest prunes it with the rest of old basetemps. + """ + basetemp = config.option.basetemp + if not basetemp or "PYTEST_XDIST_WORKER" not in os.environ: + return None + return Path(basetemp).parent / _DONE_DIR_NAME + + +def mark_worker_done(config: pytest.Config) -> None: + """Record that this worker has run all its tests and torn down its fixtures.""" + done_dir = _done_dir(config) + if done_dir is None: + return + done_dir.mkdir(exist_ok=True) + (done_dir / os.environ["PYTEST_XDIST_WORKER"]).touch() + + +def wait_for_other_workers( + config: pytest.Config, timeout: float, *, poll_interval: float = 2.0 +) -> list[str]: + """Block until every other worker has called :func:`mark_worker_done`. + + Returns the workers still running when *timeout* expired (empty on success, and always + empty in a serial run or with one worker). A worker xdist replaced after a crash never + reports, so the wait then runs to the timeout. + """ + done_dir = _done_dir(config) + count = int(os.environ.get("PYTEST_XDIST_WORKER_COUNT", "1")) + if done_dir is None or count <= 1: + return [] + others = {f"gw{i}" for i in range(count)} - {os.environ["PYTEST_XDIST_WORKER"]} + deadline = time.monotonic() + timeout + while True: + done = {p.name for p in done_dir.iterdir()} if done_dir.is_dir() else set() + waiting = sorted(others - done) + if not waiting or time.monotonic() >= deadline: + return waiting + time.sleep(poll_interval) diff --git a/src/vip_tests/workbench/conftest.py b/src/vip_tests/workbench/conftest.py index 993c05cad..b33552f21 100644 --- a/src/vip_tests/workbench/conftest.py +++ b/src/vip_tests/workbench/conftest.py @@ -30,6 +30,7 @@ vip_names_from_select_labels as _vip_names_from_select_labels, ) from vip_tests.connect.bundles import _SHINY_APP_R, _latest_version, manifest_raw_url +from vip_tests.workbench.barrier import mark_worker_done from vip_tests.workbench.capacity import ( MAX_AUTO_DETECTED_PROFILES, ResourceProfileDisabledError, @@ -185,10 +186,17 @@ def _workbench_group_name(ide_markers: set[str], module_stem: str) -> str: # Scenarios that end the session other workers share run in a group of their own. xdist queues # groups by their first test in collection order, so sharing a group with an early scenario would -# hand them out at the start of the run; alone, sign-out's order(95) queues it last. +# hand them out at the start of the run; alone, sign-out's order(95) queues it last. Queued last +# still overlaps the other workers' final groups, so the scenario also waits on barrier.py. _OWN_GROUP_SCENARIOS = {"test_workbench_signout": "workbench_signout"} +@pytest.hookimpl(trylast=True) +def pytest_sessionfinish(session: pytest.Session) -> None: + """Tell the sign-out barrier this worker is done; trylast so fixtures are torn down first.""" + mark_worker_done(session.config) + + @pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems( config: pytest.Config, diff --git a/src/vip_tests/workbench/test_auth.py b/src/vip_tests/workbench/test_auth.py index 1f741a21e..5b57378a2 100644 --- a/src/vip_tests/workbench/test_auth.py +++ b/src/vip_tests/workbench/test_auth.py @@ -8,6 +8,9 @@ from playwright.sync_api import Browser, Page, expect from pytest_bdd import given, scenario, then, when +from vip import attest +from vip.plugin import _auth_session_key +from vip_tests.workbench.barrier import SIGNOUT_BARRIER_TIMEOUT, wait_for_other_workers from vip_tests.workbench.conftest import ( TIMEOUT_DIALOG, TIMEOUT_PAGE_LOAD, @@ -45,6 +48,23 @@ def test_workbench_signout(): pass +@pytest.fixture +def _other_workers_finished(request: pytest.FixtureRequest) -> None: + """Under a shared auth session, wait until no other xdist worker can still be using it. + + Workbench sign-out revokes every cookie the account holds, not just this browser's. + """ + if request.config.stash.get(_auth_session_key, None) is None: + return + waiting = wait_for_other_workers(request.config, SIGNOUT_BARRIER_TIMEOUT) + if waiting: + attest.unproven( + f"Workers {', '.join(waiting)} were still running after " + f"{SIGNOUT_BARRIER_TIMEOUT:.0f}s; signing out would end the session they share. " + "Raise VIP_SIGNOUT_BARRIER_TIMEOUT or rerun this scenario with -n 0." + ) + + @pytest.fixture def _restore_session_after_signout(page: Page, workbench_url: str): """Put the shared Workbench session back after the sign-out scenario.""" @@ -141,7 +161,7 @@ def current_user_displayed(page: Page): @when("I sign out of Workbench") -def sign_out(page: Page, _restore_session_after_signout): +def sign_out(page: Page, _other_workers_finished, _restore_session_after_signout): """Click the sign-out button on the Workbench homepage. Tries the legacy ``#signOutBtn`` first; on newer Workbench versions the From d6a64de15b82b76c62033d3e72a72a6769cf0fdf Mon Sep 17 00:00:00 2001 From: Ian Flores Siaca <18703558+ian-flores@users.noreply.github.com> Date: Thu, 19 Nov 2026 09:31:55 -0800 Subject: [PATCH 4/4] fix(workbench): refuse a symlinked or foreign git clone workdir The per-worker /tmp/vip_git_ path is predictable, so another user could pre-create it as a symlink and aim the clone's rm -rf at the test account's files. The command now stops unless the directory is a real one the test user owns, and makes it 0700 first. --- selftests/test_workbench_git_ops.py | 44 +++++++++++++++++++++++++ src/vip_tests/workbench/test_git_ops.py | 8 +++-- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/selftests/test_workbench_git_ops.py b/selftests/test_workbench_git_ops.py index 9abd9ccca..69774b566 100644 --- a/selftests/test_workbench_git_ops.py +++ b/selftests/test_workbench_git_ops.py @@ -16,6 +16,9 @@ from __future__ import annotations +import stat +import subprocess +import sys from importlib.util import find_spec from pathlib import Path @@ -93,6 +96,47 @@ def fake_terminal_run(page, cmd, timeout=30_000, *, readback_lang="r"): assert "rm -rf /home/vip/other-repo" in captured["cmd"] +def _clone_prefix(monkeypatch, workdir: Path) -> str: + """The clone command up to the clone itself, so a real shell can run its directory guard.""" + captured: dict = {} + + def fake_terminal_run(page, cmd, timeout=30_000, *, readback_lang="r"): + captured["cmd"] = cmd + return "" + + monkeypatch.setattr(git_ops, "terminal_run", fake_terminal_run) + git_cfg = _FakeGitCfg("https://github.com/example/repo.git") + git_ops._do_clone(object(), {}, git_cfg, "r", workdir=str(workdir)) # type: ignore[arg-type] + return captured["cmd"].split(" && cd ", 1)[0] + + +@pytest.mark.skipif(sys.platform == "win32", reason="runs the clone command in a POSIX shell") +class TestCloneDirGuard: + """Another user can pre-create the predictable per-worker path; rm -rf must not follow it.""" + + def test_refuses_a_symlinked_workdir(self, monkeypatch, tmp_path): + victim = tmp_path / "home" + (victim / "repo").mkdir(parents=True) + (victim / "repo" / "keep.txt").write_text("user data", encoding="utf-8") + workdir = tmp_path / "vip_git_gw0" + workdir.symlink_to(victim) + + result = subprocess.run(["sh", "-c", _clone_prefix(monkeypatch, workdir)], check=False) + + assert result.returncode != 0 + assert (victim / "repo" / "keep.txt").read_text(encoding="utf-8") == "user data" + + def test_clears_the_previous_clone_in_its_own_workdir(self, monkeypatch, tmp_path): + workdir = tmp_path / "vip_git_gw0" + (workdir / "repo").mkdir(parents=True) + + result = subprocess.run(["sh", "-c", _clone_prefix(monkeypatch, workdir)], check=False) + + assert result.returncode == 0 + assert not (workdir / "repo").exists() + assert stat.S_IMODE(workdir.stat().st_mode) == 0o700 + + # --------------------------------------------------------------------------- # #479 -- Gherkin step order guard # --------------------------------------------------------------------------- diff --git a/src/vip_tests/workbench/test_git_ops.py b/src/vip_tests/workbench/test_git_ops.py index 6ba7139bd..c9db7ef34 100644 --- a/src/vip_tests/workbench/test_git_ops.py +++ b/src/vip_tests/workbench/test_git_ops.py @@ -383,7 +383,10 @@ def _do_clone( """Run ``git clone`` in the IDE terminal and return the cloned directory path. Defaults to ``/tmp/vip_git_`` so parallel workers, which may be different - users, never remove or clone over each other's checkout. + users, never remove or clone over each other's checkout. The command stops unless + *workdir* is a real directory this user owns, so another user who pre-creates that + predictable path (say, as a symlink into the test account's home) can't aim ``rm -rf`` + at it; /tmp's sticky bit then keeps anyone else from swapping it out. """ if workdir is None: workdir = f"/tmp/vip_git_{current_worker_id()}" @@ -394,7 +397,8 @@ def _do_clone( terminal_run( page, - f"rm -rf {shlex.quote(clone_dir)} && mkdir -p {q_workdir} && cd {q_workdir} && " + f"mkdir -p {q_workdir} && test ! -L {q_workdir} && test -O {q_workdir} && " + f"chmod 700 {q_workdir} && rm -rf {shlex.quote(clone_dir)} && cd {q_workdir} && " f"GIT_TERMINAL_PROMPT=0 git clone {shlex.quote(auth_url)}", timeout=_TIMEOUT_GIT_NETWORK, readback_lang=readback_lang,