From 9ef5b5a9427c27aaa96b4ef662382acb0a7de372 Mon Sep 17 00:00:00 2001 From: Alex Chang Date: Wed, 9 Sep 2026 13:47:02 -0400 Subject: [PATCH 1/3] test: report slow tests while investigating CI regression --- scripts/test | 2 +- scripts/test-pydantic-v1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test b/scripts/test index 68546c7c4d..a9486d15cf 100755 --- a/scripts/test +++ b/scripts/test @@ -60,7 +60,7 @@ if [ "${OPENAI_TEST_HTTP_CLIENT:-httpx}" = "httpx2" ]; then fi echo "==> Running tests" -uv run --locked --all-extras pytest "$@" +uv run --locked --all-extras pytest --durations=20 "$@" echo "==> Running Pydantic v1 tests" ./scripts/test-pydantic-v1 "$@" diff --git a/scripts/test-pydantic-v1 b/scripts/test-pydantic-v1 index a79eb3c37d..b3261bd58f 100755 --- a/scripts/test-pydantic-v1 +++ b/scripts/test-pydantic-v1 @@ -5,4 +5,4 @@ cd "$(dirname "$0")/.." # Keep the compatibility environment separate from the default Pydantic v2 one. export UV_PROJECT_ENVIRONMENT="${OPENAI_PYDANTIC_V1_ENV:-.venv-pydantic-v1}" uv sync --locked --all-extras --no-default-groups --group dev --group pydantic-v1 -exec uv run --no-sync python -m pytest --ignore=tests/functional "$@" +exec uv run --no-sync python -m pytest --durations=20 --ignore=tests/functional "$@" From 6dc4ed05c037c0d31ca8048e0b898bb99f1ddf14 Mon Sep 17 00:00:00 2001 From: Alex Chang Date: Wed, 9 Sep 2026 13:49:20 -0400 Subject: [PATCH 2/3] test: avoid repeated Python startup in dependency security cases --- pyproject.toml | 2 +- scripts/test | 2 +- scripts/test-pydantic-v1 | 2 +- tests/test_uv_workflows.py | 93 +++++++++++++++++++++++++++++++++++--- 4 files changed, 89 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7036b92853..e0ece9a922 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -140,7 +140,7 @@ path = "scripts/hatch_metadata.py" [tool.pytest.ini_options] testpaths = ["tests"] -addopts = "--tb=short -n auto" +addopts = "--tb=short -n auto --durations=20" xfail_strict = true asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "session" diff --git a/scripts/test b/scripts/test index a9486d15cf..68546c7c4d 100755 --- a/scripts/test +++ b/scripts/test @@ -60,7 +60,7 @@ if [ "${OPENAI_TEST_HTTP_CLIENT:-httpx}" = "httpx2" ]; then fi echo "==> Running tests" -uv run --locked --all-extras pytest --durations=20 "$@" +uv run --locked --all-extras pytest "$@" echo "==> Running Pydantic v1 tests" ./scripts/test-pydantic-v1 "$@" diff --git a/scripts/test-pydantic-v1 b/scripts/test-pydantic-v1 index b3261bd58f..a79eb3c37d 100755 --- a/scripts/test-pydantic-v1 +++ b/scripts/test-pydantic-v1 @@ -5,4 +5,4 @@ cd "$(dirname "$0")/.." # Keep the compatibility environment separate from the default Pydantic v2 one. export UV_PROJECT_ENVIRONMENT="${OPENAI_PYDANTIC_V1_ENV:-.venv-pydantic-v1}" uv sync --locked --all-extras --no-default-groups --group dev --group pydantic-v1 -exec uv run --no-sync python -m pytest --durations=20 --ignore=tests/functional "$@" +exec uv run --no-sync python -m pytest --ignore=tests/functional "$@" diff --git a/tests/test_uv_workflows.py b/tests/test_uv_workflows.py index 779777dabd..9899db336f 100644 --- a/tests/test_uv_workflows.py +++ b/tests/test_uv_workflows.py @@ -1,19 +1,25 @@ from __future__ import annotations +import io import os import re import sys import json import shutil +import traceback import subprocess +from types import CodeType from typing import Any, cast from pathlib import Path +from contextlib import redirect_stderr, redirect_stdout import pytest from packaging.markers import Marker from packaging.version import Version from packaging.requirements import Requirement +from openai._utils import lru_cache + if sys.version_info >= (3, 11): import tomllib else: @@ -1213,6 +1219,12 @@ def security_dependency_floor_program() -> str: return program +@lru_cache(maxsize=1) +def compiled_security_dependency_floor_program() -> CodeType: + # Compile once per worker, but execute with fresh globals for every case. + return compile(security_dependency_floor_program(), str(ROOT / "scripts/check-dependency-security.py"), "exec") + + @pytest.mark.parametrize( ("variant", "accepted"), [ @@ -1380,6 +1392,7 @@ def run_security_dependency_floor_check( base_lock_optional_dependencies: dict[tuple[str, str], dict[str, list[dict[str, object]]]] | None = None, head_lock_optional_dependencies: dict[tuple[str, str], dict[str, list[dict[str, object]]]] | None = None, origin: str = "https://github.com/openai/openai-python", + run_in_subprocess: bool = False, ) -> subprocess.CompletedProcess[str]: def project( requirements: list[str], @@ -1476,16 +1489,82 @@ def edges(values: list[dict[str, object]]) -> str: ) fake_git.chmod(0o755) environment = dict(os.environ, BASE_SHA=sha, PATH=str(tmp_path) + os.pathsep + os.environ["PATH"]) - return subprocess.run( - [sys.executable, "-c", security_dependency_floor_program()], - cwd=tmp_path, - env=environment, - capture_output=True, - text=True, - check=False, + if run_in_subprocess: + return subprocess.run( + [sys.executable, "-c", security_dependency_floor_program()], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + check=False, + ) + + # The policy matrix used to launch five Python processes per case. Execute + # the same checker and Git stub in-process; keep real CLI coverage below. + git_program = compile(fake_git.read_text(), str(fake_git), "exec") + + def run_git(args: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str]: + assert args[0] == "git", "Unexpected subprocess in dependency security checker" + output = io.StringIO() + with pytest.MonkeyPatch.context() as patch, redirect_stdout(output): + patch.setattr(sys, "argv", args) + exec(git_program, {"__name__": "__main__"}) + return subprocess.CompletedProcess(args, 0, output.getvalue(), "") + + stdout, stderr = io.StringIO(), io.StringIO() + returncode = 0 + with pytest.MonkeyPatch.context() as patch, redirect_stdout(stdout), redirect_stderr(stderr): + patch.chdir(tmp_path) + patch.setenv("BASE_SHA", sha) + patch.setattr(subprocess, "run", run_git) + try: + exec(compiled_security_dependency_floor_program(), {"__name__": "__main__"}) + except SystemExit as error: + if isinstance(error.code, int): + returncode = error.code + elif error.code is not None: + returncode = 1 + print(error.code, file=stderr) + except AssertionError: + raise + except Exception: + returncode = 1 + traceback.print_exc(file=stderr) + return subprocess.CompletedProcess( + [sys.executable, "-c", ""], returncode, stdout.getvalue(), stderr.getvalue() ) +@pytest.mark.parametrize( + ("minimum", "sha", "origin", "accepted"), + [ + ("danger>=2", "a" * 40, "https://github.com/openai/openai-python", True), + ("danger>=1", "a" * 40, "https://github.com/openai/openai-python", False), + ("danger>=2", "invalid", "https://github.com/openai/openai-python", False), + ("danger>=2", "a" * 40, "https://example.test/foreign.git", False), + ], +) +def test_security_policy_in_process_matches_cli( + tmp_path: Path, minimum: str, sha: str, origin: str, accepted: bool +) -> None: + results = [ + run_security_dependency_floor_check( + tmp_path, + base_requirements=["danger>=1"], + head_requirements=[minimum], + base_packages=[("danger", "1")], + head_packages=[("danger", "2")], + sha=sha, + origin=origin, + run_in_subprocess=isolated, + ) + for isolated in (True, False) + ] + assert results[0].returncode == results[1].returncode == (0 if accepted else 1) + assert results[0].stdout == results[1].stdout + assert results[0].stderr == results[1].stderr + + @pytest.mark.parametrize( ("base", "head", "before", "after", "optional", "accepted"), [ From 4d81eb7f3dcd2e5a6e1a0cf1c74beea6cf724c92 Mon Sep 17 00:00:00 2001 From: Alex Chang Date: Wed, 9 Sep 2026 13:58:32 -0400 Subject: [PATCH 3/3] test: restore TOML compatibility alias after policy checks --- tests/test_uv_workflows.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_uv_workflows.py b/tests/test_uv_workflows.py index 9899db336f..79d95fcb13 100644 --- a/tests/test_uv_workflows.py +++ b/tests/test_uv_workflows.py @@ -1517,6 +1517,10 @@ def run_git(args: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str] patch.chdir(tmp_path) patch.setenv("BASE_SHA", sha) patch.setattr(subprocess, "run", run_git) + if sys.version_info < (3, 11): + # The compatibility prelude writes this alias into sys.modules. + # Register it with MonkeyPatch so it cannot leak into later tests. + patch.setitem(sys.modules, "tomllib", tomllib) try: exec(compiled_security_dependency_floor_program(), {"__name__": "__main__"}) except SystemExit as error: @@ -1545,8 +1549,11 @@ def run_git(args: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[str] ], ) def test_security_policy_in_process_matches_cli( - tmp_path: Path, minimum: str, sha: str, origin: str, accepted: bool + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, minimum: str, sha: str, origin: str, accepted: bool ) -> None: + if sys.version_info < (3, 11): + monkeypatch.delitem(sys.modules, "tomllib", raising=False) + original_tomllib = sys.modules.get("tomllib") results = [ run_security_dependency_floor_check( tmp_path, @@ -1563,6 +1570,7 @@ def test_security_policy_in_process_matches_cli( assert results[0].returncode == results[1].returncode == (0 if accepted else 1) assert results[0].stdout == results[1].stdout assert results[0].stderr == results[1].stderr + assert sys.modules.get("tomllib") is original_tomllib @pytest.mark.parametrize(