-
Notifications
You must be signed in to change notification settings - Fork 5.2k
fix: sanitize newlines in NO_PROXY env var before httpx client init #3519
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
fc70b13
c3816eb
3e54a0c
c985889
aeb77b5
d433a18
fb5a3a3
ca7116b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,6 @@ | ||
| from __future__ import annotations | ||
|
|
||
| import os | ||
| import sys | ||
| import json | ||
| import math | ||
|
|
@@ -11,6 +12,8 @@ | |
| import logging | ||
| import platform | ||
| import warnings | ||
| import threading | ||
| import contextlib | ||
| import email.utils | ||
| from types import TracebackType | ||
| from random import random | ||
|
|
@@ -860,12 +863,83 @@ def _idempotency_key(self) -> str: | |
| return f"stainless-python-retry-{uuid.uuid4()}" | ||
|
|
||
|
|
||
| _no_proxy_sanitizer_lock = threading.Lock() | ||
|
|
||
|
|
||
| def _reset_no_proxy_sanitizer_lock() -> None: | ||
| """Replace the sanitizer lock after a fork. | ||
|
|
||
| If a POSIX process forks while another thread holds the lock, the child | ||
| inherits the lock in the acquired state but not the thread that can | ||
| release it, so any later client construction in the child would block | ||
| forever. ``os.register_at_fork`` replaces the lock in the child. | ||
| """ | ||
| global _no_proxy_sanitizer_lock | ||
| _no_proxy_sanitizer_lock = threading.Lock() | ||
|
|
||
|
|
||
| if hasattr(os, "register_at_fork"): | ||
| os.register_at_fork(after_in_child=_reset_no_proxy_sanitizer_lock) | ||
|
Comment on lines
+881
to
+882
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When one thread forks while another is inside Useful? React with 👍 / 👎. |
||
|
|
||
|
|
||
| @contextlib.contextmanager | ||
| def _sanitized_no_proxy() -> Generator[None, None, None]: | ||
| """Temporarily normalize line separators in NO_PROXY/no_proxy for the | ||
| duration of a httpx client construction. | ||
|
|
||
| httpx's ``get_environment_proxies()`` only splits on commas, so a line | ||
| separator in ``NO_PROXY`` (common in Docker/``.env`` files or CRLF values | ||
| where the ``\\n`` was stripped but ``\\r`` remains) becomes part of the | ||
| hostname and httpx raises ``InvalidURL`` (issue #3303). httpx reads the | ||
| environment once during ``__init__``, so we only need the sanitized value | ||
| to be visible for that window and then restore the original afterwards — | ||
| this avoids permanently mutating process-global state for unrelated | ||
| clients. | ||
|
|
||
| A module-level lock serializes concurrent client constructions so that one | ||
| call cannot restore the original (invalid) value while another call's | ||
| ``super().__init__()`` is still reading the environment. | ||
| """ | ||
| with _no_proxy_sanitizer_lock: | ||
| originals: dict[str, str] = {} | ||
| sanitized: dict[str, str] = {} | ||
| try: | ||
| for key in ("NO_PROXY", "no_proxy"): | ||
| val = os.environ.get(key) | ||
| # splitlines() recognizes every line boundary (\n, \r, \r\n, | ||
| # \v, \f, \x1c-\x1e, \x85, \u2028, \u2029), including a | ||
| # trailing separator that produces a single part. If the | ||
| # value contains any boundary, the split differs from the | ||
| # original string and sanitization is required. | ||
| if val and val.splitlines() != [val]: | ||
| originals[key] = val | ||
| parts = [part.strip() for part in val.splitlines()] | ||
| sanitized[key] = ",".join(p for p in parts if p) | ||
| os.environ[key] = sanitized[key] | ||
| yield | ||
| finally: | ||
| for key, val in originals.items(): | ||
| # Only restore if the value is still the one we sanitized — | ||
| # application code may have updated NO_PROXY while the client | ||
| # was being constructed, and clobbering that update would be | ||
| # worse than leaving the sanitized value in place. | ||
| if os.environ.get(key) == sanitized.get(key): | ||
| os.environ[key] = val | ||
|
|
||
|
|
||
| class _DefaultHttpxClient(httpx2.Client): | ||
| def __init__(self, **kwargs: Any) -> None: | ||
| kwargs.setdefault("timeout", DEFAULT_TIMEOUT) | ||
| kwargs.setdefault("limits", DEFAULT_CONNECTION_LIMITS) | ||
| kwargs.setdefault("follow_redirects", True) | ||
| super().__init__(**kwargs) | ||
| # httpx reads proxy env vars during __init__; temporarily normalize | ||
| # newlines in NO_PROXY so they don't become part of the hostname | ||
| # (issue #3303). Skip when the caller opted out of env-based proxies. | ||
| if kwargs.get("trust_env", True): | ||
| with _sanitized_no_proxy(): | ||
| super().__init__(**kwargs) | ||
| else: | ||
| super().__init__(**kwargs) | ||
|
|
||
|
|
||
| if TYPE_CHECKING: | ||
|
|
@@ -1459,7 +1533,12 @@ def __init__(self, **kwargs: Any) -> None: | |
| kwargs.setdefault("timeout", DEFAULT_TIMEOUT) | ||
| kwargs.setdefault("limits", DEFAULT_CONNECTION_LIMITS) | ||
| kwargs.setdefault("follow_redirects", True) | ||
| super().__init__(**kwargs) | ||
| # See _DefaultHttpxClient for the rationale behind the NO_PROXY guard. | ||
| if kwargs.get("trust_env", True): | ||
| with _sanitized_no_proxy(): | ||
| super().__init__(**kwargs) | ||
| else: | ||
| super().__init__(**kwargs) | ||
|
|
||
|
|
||
| _DefaultAioHttpClient: type[httpx2.AsyncClient] | ||
|
|
@@ -1480,7 +1559,12 @@ def __init__(self, **kwargs: Any) -> None: | |
| kwargs.setdefault("timeout", DEFAULT_TIMEOUT) | ||
| kwargs.setdefault("limits", DEFAULT_CONNECTION_LIMITS) | ||
| kwargs.setdefault("follow_redirects", True) | ||
| super().__init__(**kwargs) | ||
| # See _DefaultHttpxClient for the rationale behind the NO_PROXY guard. | ||
| if kwargs.get("trust_env", True): | ||
| with _sanitized_no_proxy(): | ||
| super().__init__(**kwargs) | ||
| else: | ||
| super().__init__(**kwargs) | ||
|
|
||
| _DefaultAioHttpClient = _InstalledAioHttpClient | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When a POSIX process forks while another thread is inside a default-client constructor, the child inherits this lock in the acquired state but not the thread that can release it. Any subsequent
OpenAI()or default HTTP client construction in that child then blocks permanently at_sanitized_no_proxy, even whenNO_PROXYneeds no sanitization. Register anafter_in_childfork handler that replaces the lock, or otherwise make the lock process-aware.Useful? React with 👍 / 👎.