diff --git a/AGENTS.md b/AGENTS.md index 8f96a6d..f0c17d1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -242,10 +242,13 @@ - Playwright ingestion is async-loop-safe in cloud workflows: when a caller thread already runs an asyncio event loop, rendering is offloaded away from that thread before Sync Playwright APIs are invoked. -- All Playwright rendering contexts disable service workers and apply the mandatory - context-wide Google Analytics measurement policy before creating pages: every - path on the Analytics measurement hosts, plus `/{g,j,mp,r,batch}/collect` on - enumerated mixed-purpose Google hosts; Google Tag Manager, advertising and +- All Playwright rendering contexts disable service workers, and every page applies + the mandatory Google Analytics measurement policy: every path on the Analytics + measurement hosts, and the measurement paths (`/{batch,g,j,mp,r}/collect`) on + `google.com` and `stats.g.doubleclick.net`. The same hosts and paths are rendered + as URL globs for Chromium's `Network.setBlockedURLs` and as a regex for the + Playwright route other engines fall back to; only Chromium is launched today, so + that route is not currently reached. Google Tag Manager, advertising and third-party hosts remain available. - Playwright ingestion default `wait_until` is `domcontentloaded` (override with `PLAYWRIGHT_WAIT_UNTIL`), and navigation timeout now falls back to partial DOM diff --git a/CHANGELOG.md b/CHANGELOG.md index bfa7a46..36a86fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 8.4.6 - 2026-09-11 + +### Fixed + +- Google Analytics measurement traffic is now blocked inside Chromium's network + stack with `Network.setBlockedURLs` instead of Playwright route interception, + which stopped answering once a page closed and let undecided requests through. + The policy applies to every page a context opens, and covers credentialed and + non-default-port URLs. + ## 8.4.5 - 2026-09-09 ### Fixed diff --git a/docs/render.md b/docs/render.md index 0b8867c..11a7e97 100644 --- a/docs/render.md +++ b/docs/render.md @@ -12,23 +12,39 @@ Renders a URL using `Browser` and converts HTML to XHTML with `HtmlConverter`. ### Browser Thin wrapper around Playwright that opens a page and returns the page, response, elapsed time, and resource list. -Each browser context blocks service workers and installs a mandatory context-wide -route that aborts Google Analytics measurement traffic. Two host groups are -treated differently: - -- `*.google-analytics.com` and `*.analytics.google.com` exist only to collect, - so every path on them is blocked. -- `*.google.com` and `*.stats.g.doubleclick.net` also serve traffic that must - stay reachable, so only the measurement paths are blocked there: - `/g/collect`, `/j/collect`, `/mp/collect`, `/r/collect` and `/batch/collect`. - The path rule is needed because the Google tag sends the same GA4 payload to - `www.google.com/g/collect` when the measurement hosts are unreachable. +Each browser context blocks service workers, and every page blocks Google +Analytics measurement traffic as it is created. + +The policy is defined once as hosts and measurement paths: + +- every path on `google-analytics.com` and `analytics.google.com`, which exist + only to collect measurement +- the measurement paths `/batch/collect`, `/g/collect`, `/j/collect`, + `/mp/collect` and `/r/collect` on `google.com` and `stats.g.doubleclick.net`, + which also serve traffic that must stay reachable + +Each host is listed with and without a subdomain wildcard. + +`google.com` is on the list because the Google tag sends the same GA4 payload to +`www.google.com/g/collect` when the measurement hosts are unreachable. Hosts are enumerated deliberately. A path rule applied to any host cannot be bounded, because third-party endpoint names are unpredictable, so third-party hosts are out of scope whatever they call their paths. Google Tag Manager and advertising conversion endpoints (`/ccm/collect`, `/rmkt/collect//`) remain -available. Blocked URLs and payloads are not logged or added to the +available. + +On Chromium it is rendered as URL globs and applied with +`Network.setBlockedURLs`, which blocks inside the browser's own network stack. +Other engines fall back to a Playwright route over the equivalent regex. That +fallback is weaker: a route decision is a round trip out of the browser, +Playwright stops answering as soon as the page is closed, and the browser then +sends any request it has not been told to block. The regex is the more precise +of the two -- it bounds the path and covers credentials, ports and `http://`, +which the glob syntax cannot express. Only Chromium is launched today, so the +route is not currently reached. + +Blocked URLs and payloads are not logged or added to the response-resource list. Customer-specific first-party or server-side tagging gateways that route measurement through their own path are outside this policy. Disabling service workers intentionally trades PWA offline caching and diff --git a/pyproject.toml b/pyproject.toml index 54903fc..19700fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wordlift-sdk" -version = "8.4.5" +version = "8.4.6" description = "Python toolkit for orchestrating WordLift imports and structured data workflows." authors = ["David Riccitelli "] readme = "README.md" diff --git a/tests/test_render_browser.py b/tests/test_render_browser.py index 84624ee..03bc174 100644 --- a/tests/test_render_browser.py +++ b/tests/test_render_browser.py @@ -4,6 +4,10 @@ import wordlift_sdk.render.browser as browser_module from wordlift_sdk.render.browser import Browser, BrowserOperationError +from wordlift_sdk.render.network_policy import ( + GOOGLE_ANALYTICS_URL_PATTERN, + build_blocked_url_patterns, +) class _FakePage: @@ -11,10 +15,14 @@ def __init__(self, should_raise=False, error_message="boom"): self._handlers = {} self._should_raise = should_raise self._error_message = error_message + self.routes = [] def on(self, name, handler): self._handlers[name] = handler + def route(self, pattern, handler): + self.routes.append((pattern, handler)) + def goto(self, url, wait_until, timeout): if self._should_raise: raise browser_module.PlaywrightError(self._error_message) @@ -32,25 +40,37 @@ class _Req: return response +class _FakeCdpSession: + def __init__(self): + self.sent = [] + + def send(self, method, params=None): + self.sent.append((method, params)) + + class _FakeContext: def __init__(self): self.closed = False self.script = None self.page = _FakePage() - self.route_matcher = None - self.route_handler = None + self.cdp = _FakeCdpSession() self.events = [] + self._page_handlers = [] + + def on(self, event, handler): + if event == "page": + self._page_handlers.append(handler) - def route(self, matcher, handler): - self.route_matcher = matcher - self.route_handler = handler - self.events.append("route") + def new_cdp_session(self, page): + return self.cdp def add_init_script(self, script): self.script = script def new_page(self): self.events.append("new_page") + for handler in self._page_handlers: + handler(self.page) return self.page def close(self): @@ -58,10 +78,11 @@ def close(self): class _FakeBrowser: - def __init__(self): + def __init__(self, engine="chromium"): self.closed = False self.kwargs = None self.context = _FakeContext() + self.browser_type = type("_Type", (), {"name": engine})() def new_context(self, **kwargs): self.kwargs = kwargs @@ -72,9 +93,9 @@ def close(self): class _FakePlaywright: - def __init__(self): + def __init__(self, engine="chromium"): self.chromium = self - self.browser = _FakeBrowser() + self.browser = _FakeBrowser(engine) self.stopped = False def launch(self, headless): @@ -122,21 +143,11 @@ def test_browser_enter_exit_and_open(monkeypatch: pytest.MonkeyPatch): assert pw.browser.kwargs["viewport"]["width"] == 1200 assert pw.browser.kwargs["ignore_https_errors"] is True assert pw.browser.kwargs["service_workers"] == "block" - assert pw.browser.context.events == ["route", "new_page"] - assert pw.browser.context.route_matcher.search( - "https://region1.google-analytics.com/g/collect" + assert pw.browser.context.events == ["new_page"] + assert ("Network.setBlockedURLs", {"urls": build_blocked_url_patterns()}) in ( + pw.browser.context.cdp.sent ) - class _FakeRoute: - aborted_with = None - - def abort(self, error_code): - self.aborted_with = error_code - - route = _FakeRoute() - pw.browser.context.route_handler(route) - assert route.aborted_with == "blockedbyclient" - assert pw.browser.context.closed is True assert pw.browser.closed is True assert pw.stopped is True @@ -174,3 +185,34 @@ def test_browser_open_requires_initialized_context(): browser = Browser(headless=True, timeout_ms=50, wait_until="load") with pytest.raises(RuntimeError, match="not initialized"): browser.open("https://example.org") + + +def test_pages_the_site_opens_are_covered(monkeypatch: pytest.MonkeyPatch): + # A popup the page opens itself is a new page in the same context, and the + # policy has to reach it too -- not just the page `open()` creates. + pw = _FakePlaywright() + monkeypatch.setattr(browser_module, "sync_playwright", lambda: _Manager(pw)) + + with Browser(headless=True, timeout_ms=100, wait_until="load"): + context = pw.browser.context + context.cdp.sent.clear() + for handler in context._page_handlers: + handler(_FakePage()) + + assert ("Network.setBlockedURLs", {"urls": build_blocked_url_patterns()}) in ( + context.cdp.sent + ) + + +def test_non_chromium_engines_use_the_route_fallback(monkeypatch: pytest.MonkeyPatch): + pw = _FakePlaywright(engine="firefox") + monkeypatch.setattr(browser_module, "sync_playwright", lambda: _Manager(pw)) + + with Browser(headless=True, timeout_ms=100, wait_until="load") as browser: + browser.open("https://example.org") + + context = pw.browser.context + assert [pattern for pattern, _ in context.page.routes] == [ + GOOGLE_ANALYTICS_URL_PATTERN + ] + assert context.cdp.sent == [] diff --git a/tests/test_render_network_policy.py b/tests/test_render_network_policy.py index f743289..0ca120f 100644 --- a/tests/test_render_network_policy.py +++ b/tests/test_render_network_policy.py @@ -4,64 +4,106 @@ from wordlift_sdk.render.network_policy import ( GOOGLE_ANALYTICS_URL_PATTERN, + build_blocked_url_patterns, ) +# Blocked by both strategies. +MEASUREMENT_URLS = [ + "https://www.google-analytics.com/collect?v=1&tid=UA-1", + "https://www.google-analytics.com/batch", + "https://www.google-analytics.com/analytics.js", + "https://www.google-analytics.com/__utm.gif?utmac=UA-1", + "https://google-analytics.com/g/collect?v=2", + "https://region1.google-analytics.com/g/collect?v=2&tid=G-XYZ", + "https://analytics.google.com/mp/collect?api_secret=secret", + "https://www.google.com/g/collect?v=2&tid=G-XYZ", + "https://google.com/j/collect?t=pageview", + "https://stats.g.doubleclick.net/j/collect?t=dc", + "https://stats.g.doubleclick.net/batch/collect", + "https://user:pass@google-analytics.com/g/collect?v=2", + "https://google-analytics.com:8443/g/collect?v=2", +] -@pytest.mark.parametrize( - "url", - [ - # Measurement-only hosts: every path is blocked. - "https://www.google-analytics.com/g/collect", - "https://www.google-analytics.com/collect?v=1", - "https://www.google-analytics.com/batch", - "https://user:pass@www.google-analytics.com/g/collect", - "https://region1.google-analytics.com/mp/collect", - "https://ANALYTICS.GOOGLE.COM./g/collect", - # Mixed Google hosts: only the measurement paths. - "https://www.google.com/g/collect?v=2&tid=G-39JJ9JH4VW", - "https://www.google.com/j/collect", - "https://www.google.com/mp/collect", - "https://www.google.com/r/collect", - "https://www.google.com/batch/collect", - "https://stats.g.doubleclick.net/g/collect?tid=G-1", - ], -) -def test_measurement_endpoints_are_blocked(url: str) -> None: - assert GOOGLE_ANALYTICS_URL_PATTERN.search(url) - - -@pytest.mark.parametrize( - "url", - [ - # Advertising and remarketing on the mixed hosts stay reachable. - "https://www.google.com/ccm/collect?en=page_view", - "https://www.google.com/rmkt/collect/1072206699/", - "https://pagead2.googlesyndication.com/ccm/collect?en=page_view", - "https://ad.doubleclick.net/ccm/s/collect", - "https://pagead2.googlesyndication.com/pagead/gen_204", - # Google Tag Manager stays reachable: some sites inject tags through it - # that the rendered markup depends on. - "https://www.googletagmanager.com/gtm.js?id=GTM-123", - "https://www.googletagmanager.com/gtag/js?id=G-123", - "https://tagmanager.google.com/", - # Third-party hosts are out of scope, whatever they call their paths. - "https://acme.com/g/collect", - "https://acme.com/batch/collect", - "https://sgtm.example.com/g/collect", - "https://px.ads.linkedin.com/collect?pid=1", - "https://r.clarity.ms/collect", - "https://example.com/assets/analytics.js", - # Hosts that merely look like the real ones. - "https://notgoogle.com/g/collect", - "https://evilgoogle-analytics.com/collect", - "https://google.com.evil.org/g/collect", - "https://google-analytics.com.example.org/collect", - "https://user@google-analytics.com.example.org/collect", - "https://google.com/", - "https://" + "a." * 64 + "example.com/", - "not a url", - "https://[invalid", - ], -) -def test_unrelated_and_malformed_urls_are_allowed(url: str) -> None: - assert not GOOGLE_ANALYTICS_URL_PATTERN.search(url) +# Reachable under both strategies. +REACHABLE_URLS = [ + "https://www.google.com/collections", + "https://www.google.com/search?q=wordlift", + "https://www.google.com/ccm/collect?en=conversion", + "https://stats.g.doubleclick.net/rmkt/collect/12345/", + "https://www.googletagmanager.com/gtag/js?id=G-XYZ", + "https://example.com/g/collect?v=2", +] + +# The globs are https-only; the regex also covers http. +ROUTE_ONLY_URLS = [ + "http://www.google-analytics.com/g/collect?v=2", +] + +# The globs match anywhere in the URL, so they have no path terminator and fire +# on an unrelated host that merely quotes a measurement URL. +CDP_ONLY_URLS = [ + "https://www.google.com/g/collectData123", + "https://example.com/redirect?to=user@google-analytics.com/g/collect", +] + + +def _cdp_blocks(url: str) -> bool: + for pattern in build_blocked_url_patterns(): + position = 0 + for literal in pattern.split("*"): + if not literal: + continue + found = url.find(literal, position) + if found == -1: + break + position = found + len(literal) + else: + return True + return False + + +def _route_blocks(url: str) -> bool: + return GOOGLE_ANALYTICS_URL_PATTERN.match(url) is not None + + +@pytest.mark.parametrize("url", MEASUREMENT_URLS) +def test_measurement_urls_are_blocked_by_both(url: str) -> None: + assert _cdp_blocks(url) + assert _route_blocks(url) + + +@pytest.mark.parametrize("url", REACHABLE_URLS) +def test_other_traffic_stays_reachable_under_both(url: str) -> None: + assert not _cdp_blocks(url) + assert not _route_blocks(url) + + +@pytest.mark.parametrize("url", ROUTE_ONLY_URLS) +def test_route_covers_what_the_globs_cannot(url: str) -> None: + assert _route_blocks(url) + assert not _cdp_blocks(url) + + +@pytest.mark.parametrize("url", CDP_ONLY_URLS) +def test_globs_match_path_prefixes_the_route_terminates(url: str) -> None: + assert _cdp_blocks(url) + assert not _route_blocks(url) + + +def test_advertising_endpoints_stay_reachable() -> None: + patterns = build_blocked_url_patterns() + assert not any("/ccm/" in pattern or "/rmkt/" in pattern for pattern in patterns) + assert not any("googletagmanager.com" in pattern for pattern in patterns) + assert not GOOGLE_ANALYTICS_URL_PATTERN.match( + "https://www.google.com/ccm/collect?en=conversion" + ) + + +def test_the_glob_scheme_is_literal() -> None: + # A wildcard scheme would swallow the "//" and match look-alike hosts such + # as https://evilgoogle-analytics.com/g/collect. + patterns = build_blocked_url_patterns() + assert patterns + assert all(pattern.startswith("https://") for pattern in patterns) + assert not _cdp_blocks("https://evilgoogle-analytics.com/g/collect") + assert not _route_blocks("https://evilgoogle-analytics.com/g/collect") diff --git a/wordlift_sdk/render/browser.py b/wordlift_sdk/render/browser.py index ffaa36c..71298d3 100644 --- a/wordlift_sdk/render/browser.py +++ b/wordlift_sdk/render/browser.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from time import perf_counter -from .network_policy import GOOGLE_ANALYTICS_URL_PATTERN +from .network_policy import GOOGLE_ANALYTICS_URL_PATTERN, build_blocked_url_patterns from .render_options import DEFAULT_BROWSER_REQUEST_HEADERS try: @@ -80,10 +80,7 @@ def __enter__(self) -> "Browser": context_kwargs["extra_http_headers"] = dict(DEFAULT_BROWSER_REQUEST_HEADERS) context_kwargs["service_workers"] = "block" self._context = self._browser.new_context(**context_kwargs) - self._context.route( - GOOGLE_ANALYTICS_URL_PATTERN, - lambda route: route.abort("blockedbyclient"), - ) + self._context.on("page", self._block_measurement_endpoints) self._context.add_init_script( """ Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); @@ -112,6 +109,35 @@ def __exit__(self, exc_type, exc, tb) -> None: if self._playwright is not None: self._playwright.stop() + def _block_measurement_endpoints(self, page: object) -> None: + """Apply the measurement policy to a page, the strongest way the engine allows. + + Chromium blocks inside its own network stack, which needs no callback + and so cannot be raced. Interception can: Playwright stops resolving + routes the moment ``page.close()`` is called, and Chromium then releases + every still-undecided request to the network. Other engines have no + equivalent, so they fall back to the route and inherit that weakness. + """ + if self._engine_name() == "chromium": + try: + session = self._context.new_cdp_session(page) + session.send("Network.enable") + session.send( + "Network.setBlockedURLs", {"urls": build_blocked_url_patterns()} + ) + except Exception: # pragma: no cover - page closed before we attached + pass + return + page.route( + GOOGLE_ANALYTICS_URL_PATTERN, lambda route: route.abort("blockedbyclient") + ) + + def _engine_name(self) -> str: + try: + return self._browser.browser_type.name + except Exception: # pragma: no cover - defensive + return "unknown" + def open(self, url: str) -> tuple[object | None, object | None, float, list[dict]]: if self._context is None: raise RuntimeError("Browser not initialized") diff --git a/wordlift_sdk/render/network_policy.py b/wordlift_sdk/render/network_policy.py index b0553cd..372f5b5 100644 --- a/wordlift_sdk/render/network_policy.py +++ b/wordlift_sdk/render/network_policy.py @@ -4,43 +4,107 @@ import re +# ============================================================================ +# Shared vocabulary: which hosts and paths are in scope. Both the Chromium +# glob patterns and the Playwright regex below are rendered from these same +# tuples -- keep them in sync when either mechanism's rules change. +# ============================================================================ -# Hosts that exist only to collect Analytics measurement: block every path. -_MEASUREMENT_ONLY_HOST_SUFFIXES = ( +# Hosts observed serving Google Analytics measurement endpoints. These exist +# only to collect measurement, so every path on them is blocked. +_MEASUREMENT_ONLY_HOSTS = ( "google-analytics.com", "analytics.google.com", ) -# Google hosts that also serve traffic which must stay reachable (advertising -# conversions, remarketing, search). Only the measurement paths below are -# blocked on these: -# google.com observed sending GA4 page_view to /g/collect -# stats.g.doubleclick.net documented GA4/Signals endpoint, not observed here -_MIXED_GOOGLE_HOST_SUFFIXES = ( +# `google.com` and `stats.g.doubleclick.net` also serve advertising traffic, +# which the paths below deliberately exclude. +_MIXED_HOSTS = ( "google.com", "stats.g.doubleclick.net", ) # The prefix before `collect` denotes the request type: `g` (GA4 browser), # `j` (Universal Analytics JS), `mp` (Measurement Protocol), `r` (raw) and -# `batch` (batched). Advertising paths on the mixed hosts -- `/ccm/collect`, -# `/rmkt/collect//` -- are deliberately absent so they stay reachable. +# `batch` (batched). Advertising paths (`/ccm/collect`, `/rmkt/collect//`) +# are absent so they stay reachable. +_MEASUREMENT_PATHS = ( + "/batch/collect", + "/g/collect", + "/j/collect", + "/mp/collect", + "/r/collect", +) + + +# ============================================================================ +# Chromium: `Network.setBlockedURLs` glob patterns. +# +# The dialect is `base::MatchPattern`: `*` matches any run of characters, +# everything else is literal, and the pattern must match the whole URL. There +# are no character classes and no anchoring finer than "the whole string" -- +# so, unlike the regex below, a wildcard here can't be bounded to "stop at the +# next `/`". That is what lets `*.`/`*@` accidentally swallow more than a +# subdomain or credentials (see docs/render.md), and why there's no path +# terminator on the mixed-host patterns. +# ============================================================================ + +# GA4 is served from regional subdomains as well as the apex -- we observed +# region1.google-analytics.com -- and one spelling does not match the other. +# `*@` covers credentials on the apex; `*.` already absorbs them on a subdomain. +_HOST_FORMS = ("", "*.", "*@") + +_PORT_FORMS = ("", ":*") + + +def _authorities(hosts: tuple[str, ...]) -> list[str]: + # e.g. for "google-analytics.com": https://google-analytics.com, + # https://google-analytics.com:*, https://*.google-analytics.com, + # https://*.google-analytics.com:*, https://*@google-analytics.com, + # https://*@google-analytics.com:* + return [ + f"https://{form}{host}{port}" + for host in hosts + for form in _HOST_FORMS + for port in _PORT_FORMS + ] + + +def build_blocked_url_patterns() -> list[str]: + # e.g. "https://*.google-analytics.com/**" and + # "https://*.google.com:*/g/collect*" + return [ + f"{authority}/**" for authority in _authorities(_MEASUREMENT_ONLY_HOSTS) + ] + [ + f"{authority}{path}*" + for authority in _authorities(_MIXED_HOSTS) + for path in _MEASUREMENT_PATHS + ] + + +# ============================================================================ +# Other engines: Playwright route regex. # -# Hosts are enumerated rather than matched openly: a path rule applied to any -# host cannot be bounded, since third-party endpoint names are unpredictable. -_MEASUREMENT_PATHS = r"/(?:g|j|mp|r|batch)/collect" +# This is the precise version of the same rule: unlike the glob above, it can +# bound the path (`_TERMINATOR`) and see userinfo, ports and `http://`. Only +# Chromium is launched today, so this path is not currently reached. +# ============================================================================ _SUBDOMAINS = r"(?:[^./?#:@]+\.)*" _HOST_TAIL = r"\.?(?::\d+)?" _TERMINATOR = r"(?:[/?#]|$)" -_ONLY = "|".join(re.escape(host) for host in _MEASUREMENT_ONLY_HOST_SUFFIXES) -_MIXED = "|".join(re.escape(host) for host in _MIXED_GOOGLE_HOST_SUFFIXES) +_ONLY_HOSTS_RE = "|".join(re.escape(host) for host in _MEASUREMENT_ONLY_HOSTS) +_MIXED_HOSTS_RE = "|".join(re.escape(host) for host in _MIXED_HOSTS) +_PATHS_RE = "|".join(re.escape(path) for path in _MEASUREMENT_PATHS) +# Matches, e.g.: https://region1.google-analytics.com/g/collect and +# https://user:pass@google.com:8443/g/collect?x=1 +# Does not match: https://www.google.com/g/collectXYZ (terminator blocks it) GOOGLE_ANALYTICS_URL_PATTERN = re.compile( r"^https?://(?:[^/?#@]*@)?(?:" - rf"{_SUBDOMAINS}(?:{_ONLY}){_HOST_TAIL}{_TERMINATOR}" - rf"|{_SUBDOMAINS}(?:{_MIXED}){_HOST_TAIL}{_MEASUREMENT_PATHS}{_TERMINATOR}" + rf"{_SUBDOMAINS}(?:{_ONLY_HOSTS_RE}){_HOST_TAIL}{_TERMINATOR}" + rf"|{_SUBDOMAINS}(?:{_MIXED_HOSTS_RE}){_HOST_TAIL}(?:{_PATHS_RE}){_TERMINATOR}" r")", re.IGNORECASE, )