From 197ce14a5e407d61ac35b38b45c0cd042dd6278c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 11 Aug 2026 11:47:23 +0900 Subject: [PATCH 01/13] test(browser): require pinned Chrome Agent Task execution --- .../test_agent_task_pinned_chrome_contract.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 tests/test_agent_task_pinned_chrome_contract.py diff --git a/tests/test_agent_task_pinned_chrome_contract.py b/tests/test_agent_task_pinned_chrome_contract.py new file mode 100644 index 000000000..63830f3eb --- /dev/null +++ b/tests/test_agent_task_pinned_chrome_contract.py @@ -0,0 +1,64 @@ +"""Contract for executing the controlled Agent Task fixture on pinned Chrome.""" + +from __future__ import annotations + +import pathlib +import runpy +import unittest + +ROOT = pathlib.Path(__file__).resolve().parents[1] +RUNNER = ROOT / "scripts" / "ci" / "run_mv3_compatibility.py" +FIXTURE = ROOT / "tests" / "fixtures" / "agent_task_basic" / "index.html" +WORKFLOW = ROOT / ".github" / "workflows" / "mv3-compatibility.yml" + + +class AgentTaskPinnedChromeContractTests(unittest.TestCase): + """Keep the first real-browser Agent Task evidence bounded and reproducible.""" + + def test_runner_exposes_a_separate_agent_task_browser_boundary(self) -> None: + """The pinned-browser runner must execute the controlled Agent Task fixture.""" + + namespace = runpy.run_path(str(RUNNER), run_name="agent_task_contract") + for expected in ( + "AGENT_TASK_FIXTURE", + "AGENT_TASK_REPEATABILITY_TRIALS", + "_run_agent_task_browser_pass", + "_run_agent_task_trial", + ): + with self.subTest(expected=expected): + self.assertIn(expected, namespace) + + def test_agent_task_pass_uses_real_webdriver_input_and_post_condition(self) -> None: + """The evidence lane must type, click, and verify fixture state in real Chrome.""" + + runner = RUNNER.read_text(encoding="utf-8") + for expected in ( + "tests/fixtures/agent_task_basic", + '"--disable-extensions"', + '"/value"', + '"/click"', + '"/attribute/data-state"', + '"submitted"', + '"profile_cleaned"', + '"agent_task"', + '"trial_pass_rate"', + ): + with self.subTest(expected=expected): + self.assertIn(expected, runner) + + def test_agent_task_fixture_runs_under_the_existing_pinned_chrome_job(self) -> None: + """No floating browser or second workflow may be introduced for this slice.""" + + workflow = WORKFLOW.read_text(encoding="utf-8") + runner = RUNNER.read_text(encoding="utf-8") + self.assertTrue(FIXTURE.is_file()) + self.assertIn('CHROME_VERSION: "150.0.7871.129"', workflow) + self.assertIn("run_mv3_compatibility.py", workflow) + self.assertIn("150.0.7871.129", runner) + self.assertNotIn("google-chrome-stable", runner.lower()) + self.assertNotIn("COPILOT_GITHUB_TOKEN", runner) + self.assertNotIn("NVIDIA_NIM_API_KEY", runner) + + +if __name__ == "__main__": + unittest.main() From f9917cdd8050c9fdf0aefa669f4d981af85479d6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 11 Aug 2026 11:51:45 +0900 Subject: [PATCH 02/13] feat(browser): execute controlled Agent Task in pinned Chrome --- scripts/ci/run_mv3_compatibility.py | 323 ++++++++++++++++++++++++---- 1 file changed, 286 insertions(+), 37 deletions(-) diff --git a/scripts/ci/run_mv3_compatibility.py b/scripts/ci/run_mv3_compatibility.py index 28a3fb1e2..bb077ad94 100755 --- a/scripts/ci/run_mv3_compatibility.py +++ b/scripts/ci/run_mv3_compatibility.py @@ -1,12 +1,15 @@ #!/usr/bin/env python3 -"""Run bounded repeatable Manifest V3 compatibility evidence against pinned Chromium. +"""Run bounded repeatable real-browser evidence against pinned Chromium. This is a release/CI evidence runner, not a product browser adapter. It uses the W3C WebDriver HTTP protocol only to prove that a real Chrome for Testing build can load the controlled MV3 fixture and repeatedly exercise service-worker, content-script, storage, declarative-net-request, tabs, windows, scripting, commands, side-panel, bookmarks, history, real browser-click, and -restart-persistence behavior. +restart-persistence behavior. It also executes the controlled Agent Task fixture +with extensions disabled in a fresh profile, performs real WebDriver input and +click operations, verifies the observable post-condition, and proves profile +cleanup without treating page content as instruction or authority. """ from __future__ import annotations @@ -27,9 +30,12 @@ ROOT = pathlib.Path(__file__).resolve().parents[2] FIXTURE = ROOT / "tests" / "fixtures" / "mv3_basic" +AGENT_TASK_FIXTURE = ROOT / "tests/fixtures/agent_task_basic" PINNED_CHROME_VERSION = "150.0.7871.129" PINNED_CHROME_REVISION = "r1639810" REPEATABILITY_TRIALS = 3 +AGENT_TASK_REPEATABILITY_TRIALS = 3 +AGENT_TASK_INPUT_VALUE = "originweave controlled input" REQUEST_TIMEOUT_SECONDS = 5.0 STARTUP_TIMEOUT_SECONDS = 20.0 FIXTURE_TIMEOUT_SECONDS = 20.0 @@ -150,6 +156,29 @@ def _execute(driver_port: int, session_id: str, script: str) -> Any: return response.get("value") +def _find_element(driver_port: int, session_id: str, selector: str) -> str: + """Find one fixture element and return its validated ChromeDriver identifier.""" + + found = _json_request( + driver_port, + "POST", + _webdriver_path(session_id, "/element"), + {"using": "css selector", "value": selector}, + ) + element = found.get("value", {}) + element_id = element.get(W3C_ELEMENT_KEY) if isinstance(element, dict) else None + if not isinstance(element_id, str): + raise RuntimeError("WebDriver did not return a W3C element identifier") + return _path_token(element_id, "element identifier") + + +def _element_command_path(session_id: str, element_id: str, suffix: str) -> str: + """Build a bounded WebDriver element command path from validated identifiers.""" + + safe_element = _path_token(element_id, "element identifier") + return _webdriver_path(session_id, f"/element/{safe_element}{suffix}") + + def _wait_for_extension_evidence( driver_port: int, session_id: str, @@ -220,37 +249,18 @@ def _wait_for_extension_evidence( def _exercise_real_click(driver_port: int, session_id: str) -> str: """Use the WebDriver element-click command and verify the DOM post-condition.""" - found = _json_request( - driver_port, - "POST", - _webdriver_path(session_id, "/element"), - {"using": "css selector", "value": "#fixture-button"}, - ) - element = found.get("value", {}) - element_id = element.get(W3C_ELEMENT_KEY) if isinstance(element, dict) else None - if not isinstance(element_id, str): - raise RuntimeError("WebDriver did not return a W3C element identifier") - safe_element = _path_token(element_id, "element identifier") + safe_element = _find_element(driver_port, session_id, "#fixture-button") _json_request( driver_port, "POST", - _webdriver_path(session_id, f"/element/{safe_element}/click"), + _element_command_path(session_id, safe_element, "/click"), {}, ) - output = _json_request( - driver_port, - "POST", - _webdriver_path(session_id, "/element"), - {"using": "css selector", "value": "#fixture-output"}, - ).get("value", {}) - output_id = output.get(W3C_ELEMENT_KEY) if isinstance(output, dict) else None - if not isinstance(output_id, str): - raise RuntimeError("WebDriver did not return the fixture output element") - safe_output = _path_token(output_id, "element identifier") + safe_output = _find_element(driver_port, session_id, "#fixture-output") text = _json_request( driver_port, "GET", - _webdriver_path(session_id, f"/element/{safe_output}/text"), + _element_command_path(session_id, safe_output, "/text"), ).get("value") if text != "clicked": raise RuntimeError(f"real click post-condition failed: {text!r}") @@ -433,8 +443,201 @@ def _run_restart_trial( } +def _run_agent_task_browser_pass( + chrome_bin: pathlib.Path, + chromedriver_bin: pathlib.Path, + fixture_url: str, + profile_dir: str, +) -> dict[str, Any]: + """Execute one synthetic Agent Task through real WebDriver input in pinned Chrome.""" + + started = time.monotonic() + driver_port = _free_loopback_port() + session_id: str | None = None + driver = subprocess.Popen( + [str(chromedriver_bin), f"--port={driver_port}", "--allowed-ips=127.0.0.1"], + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + text=True, + ) + try: + _wait_for_driver(driver_port) + session = _json_request( + driver_port, + "POST", + "/session", + { + "capabilities": { + "alwaysMatch": { + "browserName": "chrome", + "goog:chromeOptions": { + "binary": str(chrome_bin), + "args": [ + "--headless=new", + "--no-first-run", + "--disable-default-apps", + "--disable-component-update", + "--disable-sync", + "--disable-dev-shm-usage", + "--no-sandbox", + "--disable-extensions", + f"--user-data-dir={profile_dir}", + ], + }, + } + } + }, + ).get("value", {}) + if not isinstance(session, dict): + raise RuntimeError("ChromeDriver Agent Task session response is malformed") + raw_session_id = session.get("sessionId") + capabilities = session.get("capabilities", {}) + if not isinstance(raw_session_id, str): + raise RuntimeError("ChromeDriver did not return an Agent Task session id") + session_id = _path_token(raw_session_id, "session identifier") + browser_version = ( + capabilities.get("browserVersion") if isinstance(capabilities, dict) else None + ) + if browser_version != PINNED_CHROME_VERSION: + raise RuntimeError( + f"unexpected Agent Task Chrome version: expected {PINNED_CHROME_VERSION}, " + f"got {browser_version!r}" + ) + + _json_request( + driver_port, + "POST", + _webdriver_path(session_id, "/url"), + {"url": fixture_url}, + ) + input_element = _find_element(driver_port, session_id, "#task-text") + _json_request( + driver_port, + "POST", + _element_command_path(session_id, input_element, "/clear"), + {}, + ) + _json_request( + driver_port, + "POST", + _element_command_path(session_id, input_element, "/value"), + {"text": AGENT_TASK_INPUT_VALUE, "value": list(AGENT_TASK_INPUT_VALUE)}, + ) + submit_element = _find_element( + driver_port, + session_id, + "#agent-task-form button[type=submit]", + ) + _json_request( + driver_port, + "POST", + _element_command_path(session_id, submit_element, "/click"), + {}, + ) + result_element = _find_element(driver_port, session_id, "#task-result") + state = _json_request( + driver_port, + "GET", + _element_command_path(session_id, result_element, "/attribute/data-state"), + ).get("value") + text = _json_request( + driver_port, + "GET", + _element_command_path(session_id, result_element, "/text"), + ).get("value") + if state != "submitted": + raise RuntimeError(f"Agent Task state post-condition failed: {state!r}") + if text != AGENT_TASK_INPUT_VALUE: + raise RuntimeError("Agent Task result did not match the synthetic typed value") + return { + "browser_version": browser_version, + "post_condition": True, + "input_echo_verified": True, + "extensions_disabled": True, + "duration_ms": round((time.monotonic() - started) * 1000), + } + finally: + if session_id is not None: + with contextlib.suppress(Exception): + _json_request( + driver_port, + "DELETE", + _webdriver_path(session_id, ""), + {}, + ) + driver.terminate() + try: + driver.wait(timeout=5) + except subprocess.TimeoutExpired: + driver.kill() + driver.wait(timeout=5) + + +def _run_agent_task_trial( + chrome_bin: pathlib.Path, + chromedriver_bin: pathlib.Path, + fixture_url: str, + trial_number: int, +) -> dict[str, Any]: + """Run one isolated Agent Task browser trial and prove its profile is removed.""" + + trial_started = time.monotonic() + profile_path: pathlib.Path + with tempfile.TemporaryDirectory( + prefix=f"originweave-agent-task-trial-{trial_number}-" + ) as profile_dir: + profile_path = pathlib.Path(profile_dir) + result = _run_agent_task_browser_pass( + chrome_bin, + chromedriver_bin, + fixture_url, + profile_dir, + ) + profile_cleaned = not profile_path.exists() + if not profile_cleaned: + raise RuntimeError(f"Agent Task profile cleanup failed in trial {trial_number}") + + return { + "trial_number": trial_number, + "passed": True, + "browser_version": result["browser_version"], + "post_condition": result["post_condition"], + "input_echo_verified": result["input_echo_verified"], + "extensions_disabled": result["extensions_disabled"], + "profile_cleaned": profile_cleaned, + "duration_ms": round((time.monotonic() - trial_started) * 1000), + } + + +def _start_fixture_server(directory: pathlib.Path) -> tuple[http.server.ThreadingHTTPServer, threading.Thread]: + """Start one loopback-only static fixture server for a bounded browser lane.""" + + server = http.server.ThreadingHTTPServer( + ("127.0.0.1", 0), + lambda *args, **kwargs: QuietFixtureHandler( + *args, + directory=str(directory), + **kwargs, + ), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread + + +def _stop_fixture_server( + server: http.server.ThreadingHTTPServer, + thread: threading.Thread, +) -> None: + """Stop one bounded fixture server and join its helper thread.""" + + server.shutdown() + server.server_close() + thread.join(timeout=5) + + def main() -> int: - """Run three independent restart trials and emit bounded repeatability evidence.""" + """Run bounded MV3 and Agent Task trials and emit credential-free evidence.""" chrome_bin = pathlib.Path(os.environ.get("CHROME_BIN", "")) chromedriver_bin = pathlib.Path(os.environ.get("CHROMEDRIVER_BIN", "")) @@ -444,15 +647,11 @@ def main() -> int: raise SystemExit("CHROMEDRIVER_BIN must point to the matching pinned ChromeDriver") if not (FIXTURE / "manifest.json").is_file(): raise SystemExit("MV3 fixture manifest is missing") + if not (AGENT_TASK_FIXTURE / "index.html").is_file(): + raise SystemExit("Agent Task fixture is missing") - fixture_server = http.server.ThreadingHTTPServer( - ("127.0.0.1", 0), - lambda *args, **kwargs: QuietFixtureHandler( - *args, directory=str(FIXTURE), **kwargs - ), - ) - fixture_thread = threading.Thread(target=fixture_server.serve_forever, daemon=True) - fixture_thread.start() + fixture_server, fixture_thread = _start_fixture_server(FIXTURE) + agent_task_server, agent_task_thread = _start_fixture_server(AGENT_TASK_FIXTURE) started = time.monotonic() try: @@ -496,6 +695,43 @@ def main() -> int: for name in first_surfaces } + agent_task_url = ( + f"http://127.0.0.1:{agent_task_server.server_port}/index.html" + ) + agent_task_trials: list[dict[str, Any]] = [] + for trial_number in range(1, AGENT_TASK_REPEATABILITY_TRIALS + 1): + try: + agent_task_trials.append( + _run_agent_task_trial( + chrome_bin, + chromedriver_bin, + agent_task_url, + trial_number, + ) + ) + except (OSError, ValueError, RuntimeError, json.JSONDecodeError): + agent_task_trials.append( + { + "trial_number": trial_number, + "passed": False, + } + ) + + agent_task_successful_trials = sum( + 1 for trial in agent_task_trials if trial.get("passed") is True + ) + agent_task_trial_pass_rate = ( + agent_task_successful_trials / AGENT_TASK_REPEATABILITY_TRIALS + ) + agent_task_surfaces_complete = all( + trial.get("post_condition") is True + and trial.get("input_echo_verified") is True + and trial.get("extensions_disabled") is True + and trial.get("profile_cleaned") is True + for trial in agent_task_trials + if trial.get("passed") is True + ) + evidence = { "chrome_version": PINNED_CHROME_VERSION, "chrome_revision": PINNED_CHROME_REVISION, @@ -509,6 +745,12 @@ def main() -> int: if successful_results else [] ), + "agent_task": { + "repeatability_trials": AGENT_TASK_REPEATABILITY_TRIALS, + "successful_trials": agent_task_successful_trials, + "trial_pass_rate": agent_task_trial_pass_rate, + "trial_results": agent_task_trials, + }, "duration_ms": round((time.monotonic() - started) * 1000), } print(json.dumps(evidence, sort_keys=True)) @@ -519,11 +761,18 @@ def main() -> int: ) if not common_surfaces or not all(common_surfaces.values()): raise RuntimeError("Manifest V3 repeatability surfaces were incomplete") + if agent_task_successful_trials != AGENT_TASK_REPEATABILITY_TRIALS: + raise RuntimeError( + "Agent Task repeatability gate failed: " + f"{agent_task_successful_trials}/{AGENT_TASK_REPEATABILITY_TRIALS} " + "trials passed" + ) + if not agent_task_surfaces_complete: + raise RuntimeError("Agent Task repeatability surfaces were incomplete") return 0 finally: - fixture_server.shutdown() - fixture_server.server_close() - fixture_thread.join(timeout=5) + _stop_fixture_server(agent_task_server, agent_task_thread) + _stop_fixture_server(fixture_server, fixture_thread) if __name__ == "__main__": From 41c10cf1631699eeeedfc5f265b1211dd97e5a86 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 11 Aug 2026 23:47:18 +0900 Subject: [PATCH 03/13] chore(browser): sync hardened fixture contract --- tests/test_agent_task_fixture_contract.py | 66 +++++++++++++++++++---- 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/tests/test_agent_task_fixture_contract.py b/tests/test_agent_task_fixture_contract.py index 2a1465b2e..2565a35c7 100644 --- a/tests/test_agent_task_fixture_contract.py +++ b/tests/test_agent_task_fixture_contract.py @@ -10,6 +10,21 @@ FIXTURE = ROOT / "tests" / "fixtures" / "agent_task_basic" / "index.html" +def _is_credential_input(attributes: dict[str, str | None]) -> bool: + """Return whether parsed input attributes describe a credential surface.""" + + input_type = (attributes.get("type") or "").strip().lower() + if input_type == "password": + return True + + autocomplete = (attributes.get("autocomplete") or "").strip().lower() + autocomplete_tokens = autocomplete.split() + return any( + token == "one-time-code" or "password" in token + for token in autocomplete_tokens + ) + + class _FixtureParser(HTMLParser): """Collect the small semantic surface required by the deterministic fixture.""" @@ -18,6 +33,7 @@ def __init__(self) -> None: self.ids: set[str] = set() self.labels_for: set[str] = set() self.input_names: set[str] = set() + self.input_attributes: list[dict[str, str | None]] = [] self.button_types: set[str] = set() self.hidden_injection_markers = 0 @@ -30,12 +46,15 @@ def handle_starttag( self.ids.add(element_id) if tag == "label" and attributes.get("for"): self.labels_for.add(attributes["for"]) - if tag == "input" and attributes.get("name"): - self.input_names.add(attributes["name"]) + if tag == "input": + self.input_attributes.append(attributes) + if attributes.get("name"): + self.input_names.add(attributes["name"]) if tag == "button" and attributes.get("type"): self.button_types.add(attributes["type"]) if ( attributes.get("data-originweave-untrusted") == "prompt-injection" + and "hidden" in attributes and attributes.get("aria-hidden") == "true" ): self.hidden_injection_markers += 1 @@ -70,20 +89,49 @@ def test_fixture_contains_explicit_untrusted_hidden_prompt_injection(self) -> No self.assertIn("UNTRUSTED_PAGE_INSTRUCTION", self.html) self.assertIn("request new browser capabilities", self.html) + def test_hidden_injection_requires_the_actual_hidden_attribute(self) -> None: + """ARIA metadata alone must not satisfy the hidden-injection fixture contract.""" + + parser = _FixtureParser() + parser.feed( + "" + "" + ) + self.assertEqual(parser.hidden_injection_markers, 1) + def test_fixture_is_synthetic_and_has_no_credential_fields(self) -> None: """The controlled workflow must not require or imitate real secret collection.""" + for attributes in self.parser.input_attributes: + with self.subTest(attributes=attributes): + self.assertFalse(_is_credential_input(attributes)) + lowered = self.html.lower() - for forbidden in ( - 'type="password"', - 'autocomplete="current-password"', - 'autocomplete="one-time-code"', - "api_key", - "secret_key", - ): + for forbidden in ("api_key", "secret_key"): with self.subTest(forbidden=forbidden): self.assertNotIn(forbidden, lowered) + def test_credential_detection_is_quote_independent(self) -> None: + """Parsed credential semantics must reject single-quoted and tokenized forms.""" + + for html in ( + "", + "", + "", + "", + "", + ): + with self.subTest(html=html): + parser = _FixtureParser() + parser.feed(html) + self.assertEqual(len(parser.input_attributes), 1) + self.assertTrue(_is_credential_input(parser.input_attributes[0])) + + parser = _FixtureParser() + parser.feed("") + self.assertEqual(len(parser.input_attributes), 1) + self.assertFalse(_is_credential_input(parser.input_attributes[0])) + if __name__ == "__main__": unittest.main() From 9ed317b5053ed3027cb1420e3da69a29ee88eff0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 11 Aug 2026 23:55:54 +0900 Subject: [PATCH 04/13] test(browser): require unchanged Agent Task URL --- tests/test_agent_task_pinned_chrome_contract.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_agent_task_pinned_chrome_contract.py b/tests/test_agent_task_pinned_chrome_contract.py index 63830f3eb..bf084090c 100644 --- a/tests/test_agent_task_pinned_chrome_contract.py +++ b/tests/test_agent_task_pinned_chrome_contract.py @@ -46,6 +46,19 @@ def test_agent_task_pass_uses_real_webdriver_input_and_post_condition(self) -> N with self.subTest(expected=expected): self.assertIn(expected, runner) + def test_agent_task_submission_preserves_the_loaded_url(self) -> None: + """Submission must prove that the controlled action did not navigate away.""" + + runner = RUNNER.read_text(encoding="utf-8") + for expected in ( + "initial_url", + "post_submit_url", + "url_unchanged", + ): + with self.subTest(expected=expected): + self.assertIn(expected, runner) + self.assertIn("Agent Task URL changed during submission", runner) + def test_agent_task_fixture_runs_under_the_existing_pinned_chrome_job(self) -> None: """No floating browser or second workflow may be introduced for this slice.""" From 65be355e6f771ea8bbe5f0418ce31e5f86832abe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 12 Aug 2026 01:15:40 +0900 Subject: [PATCH 05/13] test(browser): prove Agent Task URL remains unchanged --- scripts/ci/run_mv3_compatibility.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/ci/run_mv3_compatibility.py b/scripts/ci/run_mv3_compatibility.py index bb077ad94..131edfd0d 100755 --- a/scripts/ci/run_mv3_compatibility.py +++ b/scripts/ci/run_mv3_compatibility.py @@ -510,6 +510,15 @@ def _run_agent_task_browser_pass( _webdriver_path(session_id, "/url"), {"url": fixture_url}, ) + initial_url = _json_request( + driver_port, + "GET", + _webdriver_path(session_id, "/url"), + ).get("value") + if initial_url != fixture_url: + raise RuntimeError( + f"Agent Task initial URL mismatch: expected {fixture_url!r}, got {initial_url!r}" + ) input_element = _find_element(driver_port, session_id, "#task-text") _json_request( driver_port, @@ -534,6 +543,14 @@ def _run_agent_task_browser_pass( _element_command_path(session_id, submit_element, "/click"), {}, ) + post_submit_url = _json_request( + driver_port, + "GET", + _webdriver_path(session_id, "/url"), + ).get("value") + url_unchanged = post_submit_url == initial_url + if not url_unchanged: + raise RuntimeError("Agent Task URL changed during submission") result_element = _find_element(driver_port, session_id, "#task-result") state = _json_request( driver_port, @@ -553,6 +570,7 @@ def _run_agent_task_browser_pass( "browser_version": browser_version, "post_condition": True, "input_echo_verified": True, + "url_unchanged": url_unchanged, "extensions_disabled": True, "duration_ms": round((time.monotonic() - started) * 1000), } @@ -603,6 +621,7 @@ def _run_agent_task_trial( "browser_version": result["browser_version"], "post_condition": result["post_condition"], "input_echo_verified": result["input_echo_verified"], + "url_unchanged": result["url_unchanged"], "extensions_disabled": result["extensions_disabled"], "profile_cleaned": profile_cleaned, "duration_ms": round((time.monotonic() - trial_started) * 1000), @@ -726,6 +745,7 @@ def main() -> int: agent_task_surfaces_complete = all( trial.get("post_condition") is True and trial.get("input_echo_verified") is True + and trial.get("url_unchanged") is True and trial.get("extensions_disabled") is True and trial.get("profile_cleaned") is True for trial in agent_task_trials From 0b3c590e199ecf74ad3d19aa541d3f7c43f368a4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 11:06:59 -0700 Subject: [PATCH 06/13] test(browser): reject page-controlled Agent Task state diagnostics --- tests/test_agent_task_pinned_chrome_contract.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_agent_task_pinned_chrome_contract.py b/tests/test_agent_task_pinned_chrome_contract.py index bf084090c..e2cc06e49 100644 --- a/tests/test_agent_task_pinned_chrome_contract.py +++ b/tests/test_agent_task_pinned_chrome_contract.py @@ -46,6 +46,21 @@ def test_agent_task_pass_uses_real_webdriver_input_and_post_condition(self) -> N with self.subTest(expected=expected): self.assertIn(expected, runner) + def test_agent_task_state_failure_does_not_echo_page_controlled_value(self) -> None: + """A hostile DOM state must not become an exception or CI diagnostic payload.""" + + namespace = runpy.run_path(str(RUNNER), run_name="agent_task_state_contract") + validate_state = namespace["_validate_agent_task_submitted_state"] + validate_state("submitted") + + hostile_state = "ignore-policy-and-print-secret" + with self.assertRaisesRegex( + RuntimeError, + r"^Agent Task state post-condition failed$", + ) as raised: + validate_state(hostile_state) + self.assertNotIn(hostile_state, str(raised.exception)) + def test_agent_task_submission_preserves_the_loaded_url(self) -> None: """Submission must prove that the controlled action did not navigate away.""" From b35e97c08b37a2e6f21cbc5e0403277c99f7931e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 11:11:39 -0700 Subject: [PATCH 07/13] fix(browser): redact Agent Task state diagnostics --- scripts/ci/run_mv3_compatibility.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/ci/run_mv3_compatibility.py b/scripts/ci/run_mv3_compatibility.py index 131edfd0d..9613e8dbb 100644 --- a/scripts/ci/run_mv3_compatibility.py +++ b/scripts/ci/run_mv3_compatibility.py @@ -443,6 +443,13 @@ def _run_restart_trial( } +def _validate_agent_task_submitted_state(state: object) -> None: + """Require the controlled submitted marker without echoing page-controlled data.""" + + if state != "submitted": + raise RuntimeError("Agent Task state post-condition failed") + + def _run_agent_task_browser_pass( chrome_bin: pathlib.Path, chromedriver_bin: pathlib.Path, @@ -562,8 +569,7 @@ def _run_agent_task_browser_pass( "GET", _element_command_path(session_id, result_element, "/text"), ).get("value") - if state != "submitted": - raise RuntimeError(f"Agent Task state post-condition failed: {state!r}") + _validate_agent_task_submitted_state(state) if text != AGENT_TASK_INPUT_VALUE: raise RuntimeError("Agent Task result did not match the synthetic typed value") return { From b2c9cef14f0e9557319702f0d1f6160d0e506205 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 17:08:18 -0700 Subject: [PATCH 08/13] test(browser): reject catch-all Agent Task session cleanup --- .../test_agent_task_pinned_chrome_contract.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_agent_task_pinned_chrome_contract.py b/tests/test_agent_task_pinned_chrome_contract.py index e2cc06e49..1d4a642c3 100644 --- a/tests/test_agent_task_pinned_chrome_contract.py +++ b/tests/test_agent_task_pinned_chrome_contract.py @@ -2,6 +2,7 @@ from __future__ import annotations +import inspect import pathlib import runpy import unittest @@ -61,6 +62,25 @@ def test_agent_task_state_failure_does_not_echo_page_controlled_value(self) -> N validate_state(hostile_state) self.assertNotIn(hostile_state, str(raised.exception)) + def test_agent_task_session_cleanup_never_suppresses_programming_failures(self) -> None: + """Unexpected cleanup defects must fail closed instead of becoming successful evidence.""" + + namespace = runpy.run_path(str(RUNNER), run_name="agent_task_cleanup_contract") + self.assertIn("_cleanup_agent_task_browser_session", namespace) + cleanup_session = namespace["_cleanup_agent_task_browser_session"] + browser_pass_source = inspect.getsource(namespace["_run_agent_task_browser_pass"]) + self.assertNotIn("contextlib.suppress(Exception)", browser_pass_source) + + def unexpected_cleanup_failure(*_args: object, **_kwargs: object) -> dict[str, object]: + raise AssertionError("unexpected cleanup programming failure") + + cleanup_session.__globals__["_json_request"] = unexpected_cleanup_failure + with self.assertRaisesRegex( + AssertionError, + r"^unexpected cleanup programming failure$", + ): + cleanup_session(9515, "session-1") + def test_agent_task_submission_preserves_the_loaded_url(self) -> None: """Submission must prove that the controlled action did not navigate away.""" From 6d30e9e95b0b84444f3d36cac554b3765406edff Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 17:12:17 -0700 Subject: [PATCH 09/13] fix(browser): fail closed on Agent Task session cleanup --- scripts/ci/run_mv3_compatibility.py | 33 +++++++++++++++++------------ 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/scripts/ci/run_mv3_compatibility.py b/scripts/ci/run_mv3_compatibility.py index 9613e8dbb..98e4e7605 100644 --- a/scripts/ci/run_mv3_compatibility.py +++ b/scripts/ci/run_mv3_compatibility.py @@ -450,6 +450,17 @@ def _validate_agent_task_submitted_state(state: object) -> None: raise RuntimeError("Agent Task state post-condition failed") +def _cleanup_agent_task_browser_session(driver_port: int, session_id: str) -> None: + """Delete one Agent Task WebDriver session without suppressing cleanup failures.""" + + _json_request( + driver_port, + "DELETE", + _webdriver_path(session_id, ""), + {}, + ) + + def _run_agent_task_browser_pass( chrome_bin: pathlib.Path, chromedriver_bin: pathlib.Path, @@ -581,20 +592,16 @@ def _run_agent_task_browser_pass( "duration_ms": round((time.monotonic() - started) * 1000), } finally: - if session_id is not None: - with contextlib.suppress(Exception): - _json_request( - driver_port, - "DELETE", - _webdriver_path(session_id, ""), - {}, - ) - driver.terminate() try: - driver.wait(timeout=5) - except subprocess.TimeoutExpired: - driver.kill() - driver.wait(timeout=5) + if session_id is not None: + _cleanup_agent_task_browser_session(driver_port, session_id) + finally: + driver.terminate() + try: + driver.wait(timeout=5) + except subprocess.TimeoutExpired: + driver.kill() + driver.wait(timeout=5) def _run_agent_task_trial( From d1f0dfc611c07b847477bfe4671eba0a4d83d065 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 10:35:59 +0900 Subject: [PATCH 10/13] docs(browser): record pinned Agent Task evidence --- CHANGELOG.md | 1 + docs/DOCUMENTATION_FITNESS.md | 4 ++-- docs/TEST_STRATEGY.md | 7 +++++++ .../action-postcondition-evidence.md | 21 +++++++++++++------ .../test_agent_task_pinned_chrome_contract.py | 17 +++++++++++++++ 5 files changed, 42 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d17419927..dfd858e22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ All notable changes to OriginWeave are documented in this file. The format follo - Real loopback TCP integration proof plus deterministic timeout, refusal, retry, peer-inspection, peer-mismatch, canonicalization, IPv6 metadata, and single-use replay tests. - Real loopback rustls integration covering trusted DNS SAN, Common-Name fallback rejection, wrong-name and untrusted-root rejection, fixed-time expiry and not-yet-valid failures, exact IPv4 and IPv6 SANs, TLS 1.2/TLS 1.3, required and optional ALPN, and transport-origin binding. - Cumulative interactive-first RAM, VRAM, batch, local-model, admission, pause, and compositor-pressure mitigation plans, including active-consumer reduction at exact hard limits. +- Real pinned-Chrome WebDriver evidence for the controlled Agent Task fixture: the CI lane uses an isolated profile, disables extensions, types and submits synthetic text, observes the same-document post-condition, and proves profile cleanup; this does not claim a shipped OriginWeave browser adapter. - Universally value-redacted network evidence with explicit path, metadata, and provenance bounds; ambiguous path rejection; validated source URLs; lowercase SHA-256 identifiers; and verification state. - Rust 1.97.1 build contract, strict Clippy and rustdoc gates, and exact production function, line, region, and branch coverage enforcement. - Hourly bounded OpenCode product-development workflow using `NVIDIA_NIM_API_KEY`, an unprivileged disposable workspace, loopback-only model broker, independently verified patches, and publication through a dedicated `OPENCODE_PR_TOKEN` that cannot review or merge. diff --git a/docs/DOCUMENTATION_FITNESS.md b/docs/DOCUMENTATION_FITNESS.md index 69f603252..cd637ae65 100644 --- a/docs/DOCUMENTATION_FITNESS.md +++ b/docs/DOCUMENTATION_FITNESS.md @@ -158,9 +158,9 @@ Active #64 makes a successful action-outcome value require existing verified pro ### 3.19 Controlled Agent Task fixture -Active #65 supplies a deterministic synthetic local web fixture with a labelled semantic input, submit control, same-document post-condition and explicitly hidden/untrusted prompt-injection text. The fixture contains no credential collection surface and requires no live third-party site. +Active #65 supplies a deterministic synthetic local web fixture with a labelled semantic input, submit control, same-document post-condition and explicitly hidden/untrusted prompt-injection text. Active #70 executes that fixture through real WebDriver on pinned Chrome with an isolated profile, disabled extensions, synthetic input, same-document post-condition verification and profile cleanup. The fixture contains no credential collection surface and requires no live third-party site. -**Resolution:** the fixture makes the future real Chromium vertical slice reproducible without turning a third-party site into a test dependency. It is not a browser adapter, semantic extractor, input dispatcher, policy engine, trusted clock, process-attribution source or proof of real Chromium execution. +**Resolution:** the #65/#70 lane makes controlled browser-level evidence reproducible without turning a third-party site into a test dependency. It is not a browser adapter, semantic extractor, OriginWeave input-dispatch authority, policy engine, trusted clock, process-attribution source or proof of the shipped product runtime. ### 3.20 Bounded browser process-set resource evidence diff --git a/docs/TEST_STRATEGY.md b/docs/TEST_STRATEGY.md index ba3c31624..56dd6f18f 100644 --- a/docs/TEST_STRATEGY.md +++ b/docs/TEST_STRATEGY.md @@ -78,6 +78,13 @@ session creation -> task close/recovery ``` +Active PR #70 exercises the controlled local Agent Task fixture on the pinned +Chrome for Testing build through real WebDriver input, same-document +post-condition observation and ephemeral-profile cleanup. That lane proves +browser-level fixture execution only; it does not replace the OriginWeave +BiDi/CDP authority adapter, semantic node contract, policy dispatch or +protected-main runtime acceptance required by issue #28. + ### 3.5 Buyer acceptance Versioned task packs measure repeatable product outcomes rather than one lucky agent run. The benchmark artifact records browser build, OriginWeave version, model/provider/reasoning configuration, seed where supported, policy profile, hardware profile and source fixtures. diff --git a/docs/traceability/action-postcondition-evidence.md b/docs/traceability/action-postcondition-evidence.md index 23a6e764d..3dc12137a 100644 --- a/docs/traceability/action-postcondition-evidence.md +++ b/docs/traceability/action-postcondition-evidence.md @@ -55,6 +55,14 @@ On that unchanged exact head, CI run `31445201739` succeeds; Rust contracts job This remains controlled test infrastructure rather than browser-execution evidence. The fixture itself does not establish WebDriver BiDi/CDP transport, Chromium semantic extraction, policy dispatch, native input, post-condition provenance, profile teardown or process attribution. +### PR #70 — pinned Chrome execution of the controlled Agent Task fixture + +**Capability maturity:** `IMPLEMENTED_ON_ACTIVE_PR` + +PR #70 reuses the existing pinned Chrome for Testing workflow and executes the #65 fixture through loopback ChromeDriver with extensions disabled and a fresh temporary profile. Each bounded trial performs real WebDriver clear/type/click operations, observes the `submitted` state and synthetic value through element endpoints, verifies that submission preserves the loaded URL, and proves that the temporary profile is removed after teardown. The runner emits credential-free repeatability evidence and fails the lane when any trial or post-condition is incomplete. + +This is real WebDriver evidence for a controlled local fixture, not a product browser adapter. It does not establish WebDriver BiDi/CDP authority translation, OriginWeave semantic observation or node handles, policy-authorized typed action dispatch, trusted browser-process attribution, or protected-main product runtime completion. + ## 4. Non-transitive success semantics The intended first-slice chain is: @@ -77,10 +85,10 @@ Unverified -/> successful action completion Rejected -/> successful action completion caller-supplied timestamp ordering -/> proof of trusted clock provenance VerifiedActionOutcomeEvidence type existence -/> proof of real Chromium execution -controlled fixture success -/> proof of real Chromium execution +controlled fixture success -/> proof of an OriginWeave product browser runtime ``` -PR #64 now rejects a caller-supplied observation timestamp that predates caller-supplied dispatch time, but the type cannot independently prove the clock source, that a real browser actually dispatched the action, that the supplied provenance belongs to the claimed browser target/node, or that the observed state was caused by that action. PR #65 supplies deterministic hostile input and a post-condition target but no browser execution. Those claims remain the responsibility of the real adapter/runtime composition under issue #28. +PR #64 now rejects a caller-supplied observation timestamp that predates caller-supplied dispatch time, but the type cannot independently prove the clock source, that a real browser actually dispatched the action, that the supplied provenance belongs to the claimed browser target/node, or that the observed state was caused by that action. PR #70 proves real Chromium execution against the controlled fixture, but its test-harness CSS locators and direct WebDriver calls are not the OriginWeave adapter/runtime composition required under issue #28. ## 5. Active prerequisite graph for issue #28 @@ -93,15 +101,16 @@ The first real Chromium vertical slice remains distributed across bounded active - PR #49 — ephemeral compatibility-profile lifecycle regression stacked on #43; - PR #51 — bounded browser-task telemetry plus one explicitly supplied Linux PID `VmRSS` sampler; Chromium process discovery/process-set attribution remains outside that slice; - PR #64 — verified and caller-timestamp-ordered post-condition action-outcome evidence; and -- PR #65 — controlled hostile local Agent Task workflow fixture, gate-clean and Ready for review. +- PR #65 — controlled hostile local Agent Task workflow fixture; and +- PR #70 — real WebDriver execution of that fixture on pinned Chrome, without claiming a product browser adapter. -These active PRs are non-shipped evidence. They do not themselves compose WebDriver BiDi/CDP transport, trusted Chromium process attribution, policy-authorized real input dispatch, causal post-condition observation, or deterministic end-to-end teardown/recovery into one protected-main runtime. +These active PRs are non-shipped evidence. PR #70 proves a bounded browser-level fixture flow, but the active set does not itself compose WebDriver BiDi/CDP transport, OriginWeave authority translation, trusted Chromium process attribution, policy-authorized real input dispatch, causal post-condition observation, or deterministic end-to-end teardown/recovery into one protected-main runtime. ## 6. Remaining issue #28 boundary This dossier does **not** close issue #28. Material remaining work includes: -- pinned stock Chromium exercised as one reproducible end-to-end Agent Task runtime path, not only extension compatibility fixtures; +- a production Agent Task runtime path that composes pinned stock Chromium with OriginWeave authority, rather than only the controlled #70 fixture and extension compatibility fixtures; - isolated Agent Task profile/context lifecycle and cleanup in the production vertical path; - versioned WebDriver BiDi adapter plus explicitly bounded CDP observation fallback where needed; - real semantic observation feeding typed query and policy-authorized typed action; @@ -113,4 +122,4 @@ This dossier does **not** close issue #28. Material remaining work includes: ## 7. Documentation fitness consequence -The ADR/PRD/TRD/Architecture/UML/ERD graph remains **DESIGN-SUFFICIENT / PROTECTED-MAIN-PARTIAL**. PR #64 narrows a typed evidence gap already governed by existing provenance/action-success decisions, while PR #65 supplies controlled test infrastructure for the eventual real-browser proof. Neither introduces a new trust domain, deployed component, persistence owner, database schema, or independent architecture decision, so a new ADR or physical ERD entity would overstate the implementation. Detailed real-Chromium dispatch/post-condition sequence diagrams should be reconciled when the executable adapter chain stabilizes rather than manufacturing as-built detail before that runtime exists. +The ADR/PRD/TRD/Architecture/UML/ERD graph remains **DESIGN-SUFFICIENT / PROTECTED-MAIN-PARTIAL**. PR #64 narrows a typed evidence gap, PR #65 supplies the controlled fixture, and PR #70 supplies real WebDriver evidence for that fixture. Neither introduces a new trust domain, deployed component, persistence owner, database schema, or independent architecture decision, so a new ADR or physical ERD entity would overstate the implementation. Detailed real-Chromium dispatch/post-condition sequence diagrams should be reconciled when the executable adapter chain stabilizes rather than manufacturing as-built detail before that runtime exists. diff --git a/tests/test_agent_task_pinned_chrome_contract.py b/tests/test_agent_task_pinned_chrome_contract.py index 1d4a642c3..cf4ef2658 100644 --- a/tests/test_agent_task_pinned_chrome_contract.py +++ b/tests/test_agent_task_pinned_chrome_contract.py @@ -11,6 +11,9 @@ RUNNER = ROOT / "scripts" / "ci" / "run_mv3_compatibility.py" FIXTURE = ROOT / "tests" / "fixtures" / "agent_task_basic" / "index.html" WORKFLOW = ROOT / ".github" / "workflows" / "mv3-compatibility.yml" +CHANGELOG = ROOT / "CHANGELOG.md" +TRACEABILITY = ROOT / "docs" / "traceability" / "action-postcondition-evidence.md" +FITNESS = ROOT / "docs" / "DOCUMENTATION_FITNESS.md" class AgentTaskPinnedChromeContractTests(unittest.TestCase): @@ -107,6 +110,20 @@ def test_agent_task_fixture_runs_under_the_existing_pinned_chrome_job(self) -> N self.assertNotIn("COPILOT_GITHUB_TOKEN", runner) self.assertNotIn("NVIDIA_NIM_API_KEY", runner) + def test_documentation_separates_active_browser_evidence_from_product_runtime(self) -> None: + """Documentation must record the real fixture evidence without shipping the adapter claim.""" + + changelog = CHANGELOG.read_text(encoding="utf-8") + traceability = TRACEABILITY.read_text(encoding="utf-8") + fitness = FITNESS.read_text(encoding="utf-8") + self.assertIn("Real pinned-Chrome WebDriver evidence", changelog) + self.assertIn("does not claim a shipped OriginWeave browser adapter", changelog) + self.assertIn("PR #70", traceability) + self.assertIn("real WebDriver", traceability) + self.assertIn("not a product browser adapter", traceability) + self.assertIn("pinned Chrome", fitness) + self.assertIn("not a browser adapter", fitness) + if __name__ == "__main__": unittest.main() From 991e4d814296f5ada357b61bb6c0275bc3326e94 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 16:12:25 -0700 Subject: [PATCH 11/13] test(browser): expose cleanup evidence defects --- .../test_agent_task_pinned_chrome_contract.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/tests/test_agent_task_pinned_chrome_contract.py b/tests/test_agent_task_pinned_chrome_contract.py index cf4ef2658..84846a6b8 100644 --- a/tests/test_agent_task_pinned_chrome_contract.py +++ b/tests/test_agent_task_pinned_chrome_contract.py @@ -65,6 +65,86 @@ def test_agent_task_state_failure_does_not_echo_page_controlled_value(self) -> N validate_state(hostile_state) self.assertNotIn(hostile_state, str(raised.exception)) + def test_browser_session_cleanup_never_uses_catch_all_suppression(self) -> None: + """MV3 and Agent Task cleanup must fail closed without catch-all suppression.""" + + namespace = runpy.run_path(str(RUNNER), run_name="browser_cleanup_contract") + for function_name in ("_run_browser_pass", "_run_agent_task_browser_pass"): + source = inspect.getsource(namespace[function_name]) + with self.subTest(function_name=function_name): + self.assertNotIn("contextlib.suppress(Exception)", source) + + def test_expected_cleanup_failure_preserves_the_primary_browser_failure(self) -> None: + """A recoverable DELETE failure must retain the causal browser-pass failure.""" + + namespace = runpy.run_path(str(RUNNER), run_name="cleanup_cause_contract") + self.assertIn("_cleanup_browser_session_preserving_primary", namespace) + self.assertIn("BrowserSessionCleanupError", namespace) + cleanup = namespace["_cleanup_browser_session_preserving_primary"] + cleanup_error_type = namespace["BrowserSessionCleanupError"] + primary = RuntimeError("primary browser failure") + + def expected_cleanup_failure(*_args: object, **_kwargs: object) -> None: + raise OSError("host-controlled cleanup detail") + + cleanup.__globals__["_cleanup_browser_session"] = expected_cleanup_failure + with self.assertRaises(cleanup_error_type) as raised: + cleanup(9515, "session-1", primary) + self.assertIs(raised.exception.__cause__, primary) + self.assertEqual(raised.exception.cleanup_error_type, "OSError") + self.assertNotIn("host-controlled cleanup detail", str(raised.exception)) + + def test_unexpected_cleanup_programming_failure_is_not_normalized(self) -> None: + """Programming failures in cleanup must propagate rather than enter fallback handling.""" + + namespace = runpy.run_path(str(RUNNER), run_name="cleanup_programming_contract") + self.assertIn("_cleanup_browser_session_preserving_primary", namespace) + cleanup = namespace["_cleanup_browser_session_preserving_primary"] + primary = RuntimeError("primary browser failure") + + def unexpected_cleanup_failure(*_args: object, **_kwargs: object) -> None: + raise AssertionError("unexpected cleanup programming failure") + + cleanup.__globals__["_cleanup_browser_session"] = unexpected_cleanup_failure + with self.assertRaisesRegex( + AssertionError, + r"^unexpected cleanup programming failure$", + ): + cleanup(9515, "session-1", primary) + + def test_agent_task_profile_cleanup_evidence_records_a_real_transition(self) -> None: + """Profile cleanup evidence must prove the profile existed before it became absent.""" + + namespace = runpy.run_path(str(RUNNER), run_name="profile_cleanup_contract") + trial_source = inspect.getsource(namespace["_run_agent_task_trial"]) + self.assertIn("profile_observed_before_cleanup", trial_source) + self.assertIn("temporary_profile.cleanup()", trial_source) + self.assertNotIn("with tempfile.TemporaryDirectory", trial_source) + + def test_agent_task_surface_completeness_is_non_vacuous(self) -> None: + """Surface completeness must be false for empty or failed-trial evidence.""" + + namespace = runpy.run_path(str(RUNNER), run_name="agent_task_surface_contract") + self.assertIn("_agent_task_surfaces_complete", namespace) + surfaces_complete = namespace["_agent_task_surfaces_complete"] + self.assertFalse(surfaces_complete([])) + self.assertFalse(surfaces_complete([{"trial_number": 1, "passed": False}])) + self.assertTrue( + surfaces_complete( + [ + { + "trial_number": 1, + "passed": True, + "post_condition": True, + "input_echo_verified": True, + "url_unchanged": True, + "extensions_disabled": True, + "profile_cleaned": True, + } + ] + ) + ) + def test_agent_task_session_cleanup_never_suppresses_programming_failures(self) -> None: """Unexpected cleanup defects must fail closed instead of becoming successful evidence.""" From d2d504d10e6803b1b6e8cce69b2c180262cf76ee Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 16:16:43 -0700 Subject: [PATCH 12/13] fix(browser): preserve cleanup failure causality --- scripts/ci/run_mv3_compatibility.py | 140 +++++++++++++++++++++------- 1 file changed, 105 insertions(+), 35 deletions(-) diff --git a/scripts/ci/run_mv3_compatibility.py b/scripts/ci/run_mv3_compatibility.py index 98e4e7605..caf999028 100644 --- a/scripts/ci/run_mv3_compatibility.py +++ b/scripts/ci/run_mv3_compatibility.py @@ -14,7 +14,6 @@ from __future__ import annotations -import contextlib import http.client import http.server import json @@ -23,6 +22,7 @@ import socket import string import subprocess +import sys import tempfile import threading import time @@ -51,6 +51,26 @@ def log_message(self, _format: str, *args: object) -> None: """Suppress request logs because the fixture contains no diagnostic value.""" +class BrowserSessionCleanupError(RuntimeError): + """Report bounded WebDriver-session cleanup failure without echoing remote text.""" + + def __init__(self, cleanup_error: BaseException) -> None: + self.cleanup_error_type = type(cleanup_error).__name__ + super().__init__( + "WebDriver session cleanup failed; see the chained causal browser failure" + ) + + +class BrowserProfileCleanupError(RuntimeError): + """Report bounded profile cleanup failure without exposing filesystem details.""" + + def __init__(self, cleanup_error: BaseException) -> None: + self.cleanup_error_type = type(cleanup_error).__name__ + super().__init__( + "browser profile cleanup failed; see the chained causal browser failure" + ) + + def _free_loopback_port() -> int: """Reserve and release one loopback TCP port for a short-lived local service.""" @@ -179,6 +199,33 @@ def _element_command_path(session_id: str, element_id: str, suffix: str) -> str: return _webdriver_path(session_id, f"/element/{safe_element}{suffix}") +def _cleanup_browser_session(driver_port: int, session_id: str) -> None: + """Delete one WebDriver session through the fixed loopback authority.""" + + _json_request( + driver_port, + "DELETE", + _webdriver_path(session_id, ""), + {}, + ) + + +def _cleanup_browser_session_preserving_primary( + driver_port: int, + session_id: str, + primary_error: BaseException | None, +) -> None: + """Fail closed on expected cleanup errors while retaining an earlier causal failure.""" + + try: + _cleanup_browser_session(driver_port, session_id) + except (OSError, ValueError, RuntimeError, json.JSONDecodeError) as cleanup_error: + bounded_error = BrowserSessionCleanupError(cleanup_error) + if primary_error is None: + raise bounded_error from cleanup_error + raise bounded_error from primary_error + + def _wait_for_extension_evidence( driver_port: int, session_id: str, @@ -363,20 +410,21 @@ def _run_browser_pass( }, } finally: - if session_id is not None: - with contextlib.suppress(Exception): - _json_request( + primary_error = sys.exc_info()[1] + try: + if session_id is not None: + _cleanup_browser_session_preserving_primary( driver_port, - "DELETE", - _webdriver_path(session_id, ""), - {}, + session_id, + primary_error, ) - driver.terminate() - try: - driver.wait(timeout=5) - except subprocess.TimeoutExpired: - driver.kill() - driver.wait(timeout=5) + finally: + driver.terminate() + try: + driver.wait(timeout=5) + except subprocess.TimeoutExpired: + driver.kill() + driver.wait(timeout=5) def _run_restart_trial( @@ -453,12 +501,7 @@ def _validate_agent_task_submitted_state(state: object) -> None: def _cleanup_agent_task_browser_session(driver_port: int, session_id: str) -> None: """Delete one Agent Task WebDriver session without suppressing cleanup failures.""" - _json_request( - driver_port, - "DELETE", - _webdriver_path(session_id, ""), - {}, - ) + _cleanup_browser_session(driver_port, session_id) def _run_agent_task_browser_pass( @@ -592,9 +635,14 @@ def _run_agent_task_browser_pass( "duration_ms": round((time.monotonic() - started) * 1000), } finally: + primary_error = sys.exc_info()[1] try: if session_id is not None: - _cleanup_agent_task_browser_session(driver_port, session_id) + _cleanup_browser_session_preserving_primary( + driver_port, + session_id, + primary_error, + ) finally: driver.terminate() try: @@ -613,18 +661,32 @@ def _run_agent_task_trial( """Run one isolated Agent Task browser trial and prove its profile is removed.""" trial_started = time.monotonic() - profile_path: pathlib.Path - with tempfile.TemporaryDirectory( + temporary_profile = tempfile.TemporaryDirectory( prefix=f"originweave-agent-task-trial-{trial_number}-" - ) as profile_dir: - profile_path = pathlib.Path(profile_dir) + ) + profile_path = pathlib.Path(temporary_profile.name) + profile_observed_before_cleanup = profile_path.is_dir() + if not profile_observed_before_cleanup: + raise RuntimeError(f"Agent Task profile was not created in trial {trial_number}") + + try: result = _run_agent_task_browser_pass( chrome_bin, chromedriver_bin, fixture_url, - profile_dir, + temporary_profile.name, ) - profile_cleaned = not profile_path.exists() + finally: + primary_error = sys.exc_info()[1] + try: + temporary_profile.cleanup() + except OSError as cleanup_error: + bounded_error = BrowserProfileCleanupError(cleanup_error) + if primary_error is None: + raise bounded_error from cleanup_error + raise bounded_error from primary_error + + profile_cleaned = profile_observed_before_cleanup and not profile_path.exists() if not profile_cleaned: raise RuntimeError(f"Agent Task profile cleanup failed in trial {trial_number}") @@ -641,6 +703,22 @@ def _run_agent_task_trial( } +def _agent_task_surfaces_complete(agent_task_trials: list[dict[str, Any]]) -> bool: + """Require every recorded Agent Task trial to contain every success surface.""" + + if not agent_task_trials: + return False + return all( + trial.get("passed") is True + and trial.get("post_condition") is True + and trial.get("input_echo_verified") is True + and trial.get("url_unchanged") is True + and trial.get("extensions_disabled") is True + and trial.get("profile_cleaned") is True + for trial in agent_task_trials + ) + + def _start_fixture_server(directory: pathlib.Path) -> tuple[http.server.ThreadingHTTPServer, threading.Thread]: """Start one loopback-only static fixture server for a bounded browser lane.""" @@ -755,15 +833,7 @@ def main() -> int: agent_task_trial_pass_rate = ( agent_task_successful_trials / AGENT_TASK_REPEATABILITY_TRIALS ) - agent_task_surfaces_complete = all( - trial.get("post_condition") is True - and trial.get("input_echo_verified") is True - and trial.get("url_unchanged") is True - and trial.get("extensions_disabled") is True - and trial.get("profile_cleaned") is True - for trial in agent_task_trials - if trial.get("passed") is True - ) + agent_task_surfaces_complete = _agent_task_surfaces_complete(agent_task_trials) evidence = { "chrome_version": PINNED_CHROME_VERSION, From 5bb7c7f6cffb46d6e6841e8d80046bd7d292f345 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 16:22:45 -0700 Subject: [PATCH 13/13] test(browser): cover truncated cleanup response --- .../test_agent_task_pinned_chrome_contract.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_agent_task_pinned_chrome_contract.py b/tests/test_agent_task_pinned_chrome_contract.py index 84846a6b8..ba92405b9 100644 --- a/tests/test_agent_task_pinned_chrome_contract.py +++ b/tests/test_agent_task_pinned_chrome_contract.py @@ -2,6 +2,7 @@ from __future__ import annotations +import http.client import inspect import pathlib import runpy @@ -94,6 +95,24 @@ def expected_cleanup_failure(*_args: object, **_kwargs: object) -> None: self.assertEqual(raised.exception.cleanup_error_type, "OSError") self.assertNotIn("host-controlled cleanup detail", str(raised.exception)) + def test_malformed_cleanup_response_is_a_bounded_cleanup_failure(self) -> None: + """A truncated DELETE response must stay in the typed cleanup failure contract.""" + + namespace = runpy.run_path(str(RUNNER), run_name="cleanup_http_contract") + cleanup = namespace["_cleanup_browser_session_preserving_primary"] + cleanup_error_type = namespace["BrowserSessionCleanupError"] + primary = RuntimeError("primary browser failure") + + def malformed_cleanup_response(*_args: object, **_kwargs: object) -> None: + raise http.client.IncompleteRead(b"partial", 32) + + cleanup.__globals__["_cleanup_browser_session"] = malformed_cleanup_response + with self.assertRaises(cleanup_error_type) as raised: + cleanup(9515, "session-1", primary) + self.assertIs(raised.exception.__cause__, primary) + self.assertEqual(raised.exception.cleanup_error_type, "IncompleteRead") + self.assertNotIn("partial", str(raised.exception)) + def test_unexpected_cleanup_programming_failure_is_not_normalized(self) -> None: """Programming failures in cleanup must propagate rather than enter fallback handling."""