From f335e429a75425f1923d9b68b4a9ffc7568d5012 Mon Sep 17 00:00:00 2001 From: sykosyber Date: Wed, 4 Nov 2026 16:53:22 -0800 Subject: [PATCH 1/3] Add failing tests: a POST refused before its body is read must still deliver the refusal (FM-49) The Windows CI leg flakes in test_preview_appends_nothing_for_anyone with WinError 10053. The server answers an unauthenticated POST without reading its body; when the body lands after the answer, closing the socket resets the connection and Windows discards the refusal the client had not read. late_body_post() forces that gap on every run with a raw socket. The new ConsoleRefusals tests assert the refusal arrives whole, and that a refused body too large or of unknown length is not read and the answer says Connection: close. test_preview_appends_nothing_for_anyone now asserts the status of every answer and turns a broken connection into a failure. These fail on main; the server fix follows. Co-Authored-By: Claude Opus 5.5 --- tests/test_console.py | 81 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/tests/test_console.py b/tests/test_console.py index 62d3440..f13d3eb 100644 --- a/tests/test_console.py +++ b/tests/test_console.py @@ -12,6 +12,7 @@ import hashlib import json import os +import select import shutil import socket import sqlite3 @@ -122,6 +123,37 @@ def wait_until_listening(port: int, proc: subprocess.Popen, timeout: float = 10) raise RuntimeError("reference system did not accept connections") +def late_body_post(port: int, path: str, headers: dict, body: bytes = b"", wait: float = 1.0): + """POST over a raw socket, holding the body back until the server answers or ``wait`` seconds pass. + + ``http.client`` writes the header block and the body in two sends. Under load the server can read the headers and + answer before the body lands; a server that then closes with the body unread resets the connection, and on Windows + the reset discards the answer the client had not read yet (WinError 10053). Here that gap is forced on every run. + + Returns ``(answered_early, status, headers, body, error)``: whether an answer arrived before the body was sent, the + response read to the end of the stream, and the ``OSError`` that broke the exchange, or None. + """ + lines = [f"POST {path} HTTP/1.1", "Host: 127.0.0.1", "Connection: close", *(f"{k}: {v}" for k, v in headers.items())] + received, error = b"", None + with socket.create_connection(("127.0.0.1", port), timeout=10) as sock: + sock.sendall(("\r\n".join(lines) + "\r\n\r\n").encode("latin-1")) + answered_early = bool(select.select([sock], [], [], wait)[0]) + if answered_early: + time.sleep(0.2) # let the server finish answering and close, as it has under load by the time the body lands + try: + if body: + sock.sendall(body) + while chunk := sock.recv(65536): + received += chunk + except OSError as exc: + error = exc + head, _, payload = received.partition(b"\r\n\r\n") + status_line, *header_lines = head.decode("latin-1").split("\r\n") if head else [""] + status = int(status_line.split()[1]) if len(status_line.split()) > 1 else None + response_headers = {name.strip().lower(): value.strip() for name, _, value in (h.partition(":") for h in header_lines)} + return answered_early, status, response_headers, payload, error + + def certified_home(folder: Path, seed: bytes = SEED) -> tuple[Path, dict]: """A Build Thread home whose one candidate was checked, accepted and certified, as ``syberlabs certify`` leaves it. @@ -264,6 +296,44 @@ def setUp(self): self.addCleanup(self.cell.close) +class ConsoleRefusals(CellCase): + """A POST refused before its body is read still delivers the refusal whole (FM-49). + + The server answers an unauthenticated POST without reading the body. If the body lands after the answer, closing the + socket with it unread resets the connection, and on Windows the reset throws away the refusal the client had not read. + """ + + BODY = json.dumps({"action": "record_review"}).encode() + + def post(self, headers, body=b"", wait=1.0): + return late_body_post(self.cell.server.server_port, "/api/cases/any/preview", headers, body, wait) + + def assert_whole(self, result, status, error): + _, got_status, headers, payload, broke = result + self.assertIsNone(broke, "the connection broke before the refusal was read") + self.assertEqual(int(headers.get("content-length", -1)), len(payload), "the refusal arrived whole") + self.assertEqual((got_status, json.loads(payload)["error"]), (status, error)) + return headers + + def test_an_unauthenticated_post_whose_body_arrives_late_still_gets_its_refusal(self): + for who, headers in (("no token", {}), ("a wrong token", {"Authorization": "Bearer not-a-token"})): + with self.subTest(who): + result = self.post({"Content-Type": "application/json", "Content-Length": str(len(self.BODY)), **headers}, self.BODY) + self.assert_whole(result, 409, "unauthorized") + + def test_a_refused_body_too_large_or_of_unknown_length_is_not_read_and_the_connection_is_closed(self): + too_large = str(1024 * 1024 + 1) + cases = (({}, too_large, "unauthorized"), ({}, "abc", "unauthorized"), ({}, None, "unauthorized"), + ({"Authorization": "Bearer " + self.cell.tokens["operator"]}, too_large, "payload_size"), + ({"Authorization": "Bearer " + self.cell.tokens["operator"]}, "abc", "invalid_request")) + for auth, length, error in cases: + with self.subTest(auth=bool(auth), length=length): + headers = {**auth, **({} if length is None else {"Content-Length": length})} + result = self.post(headers) + self.assertTrue(result[0], "answered at once, without waiting for a body it will not read") + self.assertEqual(self.assert_whole(result, 409, error).get("connection"), "close") + + class ConsoleRoutes(CellCase): def test_every_new_route_requires_a_bearer_token(self): for path in ("/api/console", "/api/contracts", "/api/contracts/purchase-order/1", "/api/policies", "/api/actions", "/api/sources", @@ -1971,7 +2041,16 @@ def test_preview_appends_nothing_for_anyone(self): {"action": "absent", "args": {}}, {"action": "absent"}, {"args": {}}, {"action": "record_review", "args": []}) for body in bodies: for who in ("operator", "scheduler", "manager", "planner", "supplier", None): - self.cell.call(f"/api/cases/{case}/preview", body, who=who) + try: + status, _, raw = self.cell.call(f"/api/cases/{case}/preview", body, who=who) + except OSError as exc: + self.fail(f"preview {body} as {who}: the connection broke before the answer was read ({exc!r})") + answer = json.loads(raw) + if who is None: + self.assertEqual((status, answer["error"]), (409, "unauthorized"), body) + else: + self.assertIn(status, (200, 409), (body, who, answer)) + self.assertNotEqual(answer.get("error"), "internal_error", (body, who)) self.assertEqual(self.cell.work.inspect(case)["events"], events) self.assertEqual(self.dump(), before, "no table changed, including event_side and reservations") self.assertTrue(self.cell.work.verify_chain(case)) From 859cb693ff324ab022f0f1ad6160f6e35b25f236 Mon Sep 17 00:00:00 2001 From: sykosyber Date: Wed, 4 Nov 2026 17:06:03 -0800 Subject: [PATCH 2/3] Read a refused POST's body before answering, so a late body cannot reset the refusal away (FM-49) do_POST checked the token before reading the body. A refusal was sent and the socket closed with the body unread; a body that landed late made the close reset the connection, and Windows discarded the refusal the client had not read yet (WinError 10053, the Windows CI flake in test_preview_appends_nothing_for_anyone). Every do_POST path that answers before the body is read now handles it: - the token refusal, and any unexpected error while resolving the token: the body is read and dropped first, up to 1 MB and 5 s in total (one receive per read, so a trickling client cannot stretch the wait); - a Content-Length that is missing, malformed or over 1 MB, or a body held back for Expect: 100-continue: nothing is read and the refusal says Connection: close; - _body's own refusals (malformed or over-limit length, and an authenticated POST with no Content-Length): Connection: close. Status codes and error bodies are unchanged. Co-Authored-By: Claude Opus 5.5 --- syberwork/server.py | 58 ++++++++++++++++++++++++++++++++++++++++--- tests/test_console.py | 31 ++++++++++++++++++----- 2 files changed, 80 insertions(+), 9 deletions(-) diff --git a/syberwork/server.py b/syberwork/server.py index 843393d..80d9b6a 100644 --- a/syberwork/server.py +++ b/syberwork/server.py @@ -32,6 +32,9 @@ ATTEST_ONLY_ROLES = frozenset({"observer"}) # The first path segments of the reads ``_case_reader`` guards. CASE_READS = (("metrics",), ("api", "cases"), ("api", "contracts"), ("api", "certificates"), ("api", "builder")) +# The largest request body read, and how long a POST refused before its body was read may take to send that body. +MAX_BODY = 1024 * 1024 +REFUSED_BODY_SECONDS = 5 def _planner_configured() -> bool: @@ -209,18 +212,62 @@ def _json(self, data, status=200, headers=None): self.send_header("X-Content-Type-Options", "nosniff") for name, value in (headers or {}).items(): self.send_header(name, value) + if self.unread_body: + self.send_header("Connection", "close") self.end_headers() self.wfile.write(body) + # True when a POST is answered with its body unread: the answer then says Connection: close. + unread_body = False + def _body(self): - length = int(self.headers.get("Content-Length", "0")) - if length > 1024 * 1024 or length < 0: + if "Content-Length" not in self.headers: + self.unread_body = True # a body of unknown length (chunked, say) is never read + try: + length = int(self.headers.get("Content-Length", "0")) + except ValueError: + self.unread_body = True + raise + if length > MAX_BODY or length < 0: + self.unread_body = True raise Rejected("payload_size", "maximum body is 1 MB") data = json.loads(self.rfile.read(length)) if not isinstance(data, dict): raise Rejected("body_type", "JSON object required") return data + def _discard_body(self): + """Read and drop the body of a POST refused before its body was read. + + Closing a socket with request bytes unread resets the connection, and on Windows the reset discards the + refusal the client has not read yet (WinError 10053). So the body is read first, up to MAX_BODY and for at + most REFUSED_BODY_SECONDS in all. A length that is missing, malformed or over MAX_BODY is not read, nor is + a body the client holds back for ``Expect: 100-continue`` (this server never sends 100): the refusal says + Connection: close instead. + """ + raw = self.headers.get("Content-Length", "").strip() + length = int(raw) if raw.isascii() and raw.isdigit() else -1 + if not 0 <= length <= MAX_BODY or self.headers.get("Expect", "").strip().lower() == "100-continue": + self.unread_body = True + return + previous = self.connection.gettimeout() + deadline = time.monotonic() + REFUSED_BODY_SECONDS + try: + while length > 0: + left = deadline - time.monotonic() + if left <= 0: + self.unread_body = True + break + self.connection.settimeout(left) + chunk = self.rfile.read1(min(length, 65536)) # at most one receive, so the deadline holds + if not chunk: + break + length -= len(chunk) + except OSError: # the client stopped sending or went away: answer anyway and close + self.unread_body = True + finally: + self.connection.settimeout(previous) + def _parts(self): return [p for p in urlsplit(self.path).path.split("/") if p] @@ -359,7 +406,12 @@ def _sse(self, chunks): def do_POST(self): try: - user, parts, data = self._user(), self._parts(), self._body() + try: + user = self._user() + except Exception: + self._discard_body() # refused before the body is read (FM-49) + raise + parts, data = self._parts(), self._body() actor, roles = actor_of(user), user.get("roles", []) # the principal id, never the display name if parts == ["api", "contracts"] and "admin" in roles: work.install_contract(data); result = {"installed": True} diff --git a/tests/test_console.py b/tests/test_console.py index f13d3eb..e6c7d64 100644 --- a/tests/test_console.py +++ b/tests/test_console.py @@ -320,19 +320,38 @@ def test_an_unauthenticated_post_whose_body_arrives_late_still_gets_its_refusal( with self.subTest(who): result = self.post({"Content-Type": "application/json", "Content-Length": str(len(self.BODY)), **headers}, self.BODY) self.assert_whole(result, 409, "unauthorized") + # On every platform, not only where the reset shows: the refusal waits for the body it then reads. + self.assertFalse(result[0], "the server read the body before answering") def test_a_refused_body_too_large_or_of_unknown_length_is_not_read_and_the_connection_is_closed(self): - too_large = str(1024 * 1024 + 1) + too_large, operator = str(1024 * 1024 + 1), {"Authorization": "Bearer " + self.cell.tokens["operator"]} cases = (({}, too_large, "unauthorized"), ({}, "abc", "unauthorized"), ({}, None, "unauthorized"), - ({"Authorization": "Bearer " + self.cell.tokens["operator"]}, too_large, "payload_size"), - ({"Authorization": "Bearer " + self.cell.tokens["operator"]}, "abc", "invalid_request")) - for auth, length, error in cases: - with self.subTest(auth=bool(auth), length=length): - headers = {**auth, **({} if length is None else {"Content-Length": length})} + ({"Expect": "100-continue"}, str(len(self.BODY)), "unauthorized"), + (operator, too_large, "payload_size"), (operator, "abc", "invalid_request"), (operator, None, "invalid_request")) + for extra, length, error in cases: + with self.subTest(headers=sorted(extra), length=length): + headers = {**extra, **({} if length is None else {"Content-Length": length})} result = self.post(headers) self.assertTrue(result[0], "answered at once, without waiting for a body it will not read") self.assertEqual(self.assert_whole(result, 409, error).get("connection"), "close") + def test_a_refused_body_that_never_arrives_is_waited_for_a_bounded_time(self): + with unittest.mock.patch("syberwork.server.REFUSED_BODY_SECONDS", 0.3): + result = self.post({"Content-Length": "100"}, wait=5) + self.assertTrue(result[0], "answered once the wait for the body ran out, not after the client gave up") + self.assertEqual(self.assert_whole(result, 409, "unauthorized").get("connection"), "close") + + def test_a_refused_body_sent_a_byte_at_a_time_is_waited_for_in_total_not_per_byte(self): + with unittest.mock.patch("syberwork.server.REFUSED_BODY_SECONDS", 0.5), \ + socket.create_connection(("127.0.0.1", self.cell.server.server_port), timeout=10) as sock: + sock.sendall(b"POST /api/cases/any/preview HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Length: 100\r\n\r\n") + started, answered = time.monotonic(), False + while not answered and time.monotonic() - started < 5: + sock.sendall(b" ") + answered = bool(select.select([sock], [], [], 0.2)[0]) + self.assertTrue(answered, "a trickling client does not hold the refusal past the total wait") + self.assertLess(time.monotonic() - started, 2.5) + class ConsoleRoutes(CellCase): def test_every_new_route_requires_a_bearer_token(self): From 9bd61239bb58ce278e22533a735ff0a0ef16dda3 Mon Sep 17 00:00:00 2001 From: sykosyber Date: Wed, 4 Nov 2026 17:32:38 -0800 Subject: [PATCH 3/3] Test that an unexpected failure resolving the token drains a late body and answers 500 whole (FM-49) do_POST drains the body when _user() raises anything, not only the unauthorized refusal. This test patches _user() on the handler class to raise RuntimeError, sends the body late, and checks the complete 500 internal_error answer arrives with no reset, after the server read the body. Against main's server it fails on Windows by reset (WinError 10053) and on every platform by the wait check. Co-Authored-By: Claude Opus 5.5 --- tests/test_console.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_console.py b/tests/test_console.py index e6c7d64..a683b23 100644 --- a/tests/test_console.py +++ b/tests/test_console.py @@ -323,6 +323,13 @@ def test_an_unauthenticated_post_whose_body_arrives_late_still_gets_its_refusal( # On every platform, not only where the reset shows: the refusal waits for the body it then reads. self.assertFalse(result[0], "the server read the body before answering") + def test_an_unexpected_failure_resolving_the_token_drains_a_late_body_and_answers_500_whole(self): + handler = self.cell.server.RequestHandlerClass + with unittest.mock.patch.object(handler, "_user", side_effect=RuntimeError("principal lookup failed")): + result = self.post({"Content-Type": "application/json", "Content-Length": str(len(self.BODY))}, self.BODY) + self.assert_whole(result, 500, "internal_error") + self.assertFalse(result[0], "the server read the body before answering") + def test_a_refused_body_too_large_or_of_unknown_length_is_not_read_and_the_connection_is_closed(self): too_large, operator = str(1024 * 1024 + 1), {"Authorization": "Bearer " + self.cell.tokens["operator"]} cases = (({}, too_large, "unauthorized"), ({}, "abc", "unauthorized"), ({}, None, "unauthorized"),