Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 55 additions & 3 deletions syberwork/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@
ATTEST_ONLY_ROLES = frozenset({"observer"})
# The first path segments of the reads ``_case_reader`` guards.
CASE_READS = (("metrics",), ("api", "cases"), ("api", "contracts"), ("api", "certificates"), ("api", "builder"))
# The largest request body read, and how long a POST refused before its body was read may take to send that body.
MAX_BODY = 1024 * 1024
REFUSED_BODY_SECONDS = 5


def _planner_configured() -> bool:
Expand Down Expand Up @@ -209,18 +212,62 @@ def _json(self, data, status=200, headers=None):
self.send_header("X-Content-Type-Options", "nosniff")
for name, value in (headers or {}).items():
self.send_header(name, value)
if self.unread_body:
self.send_header("Connection", "close")
self.end_headers()
self.wfile.write(body)

# True when a POST is answered with its body unread: the answer then says Connection: close.
unread_body = False

def _body(self):
length = int(self.headers.get("Content-Length", "0"))
if length > 1024 * 1024 or length < 0:
if "Content-Length" not in self.headers:
self.unread_body = True # a body of unknown length (chunked, say) is never read
try:
length = int(self.headers.get("Content-Length", "0"))
except ValueError:
self.unread_body = True
raise
if length > MAX_BODY or length < 0:
self.unread_body = True
raise Rejected("payload_size", "maximum body is 1 MB")
data = json.loads(self.rfile.read(length))
if not isinstance(data, dict):
raise Rejected("body_type", "JSON object required")
return data

def _discard_body(self):
"""Read and drop the body of a POST refused before its body was read.

Closing a socket with request bytes unread resets the connection, and on Windows the reset discards the
refusal the client has not read yet (WinError 10053). So the body is read first, up to MAX_BODY and for at
most REFUSED_BODY_SECONDS in all. A length that is missing, malformed or over MAX_BODY is not read, nor is
a body the client holds back for ``Expect: 100-continue`` (this server never sends 100): the refusal says
Connection: close instead.
"""
raw = self.headers.get("Content-Length", "").strip()
length = int(raw) if raw.isascii() and raw.isdigit() else -1
Comment on lines +248 to +249

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Handle oversized decimal lengths before converting them

On the project's default Python 3.11+ configuration, a Content-Length with more than 4,300 ASCII digits passes isdigit() but makes int(raw) raise ValueError. That happens before unread_body is set, so the outer handler returns invalid_request without Connection: close while request bytes may still be pending, reopening the refused-body reset path on Windows.

AGENTS.md reference: AGENTS.md:L157-L161

Useful? React with 👍 / 👎.

if not 0 <= length <= MAX_BODY or self.headers.get("Expect", "").strip().lower() == "100-continue":
self.unread_body = True
return
previous = self.connection.gettimeout()
deadline = time.monotonic() + REFUSED_BODY_SECONDS
try:
while length > 0:
left = deadline - time.monotonic()
if left <= 0:
self.unread_body = True
break
Comment on lines +259 to +260

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve refusals when the body wait expires

If an unauthenticated POST declares a valid body but sends it more slowly than the five-second limit, this branch leaves part of the body unread and the handler still sends the refusal before closing. That recreates the unread-request close that can reset the response on Windows, so the client may still lose the refusal; the byte-at-a-time test checks only that the socket becomes readable, not that the full response arrives.

AGENTS.md reference: AGENTS.md:L157-L161

Useful? React with 👍 / 👎.

self.connection.settimeout(left)
chunk = self.rfile.read1(min(length, 65536)) # at most one receive, so the deadline holds
if not chunk:
break
length -= len(chunk)
except OSError: # the client stopped sending or went away: answer anyway and close
self.unread_body = True
finally:
self.connection.settimeout(previous)

def _parts(self):
return [p for p in urlsplit(self.path).path.split("/") if p]

Expand Down Expand Up @@ -359,7 +406,12 @@ def _sse(self, chunks):

def do_POST(self):
try:
user, parts, data = self._user(), self._parts(), self._body()
try:
user = self._user()
except Exception:
self._discard_body() # refused before the body is read (FM-49)
raise
parts, data = self._parts(), self._body()
actor, roles = actor_of(user), user.get("roles", []) # the principal id, never the display name
if parts == ["api", "contracts"] and "admin" in roles:
work.install_contract(data); result = {"installed": True}
Expand Down
107 changes: 106 additions & 1 deletion tests/test_console.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import hashlib
import json
import os
import select
import shutil
import socket
import sqlite3
Expand Down Expand Up @@ -122,6 +123,37 @@ def wait_until_listening(port: int, proc: subprocess.Popen, timeout: float = 10)
raise RuntimeError("reference system did not accept connections")


def late_body_post(port: int, path: str, headers: dict, body: bytes = b"", wait: float = 1.0):
"""POST over a raw socket, holding the body back until the server answers or ``wait`` seconds pass.

``http.client`` writes the header block and the body in two sends. Under load the server can read the headers and
answer before the body lands; a server that then closes with the body unread resets the connection, and on Windows
the reset discards the answer the client had not read yet (WinError 10053). Here that gap is forced on every run.

Returns ``(answered_early, status, headers, body, error)``: whether an answer arrived before the body was sent, the
response read to the end of the stream, and the ``OSError`` that broke the exchange, or None.
"""
lines = [f"POST {path} HTTP/1.1", "Host: 127.0.0.1", "Connection: close", *(f"{k}: {v}" for k, v in headers.items())]
received, error = b"", None
with socket.create_connection(("127.0.0.1", port), timeout=10) as sock:
sock.sendall(("\r\n".join(lines) + "\r\n\r\n").encode("latin-1"))
answered_early = bool(select.select([sock], [], [], wait)[0])
if answered_early:
time.sleep(0.2) # let the server finish answering and close, as it has under load by the time the body lands
try:
if body:
sock.sendall(body)
while chunk := sock.recv(65536):
received += chunk
except OSError as exc:
error = exc
head, _, payload = received.partition(b"\r\n\r\n")
status_line, *header_lines = head.decode("latin-1").split("\r\n") if head else [""]
status = int(status_line.split()[1]) if len(status_line.split()) > 1 else None
response_headers = {name.strip().lower(): value.strip() for name, _, value in (h.partition(":") for h in header_lines)}
return answered_early, status, response_headers, payload, error


def certified_home(folder: Path, seed: bytes = SEED) -> tuple[Path, dict]:
"""A Build Thread home whose one candidate was checked, accepted and certified, as ``syberlabs certify`` leaves it.

Expand Down Expand Up @@ -264,6 +296,70 @@ def setUp(self):
self.addCleanup(self.cell.close)


class ConsoleRefusals(CellCase):
"""A POST refused before its body is read still delivers the refusal whole (FM-49).

The server answers an unauthenticated POST without reading the body. If the body lands after the answer, closing the
socket with it unread resets the connection, and on Windows the reset throws away the refusal the client had not read.
"""

BODY = json.dumps({"action": "record_review"}).encode()

def post(self, headers, body=b"", wait=1.0):
return late_body_post(self.cell.server.server_port, "/api/cases/any/preview", headers, body, wait)

def assert_whole(self, result, status, error):
_, got_status, headers, payload, broke = result
self.assertIsNone(broke, "the connection broke before the refusal was read")
self.assertEqual(int(headers.get("content-length", -1)), len(payload), "the refusal arrived whole")
self.assertEqual((got_status, json.loads(payload)["error"]), (status, error))
return headers

def test_an_unauthenticated_post_whose_body_arrives_late_still_gets_its_refusal(self):
for who, headers in (("no token", {}), ("a wrong token", {"Authorization": "Bearer not-a-token"})):
with self.subTest(who):
result = self.post({"Content-Type": "application/json", "Content-Length": str(len(self.BODY)), **headers}, self.BODY)
self.assert_whole(result, 409, "unauthorized")
# On every platform, not only where the reset shows: the refusal waits for the body it then reads.
self.assertFalse(result[0], "the server read the body before answering")

def test_an_unexpected_failure_resolving_the_token_drains_a_late_body_and_answers_500_whole(self):
handler = self.cell.server.RequestHandlerClass
with unittest.mock.patch.object(handler, "_user", side_effect=RuntimeError("principal lookup failed")):
result = self.post({"Content-Type": "application/json", "Content-Length": str(len(self.BODY))}, self.BODY)
self.assert_whole(result, 500, "internal_error")
self.assertFalse(result[0], "the server read the body before answering")

def test_a_refused_body_too_large_or_of_unknown_length_is_not_read_and_the_connection_is_closed(self):
too_large, operator = str(1024 * 1024 + 1), {"Authorization": "Bearer " + self.cell.tokens["operator"]}
cases = (({}, too_large, "unauthorized"), ({}, "abc", "unauthorized"), ({}, None, "unauthorized"),
({"Expect": "100-continue"}, str(len(self.BODY)), "unauthorized"),
(operator, too_large, "payload_size"), (operator, "abc", "invalid_request"), (operator, None, "invalid_request"))
for extra, length, error in cases:
with self.subTest(headers=sorted(extra), length=length):
headers = {**extra, **({} if length is None else {"Content-Length": length})}
result = self.post(headers)
self.assertTrue(result[0], "answered at once, without waiting for a body it will not read")
self.assertEqual(self.assert_whole(result, 409, error).get("connection"), "close")

def test_a_refused_body_that_never_arrives_is_waited_for_a_bounded_time(self):
with unittest.mock.patch("syberwork.server.REFUSED_BODY_SECONDS", 0.3):
result = self.post({"Content-Length": "100"}, wait=5)
self.assertTrue(result[0], "answered once the wait for the body ran out, not after the client gave up")
self.assertEqual(self.assert_whole(result, 409, "unauthorized").get("connection"), "close")

def test_a_refused_body_sent_a_byte_at_a_time_is_waited_for_in_total_not_per_byte(self):
with unittest.mock.patch("syberwork.server.REFUSED_BODY_SECONDS", 0.5), \
socket.create_connection(("127.0.0.1", self.cell.server.server_port), timeout=10) as sock:
sock.sendall(b"POST /api/cases/any/preview HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Length: 100\r\n\r\n")
started, answered = time.monotonic(), False
while not answered and time.monotonic() - started < 5:
sock.sendall(b" ")
answered = bool(select.select([sock], [], [], 0.2)[0])
self.assertTrue(answered, "a trickling client does not hold the refusal past the total wait")
self.assertLess(time.monotonic() - started, 2.5)


class ConsoleRoutes(CellCase):
def test_every_new_route_requires_a_bearer_token(self):
for path in ("/api/console", "/api/contracts", "/api/contracts/purchase-order/1", "/api/policies", "/api/actions", "/api/sources",
Expand Down Expand Up @@ -1971,7 +2067,16 @@ def test_preview_appends_nothing_for_anyone(self):
{"action": "absent", "args": {}}, {"action": "absent"}, {"args": {}}, {"action": "record_review", "args": []})
for body in bodies:
for who in ("operator", "scheduler", "manager", "planner", "supplier", None):
self.cell.call(f"/api/cases/{case}/preview", body, who=who)
try:
status, _, raw = self.cell.call(f"/api/cases/{case}/preview", body, who=who)
except OSError as exc:
self.fail(f"preview {body} as {who}: the connection broke before the answer was read ({exc!r})")
answer = json.loads(raw)
if who is None:
self.assertEqual((status, answer["error"]), (409, "unauthorized"), body)
else:
self.assertIn(status, (200, 409), (body, who, answer))
self.assertNotEqual(answer.get("error"), "internal_error", (body, who))
self.assertEqual(self.cell.work.inspect(case)["events"], events)
self.assertEqual(self.dump(), before, "no table changed, including event_side and reservations")
self.assertTrue(self.cell.work.verify_chain(case))
Expand Down
Loading