From 78c116c90fc9c0262e3c6906d96ddf2c176a201c Mon Sep 17 00:00:00 2001 From: SyberLabs candidate Date: Wed, 28 Oct 2026 15:14:33 -0700 Subject: [PATCH] patch: jev-api-2: v3 measurement plan with sourced cost ceiling, crossover range, wall-clock deadline per Jev decision, response_digest documented, closed-word check without assert SyberLabs-Thread: 7ca8f3c4-534e-492e-bc25-2f5440e2146b SyberLabs-Candidate: c1 SyberLabs-Provider: claude@opus-5.5 --- docs/EVOLUTION.md | 26 ++++++++++++++++---------- syberlabs/jev_api.py | 41 ++++++++++++++++++++++++++++++++++++----- tests/test_jev_api.py | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 15 deletions(-) diff --git a/docs/EVOLUTION.md b/docs/EVOLUTION.md index 9f46716..d0e0d9d 100644 --- a/docs/EVOLUTION.md +++ b/docs/EVOLUTION.md @@ -167,9 +167,9 @@ Simulated, and nothing about a model: the profiles were chosen so the policies c **What an answer must be.** Before it becomes an action: status 200; one strict JSON object (a duplicate key or `NaN` is refused) of at most 64 KiB; `model` exactly the pin; `answers` holding exactly `next_action`, of type `choice`; the choice one of the options sent; `probabilities` covering exactly the options sent, each finite in [0, 1], summing to 1 within **0.02**; the choice carrying the highest probability; `confidence` finite in [0, 1]; `usage.input_tokens` and `usage.output_tokens` non-negative integers. The tolerance is measured, not assumed: on 2026-10-28 every live answer gave probabilities at two decimals whose sum was off 1 by at most 0.01 on three options (rounding), so four options can be off by 0.02. -**When it is not.** A timeout (30 s by default, at most the 120 s of `docs/AUTOSKILL.md`) is `jev_timeout`; no key, a non-200 or a transport failure is `jev_unavailable`; a body that fails a check is `jev_invalid`; a choice outside the options is `choice_outside_options`. Each takes the local `JevPolicy`'s action and records `fallback: {reason, detail}` with `detail` from a closed list (`http_401`, `http_429`, `model`, `probabilities`, ...), never response text, a header or an exception message. Nothing is retried. Three failures in a row set `switched_to_local` and the rest of that search is local without a call (`fallback: {reason: local_only}`); a success resets the count, and the next search starts with Jev again. With every call failing, a search is decision for decision the local policy's search (tested). +**When it is not.** A decision waits for Jev at most `timeout` seconds of wall clock (30 s by default, at most the 120 s of `docs/AUTOSKILL.md`): the call runs on a daemon thread, and when it is not back by then the decision falls back with `jev_timeout` (detail `deadline`). The thread is abandoned, not cancelled, so a late answer is discarded and may still be billed. The same value is also urllib's per-socket-operation timeout inside the call (detail `timeout`). A timeout is `jev_timeout`; no key, a non-200 or a transport failure is `jev_unavailable`; a body that fails a check is `jev_invalid`; a choice outside the options is `choice_outside_options`. Each takes the local `JevPolicy`'s action and records `fallback: {reason, detail}` with `detail` from a closed list (`http_401`, `http_429`, `model`, `probabilities`, ...), never response text, a header or an exception message. Nothing is retried. Three failures in a row set `switched_to_local` and the rest of that search is local without a call (`fallback: {reason: local_only}`); a success resets the count, and the next search starts with Jev again. With every call failing, a search is decision for decision the local policy's search (tested). -**What is recorded.** Every decision entry carries, beside `JevPolicy`'s audit fields: `source` (`jev_api`, `local`, or `rule` for a stop), `model`, `answers_digest` (SHA-256 of the canonical `answers` object), `request_digest` (SHA-256 of the bytes sent), `latency_ms`, `fallback`, `failures` (in a row) and `local_action`, the kind the local policy would have taken, so agreement is countable decision by decision; a Jev decision also carries `jev: {choice, probabilities, confidence, usage, dollars}` and `chosen_by: jev_api`. The key is sent in the header and nowhere else: not in a trace entry, a signal, an exception or the policy object (tested with a planted key through `http_transport` and a stand-in opener). As for every policy, the trace is in memory and the journal holds each child's `signal.policy`; a provider cannot journal a decision of its own (`spec/GAPS.md` 61), so the `evolve_jevapi` arm writes the decision entries beside each exported journal. +**What is recorded.** Every decision entry carries, beside `JevPolicy`'s audit fields: `source` (`jev_api`, `local`, or `rule` for a stop), `model`, `answers_digest` (SHA-256 of the canonical `answers` object), `request_digest` (SHA-256 of the bytes sent), `latency_ms`, `fallback` (with `response_digest`, the SHA-256 of the raw response body, when a body came back and failed: a digest, so a 422 that echoes the request leaks nothing), `failures` (in a row) and `local_action`, the kind the local policy would have taken, so agreement is countable decision by decision; a Jev decision also carries `jev: {choice, probabilities, confidence, usage, dollars}` and `chosen_by: jev_api`. The key is sent in the header and nowhere else: not in a trace entry, a signal, an exception or the policy object (tested with a planted key through `http_transport` and a stand-in opener). As for every policy, the trace is in memory and the journal holds each child's `signal.policy`; a provider cannot journal a decision of its own (`spec/GAPS.md` 61), so the `evolve_jevapi` arm writes the decision entries beside each exported journal. **Measured live on 2026-10-28** (28 calls, synthetic `Situation` states of ids and numbers, from Windows; redacted transcripts kept outside the repository; the 200 bodies and error bodies used by the tests are in `tests/fixtures/jev_api/` with request ids and cookies removed): @@ -179,24 +179,30 @@ Simulated, and nothing about a model: the profiles were chosen so the policies c | Response | `{model, answers: {next_action: {type, choice, confidence, probabilities}}, usage: {input_tokens, output_tokens}}`; no request id in the body; header `x-typesafe-request-id` | | Model | `jev-1.13.0` is served as `jev-1.13.0`; `jev-1.14.0` (released 2026-10-14) and `jev-latest` are served as `jev-1.14.0`; `jev-1.13` and `jev-9.9.9` are 400 `Unknown model`; `GET /v1/models` lists only the aliases `jev-latest` and `jev-preview` | | Latency | 20 answered calls: 142 to 338 ms, median 170 ms, about 705 to 760 input tokens each | -| Probabilities | two decimals (with float artifacts such as `0.6799999999999999`); sums 0.99 to 1.00; `confidence` is not the top probability (0.37 against 0.58) | +| Probabilities | two decimals (with float artifacts such as `0.6799999999999999`); sums 0.99 to 1.00; `confidence` is not the top probability but equals (p_max - 1/k)/(1 - 1/k) on all 20 answers, to rounding (0.37 against 0.58 on three options) | | Repeatability | the same request to `jev-1.13.0` three times gave `mutate` at 0.58, 0.56 and 0.53: not deterministic | | Errors | bad key 401, no key 403 (both `{detail: {error_type: authentication_error, message}}`), unknown model 400, `NaN` in the body 400, missing `criteria` 422 with the offending input echoed back; a 50 ms client timeout raised a socket timeout | | Limits, usage | no rate-limit or usage headers; TypeSafe documents per-organization limits in tokens and requests per second, 429 when over, 529 when overloaded, 64k tokens per request and 32k per question | | Price | $0.042 per million input tokens, output free (docs.typesafe.ai/models, read 2026-10-28): about $0.00003 a decision; the 28 calls cost about $0.0006 | | Data terms | docs.typesafe.ai/legal points to a Data Processing Agreement and a Privacy Policy that commits not to train on user data; retention periods are not stated on that page; zero data retention is offered to enterprise customers | -What the calls say about the choice, and no more: on these synthetic states Jev chose `mutate` or `evaluate` every time and gave `crossover` 0.02 to 0.15, where the local policy prices a crossover (no model call) far below a mutation. Whether that is better or worse on a real fixture is what the measurement below is for. +What the calls say about the choice, and no more: on these synthetic states Jev chose `mutate` or `evaluate` every time; it gave `crossover` 0.02 to 0.15 on the 19 three- and four-option answers and 0.37 on the one two-option answer (`crossover` against `mutate`, `tests/fixtures/jev_api/choice_two_options.json`), where the local policy prices a crossover (no model call) far below a mutation. Whether that is better or worse on a real fixture is what the measurement below is for. -**Measurement plan (pre-registered 2026-10-28; not run).** It continues the `jev-policy` measurement and needs the owner's go through the lead, because it spends on Anthropic's API. +**Measurement plan (pre-registered 2026-10-28, re-targeted to v3 the same day; not run).** It continues the `jev-policy` measurement and needs the owner's go through the lead, because it spends on Anthropic's API. -- *Arms*: `single`, `evolve` (`FixedPolicy` revision 2), `evolve_jev` (local `JevPolicy`) and `evolve_jevapi` (`TypeSafeJevPolicy`, `jev-1.13.0`), through `benchmarks/model_vs_patch.py` and the same Build Thread; K = 6 model calls per run; Claude Sonnet 5.5 at medium effort (`SYBERLABS_MODEL=claude-sonnet-5-5`, `SYBERLABS_EFFORT=medium`); the four v2 fixtures (`crlf_export`, `oauth_state`, `settlement`, `tool_path`); one repeat, then three. Command: `PYTHONPATH=. python benchmarks/model_vs_patch.py --adapter " adapters/anthropic_adapter.py" --calls 6 --repeats 3 --fixtures crlf_export oauth_state settlement tool_path --arms single evolve evolve_jev evolve_jevapi --write `, with `ANTHROPIC_API_KEY` and `TYPESAFE_API_KEY` in that shell only. +- *Set*: the four **v3** fixtures (`gaps_renumber`, `journal_chain`, `worktree_order`, `verify_layers`). v2 is uninformative at this model and at Haiku 5.5, and the paid Jev-against-fixed comparison goes to v3 by the owner's decision (`docs/MODEL_COMPARISON.md`, "Where that run goes"); v4 is the fallback. +- *Arms*: `single`, `evolve` (`FixedPolicy` revision 2), `evolve_jev` (local `JevPolicy`) and `evolve_jevapi` (`TypeSafeJevPolicy`, `jev-1.13.0`), through `benchmarks/model_vs_patch.py` and the same Build Thread; K = 6 model calls per run; Claude Sonnet 5.5 at medium effort (`SYBERLABS_MODEL=claude-sonnet-5-5`, `SYBERLABS_EFFORT=medium`), with `ANTHROPIC_API_KEY` and `TYPESAFE_API_KEY` in that shell only. +- *Repeats: four are paid, three decide.* First a pilot, `--repeats 1` (seed 1). If it is informative (below), the measurement, `--repeats 3` (seeds 1 to 3, so seed 1 runs again: the harness has no seed offset). The rules read only the three repeats of the measurement; the pilot is reported beside them and counted in the cost. Command: `PYTHONPATH=. python benchmarks/model_vs_patch.py --adapter " adapters/anthropic_adapter.py" --calls 6 --repeats <1, then 3> --fixtures gaps_renumber journal_chain worktree_order verify_layers --arms single evolve evolve_jev evolve_jevapi --write `. - *Recorded per run*: solved, model calls, tokens, `dollars` (Anthropic, from the adapter's usage and the price table) and `host_dollars`, evaluations, and for `evolve_jevapi` the row's `jev_api` (Jev calls, decisions by source, fallbacks by reason, agreement with `local_action`, Jev's input tokens and dollars, median latency) with the decision entries beside each journal. -- *Decision rule carried over, verbatim from the `jev-policy` spec*: "The decision rule for Jev: it is kept as the provider's default if, on the v2 fixtures, `evolve_jev` solves at least as many as `evolve` with at most half the dollars, over the three repeats; otherwise it stays available and not default." -- *New decision rule, `evolve_jevapi` against `evolve_jev`*: TypeSafe's Jev replaces the local heuristic as the chooser behind `evolve_jev` only if, over the same three repeats on v2, `evolve_jevapi` solves at least as many runs as `evolve_jev` **and** its dollars (Anthropic dollars plus Jev's dollars) are at most `evolve_jev`'s. Neither arm becomes the default proposer path by this rule: `repair` stays the default (owner's decision of 2026-10-23). +- *Decision rule carried over, verbatim from the `jev-policy` spec*: "The decision rule for Jev: it is kept as the provider's default if, on the v2 fixtures, `evolve_jev` solves at least as many as `evolve` with at most half the dollars, over the three repeats; otherwise it stays available and not default." It is applied on v3, where the owner's decision moved this comparison. For reference, not as a prediction: on the v3 rows of 2026-10-22 (Sonnet 5.5, medium, K = 6, three repeats, with mutate feedback) `evolve_jev` solved 12 of 12 for $0.373 against `evolve`'s 12 of 12 for $0.469, 80% of the dollars rather than half. +- *New decision rule, `evolve_jevapi` against `evolve_jev`*: TypeSafe's Jev replaces the local heuristic as the chooser behind `evolve_jev` only if, over the same three repeats on v3, `evolve_jevapi` solves at least as many runs as `evolve_jev` **and** its dollars (Anthropic dollars plus Jev's dollars) are at most `evolve_jev`'s. Neither arm becomes the default proposer path by this rule: `repair` stays the default (owner's decision of 2026-10-23). - *When the comparison counts as not measured*: if more than 10% of the decisions on which Jev was asked fell back (any reason), or any run switched to local, the `evolve_jevapi` rows measured mostly the local policy and the new rule is not applied; if `evolve_jevapi` agreed with `local_action` on every asked decision, the two arms took the same actions and no difference may be claimed from them. -- *When v2 cannot decide*: v2 was solved single-shot at this model (16 of 16, 2026-10-21; `docs/MODEL_COMPARISON.md`). If `single` solves every v2 fixture in the first repeat, every evolve arm stops at its first candidate, both rules are reported as uninformative and no default changes; the next set is v3 or v4 (`evals/tasks`), a choice for the owner. The plan is written for v2 because that is the measurement the owner asked to continue. -- *Cost estimate* (estimate, not a quote): the v2 pilot spent about $0.0086 per Sonnet 5.5 call ($0.24 for 28 calls). Ceiling: 4 fixtures x 3 repeats x (1 call for `single` + 6 for each of the three evolve arms) = 228 calls, about **$2.0**, and about **$2.5** with mutate feedback lengthening prompts; expected well under $1 if v2 is again solved at the first candidate. Jev: at most about 15 asked decisions per `evolve_jevapi` run, 12 runs, about 750 input tokens each, about 135,000 tokens, about **$0.006** at $0.042 per million (180 calls at most). Budget line asked for: $3 Anthropic, $0.01 Jev, against the program cap. +- *When v3 cannot decide*: v3 is the set on which `single` failed most runs on 2026-10-22 (3 of 12 solved at Sonnet 5.5) while every evolve arm solved 11 or 12 of 12. If, in the pilot, `single` solves all four v3 fixtures, or every `evolve*` run solves at its first candidate so that Jev is never asked a decision with two offered actions, the set cannot tell the policies apart: both rules are reported as uninformative, no default changes, and the measurement moves to the v4 fixtures (`stack_landing`, `certificate_layers`, `cell_metrics`, `thread_stations`) with the same arms, K and model, on the owner's go. +- *Cost: expected about **$1.90**, hard ceiling **$5.40** Anthropic; Jev expected about $0.005, ceiling $0.15.* The source is the E3 v3 rows of 2026-10-22 at Claude Sonnet 5.5, medium, K = 6 (`reports/receipts/2026-10-22-e3-v3-feedback-r3-jev-decision`, `-baseeval-r1` and `-single-probe` in the officer workspace; summarized in `docs/MODEL_COMPARISON.md`). Those rows carry the mutate feedback and the base evaluation, so their calls are the feedback calls this plan makes. + - Expected: the mean dollars per run in the three-repeat feedback rows: `single` $0.0101, `evolve` $0.0391, `evolve_jev` $0.0311, and `evolve_jevapi` taken at `evolve`'s $0.0391, because Jev leaned to mutation in discovery. That is $0.119 per fixture per repeat, $0.477 per repeat, and **$1.91** for the four paid repeats. + - Hard ceiling: every run spends its whole budget, 1 call for `single` and K = 6 for each of the three evolve arms, so 19 calls per fixture, 76 per repeat and **304 calls** for four repeats. Each call is priced at **$0.0176**, the highest dollars per call of any of the 84 Sonnet 5.5 v3 rows (`evolve` on `gaps_renumber`, $0.0353 for two calls). 304 x $0.0176 = **$5.36**, stated as $5.40. Nothing in the harness stops the total at that number: K bounds each run's calls, and the operator stops the run if the rows' dollars pass it. + - Jev: asked decisions are not bounded by K, because a Jev-chosen crossover uses no model call. The code bound per `evolve_jevapi` run is K mutate decisions plus one decision per generation, with the harness's 200 generations, so at most 206 calls. That makes at most 16 runs x 206 = **3,296 calls**, at no more than 1,000 input tokens each, and **$0.14** at $0.042 per million. Expected: about 10 asked decisions per run (the v3 `evolve_jev` runs took at most 5 model calls), 16 runs and about 750 tokens each, about $0.005. + - Budget line for the owner: **$5.40 Anthropic hard ceiling** (expected $1.90) and **$0.15 Jev**. - *Not expressible as a SyberEval spec yet*: `syberlabs/eval/spec.py` closes `ARM_KINDS` to `single`, `repair`, `evolve`, `evolve_jev` and `claude-code`, so neither `evolve_jevapi` nor a policy choice can be registered under `evals/`; this plan is the pre-registration (`spec/GAPS.md` 61). ### Measured on a toy fixture diff --git a/syberlabs/jev_api.py b/syberlabs/jev_api.py index 1876da6..68abb58 100644 --- a/syberlabs/jev_api.py +++ b/syberlabs/jev_api.py @@ -30,7 +30,13 @@ the search (``local_only``); a failure is never retried. Every decision's trace entry carries ``source`` (``jev_api``, ``local`` or ``rule``), ``model``, ``answers_digest``, ``request_digest``, ``latency_ms``, ``fallback`` and ``local_action``, the action the local -policy would have taken, so the two can be compared decision by decision. +policy would have taken, so the two can be compared decision by decision. A ``fallback`` for a +body that came back and failed also carries ``response_digest``, the SHA-256 of the raw body. + +The timeout is a wall-clock deadline per decision: the call runs on a daemon thread and the +decision falls back (``jev_timeout``, detail ``deadline``) when it is not back in ``timeout`` +seconds. The thread is abandoned, not cancelled; a late answer is discarded. The same value is +urllib's per-socket-operation timeout inside the call (detail ``timeout``). """ from __future__ import annotations @@ -40,6 +46,7 @@ import math import os import re +import threading import time import urllib.error import urllib.request @@ -55,7 +62,7 @@ KEY_ENV = "TYPESAFE_API_KEY" MODEL = "jev-1.13.0" # the pin docs/AUTOSKILL.md names; jev-1.14.0 (2026-10-14) is jev-latest since then PIN = re.compile(r"^jev-\d+\.\d+\.\d+$") # an exact version; the aliases jev-latest and jev-preview are refused -TIMEOUT = 30.0 # seconds; the 2026-10-28 calls took 0.14 to 0.34 s for about 750 input tokens +TIMEOUT = 30.0 # seconds of wall clock per decision; the 2026-10-28 calls took 0.14 to 0.34 s for about 750 input tokens MAX_TIMEOUT = 120.0 # docs/AUTOSKILL.md's adapter timeout, the most this policy accepts TOLERANCE = 0.02 # |sum of probabilities - 1|; two-decimal probabilities on up to four options MAX_FAILURES = 3 # failures in a row before the policy is local for the rest of the search @@ -76,7 +83,7 @@ SOURCES = ("jev_api", "local", "rule") REASONS = ("jev_timeout", "jev_unavailable", "jev_invalid", "choice_outside_options", "local_only") # Why a call failed, as a closed word: never response text, a header or an exception message. -DETAILS = ("no_key", "timeout", "transport", "http_401", "http_403", "http_429", "http_529", "http_4xx", "http_5xx", +DETAILS = ("no_key", "timeout", "deadline", "transport", "http_401", "http_403", "http_429", "http_529", "http_4xx", "http_5xx", "http_other", "too_large", "not_json", "not_object", "model", "answers", "type", "choice", "probabilities", "argmax", "confidence", "usage", "request_too_large", "switched") @@ -85,7 +92,8 @@ class JevFailure(Exception): """A call that did not yield a usable answer. ``reason`` and ``detail`` are closed words; nothing else is kept.""" def __init__(self, reason: str, detail: str, *, latency_ms: float | None = None, response_digest: str | None = None): - assert reason in REASONS and detail in DETAILS, (reason, detail) + if reason not in REASONS or detail not in DETAILS: # a real check: it holds under python -O + raise ValueError("a JevFailure takes a reason from REASONS and a detail from DETAILS") super().__init__(f"{reason}: {detail}") self.reason, self.detail, self.latency_ms, self.response_digest = reason, detail, latency_ms, response_digest @@ -272,10 +280,33 @@ def request(self, s: Situation, offered: Sequence[tuple[Action, dict]]) -> dict: "questions": {QUESTION: {"type": "choice", "instructions": INSTRUCTIONS, "criteria": {kind: CRITERIA[kind] for kind in kinds}}}} + def _within_deadline(self, body: bytes): + """The transport's answer if it is back within ``timeout`` seconds of wall clock, else ``jev_timeout``. + + The call runs on a daemon thread; past the deadline it is abandoned (Python cannot cancel it) + and whatever it returns later is discarded. An exception it raises is raised here unchanged. + """ + box: dict = {} + + def call(): + try: + box["answer"] = self.transport(body, self.timeout) + except BaseException as exc: # noqa: BLE001 - handed to the deciding thread unchanged + box["error"] = exc + + worker = threading.Thread(target=call, name="jev-api-call", daemon=True) + worker.start() + worker.join(self.timeout) + if worker.is_alive(): + raise JevFailure("jev_timeout", "deadline") + if "error" in box: + raise box["error"] + return box["answer"] + def _ask(self, body: bytes, kinds: Sequence[str]) -> tuple[dict, float]: started = time.perf_counter() try: - status, raw = self.transport(body, self.timeout) + status, raw = self._within_deadline(body) except JevFailure as failure: failure.latency_ms = round((time.perf_counter() - started) * 1000, 1) raise diff --git a/tests/test_jev_api.py b/tests/test_jev_api.py index adda403..82aedc6 100644 --- a/tests/test_jev_api.py +++ b/tests/test_jev_api.py @@ -20,6 +20,8 @@ import random import re import tempfile +import threading +import time import unittest import urllib.error from pathlib import Path @@ -341,6 +343,43 @@ def test_each_failure_falls_back_to_the_local_action_with_its_reason(self): self.assertIsNotNone(entry["latency_ms"]) self.assertEqual(provider.trace[-1]["action"], "stop") + def test_a_slow_answer_falls_back_at_the_wall_clock_deadline(self): + # An answer that would arrive long after the deadline, as from a server that drips its bytes: the + # per-socket timeout would not end it, the decision's deadline does. + release = threading.Event() + + def slow(body, timeout): + release.wait(10) + return 200, raw(response("choice_two_options")) + + policy = TypeSafeJevPolicy(generations=60, transport=slow, timeout=0.2) + policy.reset() + started = time.perf_counter() + try: + action = policy.decide(situation()) + finally: + release.set() + elapsed = time.perf_counter() - started + audit = policy.audit() + self.assertEqual((audit["fallback"]["reason"], audit["fallback"]["detail"]), ("jev_timeout", "deadline")) + self.assertEqual((audit["source"], action.kind), ("local", audit["local_action"])) + self.assertLess(elapsed, 3.0, "the decision did not wait for the answer") + self.assertGreaterEqual(audit["latency_ms"], 150) + self.assertIn("deadline", DETAILS) + # An answer inside the deadline is used as before. + quick = TypeSafeJevPolicy(generations=60, transport=lambda body, timeout: (200, raw(response("choice_two_options"))), + timeout=0.2) + quick.reset() + quick.decide(situation()) + self.assertEqual(quick.audit()["source"], "jev_api") + + def test_a_failure_takes_only_closed_words_without_relying_on_assert(self): + with self.assertRaises(ValueError): + JevFailure("jev_unavailable", "some response text") + with self.assertRaises(ValueError): + JevFailure("anything", "transport") + self.assertEqual(JevFailure("jev_timeout", "deadline").detail, "deadline") + def test_no_key_falls_back_without_opening_a_socket(self): opener = Opener("ok", raw(response("choice_two_options"))) with KeyEnv(None):