diff --git a/benchmarks/model_vs_patch.py b/benchmarks/model_vs_patch.py index 12af3fb..f1c1e3e 100644 --- a/benchmarks/model_vs_patch.py +++ b/benchmarks/model_vs_patch.py @@ -14,7 +14,14 @@ - ``evolve_empirical``: the same provider, mutator, population, generations and K, branching by ``EmpiricalPolicy`` (the fixed schedule with the round kind learned by Thompson sampling); - ``evolve_jev``: the same, branching by ``JevPolicy`` (each action bought by expected admission per - dollar; its price prior comes from the contract's ``budget.prices`` when present). + dollar; its price prior comes from the contract's ``budget.prices`` when present); +- ``evolve_jevapi``: the same, branching by ``TypeSafeJevPolicy`` (``syberlabs.jev_api``): ``JevPolicy``'s + offered actions chosen by TypeSafe's Jev API, with ``JevPolicy`` as the fallback. It calls + ``https://api.typesafe.ai/v1/systemone`` with ``TYPESAFE_API_KEY`` from the environment, so it is not + among the default arms: name it with ``--arms`` to run it. Its row adds ``jev_api`` (Jev's calls, + decisions by source, fallbacks by reason, agreement with the local choice, input tokens, Jev's dollars + at TypeSafe's published input rate, and the median latency); Jev's dollars are outside the host's + ``host_dollars``, which observes the model adapter only. The three ``evolve*`` arms differ in the branch policy and in nothing else, so a difference between them is policy against policy, not "fill the population first" @@ -63,11 +70,14 @@ from syberlabs import Kit, Rejected # noqa: E402 from benchmarks.simulated_model import PROFILES # noqa: E402 from syberlabs.evolve import CommandMutator, EmpiricalPolicy, EvolutionaryProvider, JevPolicy # noqa: E402 +from syberlabs.jev_api import REASONS as JEV_REASONS, SOURCES as JEV_SOURCES, TypeSafeJevPolicy # noqa: E402 from syberlabs.providers import model_usage, run_json, split_command, tool_usage # noqa: E402 SIMULATED = f"{sys.executable} {ROOT / 'benchmarks' / 'simulated_model.py'}" -ARMS = ("single", "best_of_k", "repair", "evolve", "evolve_empirical", "evolve_jev") -EVOLVE_ARMS = {"evolve": "fixed", "evolve_empirical": "empirical", "evolve_jev": "jev"} +ARMS = ("single", "best_of_k", "repair", "evolve", "evolve_empirical", "evolve_jev") # the default arms +JEV_API_ARM = "evolve_jevapi" # a live TypeSafe call per decision: run only when named +ALL_ARMS = ARMS + (JEV_API_ARM,) +EVOLVE_ARMS = {"evolve": "fixed", "evolve_empirical": "empirical", "evolve_jev": "jev", JEV_API_ARM: "jev_api"} # Dollars per million tokens: input, output. Cache reads bill at 0.1x input, cache writes at 1.25x. PRICES = {"claude-opus-5": (5.0, 25.0), "claude-opus-5-5": (4.0, 20.0), "claude-opus-4-8": (5.0, 25.0), "claude-sonnet-5": (2.0, 10.0), "claude-sonnet-5-5": (2.0, 10.0), "claude-haiku-4-5": (1.0, 5.0), @@ -155,10 +165,26 @@ def search(self, space): return [] +def jev_summary(policy: TypeSafeJevPolicy, trace: list[dict]) -> dict: + """The ``evolve_jevapi`` row's account of Jev: from the policy's counters and its decision entries.""" + decisions = [e for e in trace if "decision" in e] + asked = [e for e in decisions if e.get("source") == "jev_api"] + fallbacks = [e["fallback"]["reason"] for e in decisions if e.get("fallback")] + latencies = sorted(e["latency_ms"] for e in decisions if e.get("latency_ms") is not None) + return {"model": policy.model, "calls": policy.calls, + "by_source": {s: sum(1 for e in decisions if e.get("source") == s) for s in JEV_SOURCES}, + "fallbacks": {r: fallbacks.count(r) for r in JEV_REASONS if r in fallbacks}, + "switched_to_local": any((e.get("fallback") or {}).get("switched_to_local") for e in decisions), + "agreed": sum(1 for e in asked if e["action"] == e["local_action"]), "asked": len(asked), + "input_tokens": policy.input_tokens, "dollars": policy.jev_dollars, + "latency_ms_median": statistics.median(latencies) if latencies else None} + + def run(fixture: str, arm: str, adapter: str, calls: int, seed: int, folder: Path, - journals: Path | None = None, profile: str | None = None) -> dict: + journals: Path | None = None, profile: str | None = None, jev_transport=None) -> dict: """One run. ``journals`` is a directory to export the thread's journal into, or None to keep nothing. - ``profile`` names a simulated profile (``SIM_PROFILE`` for the adapter) or clears it.""" + ``profile`` names a simulated profile (``SIM_PROFILE`` for the adapter) or clears it. ``jev_transport`` + replaces the ``evolve_jevapi`` arm's HTTP call (``syberlabs.jev_api.http_transport``), for tests.""" if profile is not None and profile not in PROFILES: raise ValueError(f"profile must be one of {sorted(PROFILES)}") if profile is None: @@ -195,9 +221,12 @@ def run(fixture: str, arm: str, adapter: str, calls: int, seed: int, folder: Pat elif EVOLVE_ARMS[arm] == "empirical": provider = EvolutionaryProvider(mutator, **shape, policy=EmpiricalPolicy(**shape, stop_on_first_pass=True, salt=seed)) - else: + elif EVOLVE_ARMS[arm] == "jev": provider = EvolutionaryProvider(mutator, **shape, policy=JevPolicy(generations=shape["generations"], prices=jev_prices(doc))) + else: + provider = EvolutionaryProvider(mutator, **shape, policy=TypeSafeJevPolicy( + generations=shape["generations"], prices=jev_prices(doc), transport=jev_transport)) thread.propose(provider, seed=seed) used, usage = mutator.calls, mutator.usage policy = next((dict(e["policy"]) for e in provider.trace if "decision" in e), None) @@ -211,6 +240,7 @@ def run(fixture: str, arm: str, adapter: str, calls: int, seed: int, folder: Pat os.chdir(ROOT) used, usage = provider.used, provider.usage seconds = time.monotonic() - started + jev = jev_summary(provider.policy, provider.trace) if arm == JEV_API_ARM else None passing = [c for c in thread.candidates() if c.evaluation == "passed"] solved = bool(passing) and thread.check(passing[0].id, run=False).acceptable events = thread.history() @@ -220,12 +250,16 @@ def run(fixture: str, arm: str, adapter: str, calls: int, seed: int, folder: Pat journal = f"{journals.name}/{fixture}-{arm}-{seed}.json" (journals / f"{fixture}-{arm}-{seed}.json").write_text(json.dumps(kit.export(thread.id), indent=1) + "\n", encoding="utf-8") + if jev is not None: # the provider's trace is not journaled; Jev's decision entries are kept beside it + jev["decisions"] = f"{journals.name}/{fixture}-{arm}-{seed}.decisions.json" + (journals / f"{fixture}-{arm}-{seed}.decisions.json").write_text( + json.dumps([e for e in provider.trace if "decision" in e], indent=1) + "\n", encoding="utf-8") kit.close() simulated = any(u.get("simulated") for u in usage) served = sorted({u.get("model") for u in usage if u.get("model")}) finished = next((e["body"] for e in reversed(events) if e["kind"] == "search_finished"), None) spend = (finished or {}).get("spend") or {} - return {"fixture": fixture, "arm": arm, "seed": seed, "solved": solved, "calls": used, + row = {"fixture": fixture, "arm": arm, "seed": seed, "solved": solved, "calls": used, "model": "simulated" if simulated else ("+".join(served) if served else None), "input_tokens": sum(u.get("input_tokens", 0) for u in usage), "output_tokens": sum(u.get("output_tokens", 0) for u in usage), @@ -236,6 +270,9 @@ def run(fixture: str, arm: str, adapter: str, calls: int, seed: int, folder: Pat "candidates": sum(1 for e in events if e["kind"] == "candidate_registered"), "evaluations": sum(1 for e in events if e["kind"] == "candidate_evaluated"), "seconds": round(seconds, 2), "journal": journal} + if jev is not None: + row["jev_api"] = jev + return row def summarize(rows: list[dict], calls: int) -> str: @@ -247,7 +284,7 @@ def summarize(rows: list[dict], calls: int) -> str: "fixture arm solved calls(med) tokens in/out (mean) evaluations(med) $ (mean) " "host$ (mean) model policy provider_error"] for fixture in dict.fromkeys(r["fixture"] for r in rows): - for arm in ARMS: + for arm in ALL_ARMS: group = [r for r in rows if r["fixture"] == fixture and r["arm"] == arm] if not group: continue @@ -265,6 +302,18 @@ def summarize(rows: list[dict], calls: int) -> str: f"{'+'.join(models):21} " f"{('+'.join(policies) if policies else '-'):13} " f"{(str(len(errors)) + '/' + str(len(group)) + ' ' + ','.join(sorted(set(errors)))) if errors else 'none':}") + jev_rows = [r for r in rows if r.get("jev_api")] + if jev_rows: + lines.append("fixture jev model calls asked agreed fallbacks switched input_tokens jev$ (sum) latency_ms(med)") + for fixture in dict.fromkeys(r["fixture"] for r in jev_rows): + group = [r["jev_api"] for r in jev_rows if r["fixture"] == fixture] + medians = [g["latency_ms_median"] for g in group if g["latency_ms_median"] is not None] + lines.append(f"{fixture:11} {group[0]['model']:12} {sum(g['calls'] for g in group):>5} " + f"{sum(g['asked'] for g in group):>6} {sum(g['agreed'] for g in group):>7} " + f"{sum(sum(g['fallbacks'].values()) for g in group):>10} " + f"{sum(g['switched_to_local'] for g in group):>9} {sum(g['input_tokens'] for g in group):>13} " + f"{sum(g['dollars'] for g in group):>11.6f} " + f"{statistics.median(medians) if medians else 'n/a'}") return "\n".join(lines) @@ -274,7 +323,8 @@ def main() -> None: parser.add_argument("--calls", type=int, default=6) parser.add_argument("--repeats", type=int, default=10) parser.add_argument("--fixtures", nargs="+", default=list(FIXTURES)) - parser.add_argument("--arms", nargs="+", default=list(ARMS)) + parser.add_argument("--arms", nargs="+", default=list(ARMS), choices=list(ALL_ARMS), + help=f"default: every arm but {JEV_API_ARM}, which calls TypeSafe's Jev API") parser.add_argument("--write", help="write the JSON rows and the summary to this path prefix") parser.add_argument("--profile", choices=sorted(PROFILES), help="a per-operator success profile of the simulated adapter (SIM_PROFILE); nothing for a real one") diff --git a/docs/AUTOSKILL.md b/docs/AUTOSKILL.md index ad5bf90..6babc43 100644 --- a/docs/AUTOSKILL.md +++ b/docs/AUTOSKILL.md @@ -294,6 +294,8 @@ selector(turn, catalog, policy, answers) -> receipt `answers` is the `answers` object from one or more `systemone` responses. The pure function does not open a socket. Tests feed it recorded JSON. +**Adapter status (2026-10-28).** The AutoSkill selector and its HTTP command are not built. The one Jev adapter in this repository is the branch policy `TypeSafeJevPolicy` (`syberlabs/jev_api.py`, `docs/EVOLUTION.md`): one `choice` per search decision, ids and numbers only, validated and with a local fallback. Its live calls of 2026-10-28 correct three things this design assumed. `jev-1.13.0` is still served as itself, but `jev-1.14.0` (2026-10-14) is now `jev-latest`, and `GET /v1/models` lists only the aliases. Probabilities come at two decimals and sum to 1 only within 0.01 on three options, so "omits a sent option" is not the only check a `choice` needs: a sum tolerance and an exact key set are. A missing key is 403, not 401. TypeSafe prices input only, $0.042 per million tokens; output is free. + The HTTP command reads `TYPESAFE_API_KEY` from the environment, posts to `https://api.typesafe.ai/v1/systemone`, times out at 120 seconds, and prints the response JSON. The key never enters the receipt, the catalog, or a log line. A non-200, a missing `answers` object, or a `choice` whose probabilities omit a sent option becomes reason `jev_unavailable` and decision `none`. The hosted proxy at `https://jevtypesafeai.com/api/v1/decide` speaks the same question types and is not the adapter this design pins. Token budget stays inside the published limits: state plus questions at most about 64k tokens, and state plus the longest single question at most about 32k. The 80-character index line and the 200-option chunk exist so a 286-skill tree does not depend on being under 255 by luck. The three gate nouls share one call. Each chunk's `choice` is its own call, and request 2 batches its `choice` with the per-skill nouls. A separate round trip per noul pays the state cost again and throws away the parallelism the API gives one request. diff --git a/docs/EVOLUTION.md b/docs/EVOLUTION.md index cb98c70..9f46716 100644 --- a/docs/EVOLUTION.md +++ b/docs/EVOLUTION.md @@ -159,6 +159,46 @@ Simulated, and nothing about a model: the profiles were chosen so the policies c **Decision rule.** From the packet's measurement section: "The decision rule for Jev: it is kept as the provider's default if, on the v2 fixtures, `evolve_jev` solves at least as many as `evolve` with at most half the dollars, over the three repeats; otherwise it stays available and not default." That is: Jev becomes the default only if it solves at least as many v2 fixtures as `fixed` with at most half the dollars over three repeats. The E3 v2 pilot of 2026-10-21 (Claude Sonnet 5.5, medium, K = 6, one repeat; `docs/MODEL_COMPARISON.md`) solved all four v2 fixtures single-shot, 16 of 16 runs in 28 calls for about $0.24, with `evolve` costing four to six times `single` on every fixture, so by the plan's own rule the v2 set is uninformative for that model and the paid Jev-against-fixed comparison will not run on v2 at Sonnet 5.5 as this packet's spec assumed. The simulated comparison above is the evidence in this packet; the paid comparison is deferred to a fixture set on which `single` fails (Haiku 5.5 on v2, or a v3 set), a choice the owner makes. Until that run is made and reported, `fixed` is the default and the table above is not evidence for the rule. +### TypeSafe's Jev API as a policy + +`JevPolicy` is a local heuristic that makes no call; the name comes from TypeSafe AI's Jev, a decision API that answers typed questions (`docs/AUTOSKILL.md`). **`TypeSafeJevPolicy`** (`syberlabs/jev_api.py`, `name` `jev_api`, revision `"1"`) puts the API itself behind the same seam: `JevPolicy` builds the offered actions and every stop, and Jev chooses one offered action. It is chosen by construction only, `EvolutionaryProvider(..., policy=TypeSafeJevPolicy(generations=..., prices=...))`, or as the benchmark arm `evolve_jevapi`; the terminal's `--policy` does not offer it. + +**What Jev is asked.** One `POST https://api.typesafe.ai/v1/systemone` per decision that offers two or more actions, with `Authorization: Bearer $TYPESAFE_API_KEY` read from the environment at call time. The body is `{model, state, questions}`. `model` is the pin `jev-1.13.0` (an exact `jev-X.Y.Z`; `jev-latest` and `jev-preview` are refused at construction). `questions` is one `choice` named `next_action` whose `instructions` and `criteria` are the module's constants and whose option keys are the offered action kinds (`evaluate`, `crossover`, `mutate`, `migrate`). `state` holds ids and numbers only: the decision count, the generation label and the generations left, the capacity, the population as slots with their scores and `passed`, the remaining candidates, evaluations, seconds and dollars, the `OperatorStats` of each kind, `mutator_exhausted`, and per offered action the slots it uses and the local policy's `expected_value`, `expected_dollars`, `expected_evaluations` and `price`. Candidate ids are replaced by slot numbers; no file content, objective, check output, prompt, path or environment name is in it (`tests/test_jev_api.py`, `Request`, walks the state and refuses any string). A decision with one offered action is taken without a call; a stop is always the local rule's and asks nothing, so Jev can neither end a search nor extend one. + +**What an answer must be.** Before it becomes an action: status 200; one strict JSON object (a duplicate key or `NaN` is refused) of at most 64 KiB; `model` exactly the pin; `answers` holding exactly `next_action`, of type `choice`; the choice one of the options sent; `probabilities` covering exactly the options sent, each finite in [0, 1], summing to 1 within **0.02**; the choice carrying the highest probability; `confidence` finite in [0, 1]; `usage.input_tokens` and `usage.output_tokens` non-negative integers. The tolerance is measured, not assumed: on 2026-10-28 every live answer gave probabilities at two decimals whose sum was off 1 by at most 0.01 on three options (rounding), so four options can be off by 0.02. + +**When it is not.** A timeout (30 s by default, at most the 120 s of `docs/AUTOSKILL.md`) is `jev_timeout`; no key, a non-200 or a transport failure is `jev_unavailable`; a body that fails a check is `jev_invalid`; a choice outside the options is `choice_outside_options`. Each takes the local `JevPolicy`'s action and records `fallback: {reason, detail}` with `detail` from a closed list (`http_401`, `http_429`, `model`, `probabilities`, ...), never response text, a header or an exception message. Nothing is retried. Three failures in a row set `switched_to_local` and the rest of that search is local without a call (`fallback: {reason: local_only}`); a success resets the count, and the next search starts with Jev again. With every call failing, a search is decision for decision the local policy's search (tested). + +**What is recorded.** Every decision entry carries, beside `JevPolicy`'s audit fields: `source` (`jev_api`, `local`, or `rule` for a stop), `model`, `answers_digest` (SHA-256 of the canonical `answers` object), `request_digest` (SHA-256 of the bytes sent), `latency_ms`, `fallback`, `failures` (in a row) and `local_action`, the kind the local policy would have taken, so agreement is countable decision by decision; a Jev decision also carries `jev: {choice, probabilities, confidence, usage, dollars}` and `chosen_by: jev_api`. The key is sent in the header and nowhere else: not in a trace entry, a signal, an exception or the policy object (tested with a planted key through `http_transport` and a stand-in opener). As for every policy, the trace is in memory and the journal holds each child's `signal.policy`; a provider cannot journal a decision of its own (`spec/GAPS.md` 61), so the `evolve_jevapi` arm writes the decision entries beside each exported journal. + +**Measured live on 2026-10-28** (28 calls, synthetic `Situation` states of ids and numbers, from Windows; redacted transcripts kept outside the repository; the 200 bodies and error bodies used by the tests are in `tests/fixtures/jev_api/` with request ids and cookies removed): + +| Fact | Observed | +| --- | --- | +| Request | `{model, state, questions}`; a `choice` needs `criteria` (422 without) | +| Response | `{model, answers: {next_action: {type, choice, confidence, probabilities}}, usage: {input_tokens, output_tokens}}`; no request id in the body; header `x-typesafe-request-id` | +| Model | `jev-1.13.0` is served as `jev-1.13.0`; `jev-1.14.0` (released 2026-10-14) and `jev-latest` are served as `jev-1.14.0`; `jev-1.13` and `jev-9.9.9` are 400 `Unknown model`; `GET /v1/models` lists only the aliases `jev-latest` and `jev-preview` | +| Latency | 20 answered calls: 142 to 338 ms, median 170 ms, about 705 to 760 input tokens each | +| Probabilities | two decimals (with float artifacts such as `0.6799999999999999`); sums 0.99 to 1.00; `confidence` is not the top probability (0.37 against 0.58) | +| Repeatability | the same request to `jev-1.13.0` three times gave `mutate` at 0.58, 0.56 and 0.53: not deterministic | +| Errors | bad key 401, no key 403 (both `{detail: {error_type: authentication_error, message}}`), unknown model 400, `NaN` in the body 400, missing `criteria` 422 with the offending input echoed back; a 50 ms client timeout raised a socket timeout | +| Limits, usage | no rate-limit or usage headers; TypeSafe documents per-organization limits in tokens and requests per second, 429 when over, 529 when overloaded, 64k tokens per request and 32k per question | +| Price | $0.042 per million input tokens, output free (docs.typesafe.ai/models, read 2026-10-28): about $0.00003 a decision; the 28 calls cost about $0.0006 | +| Data terms | docs.typesafe.ai/legal points to a Data Processing Agreement and a Privacy Policy that commits not to train on user data; retention periods are not stated on that page; zero data retention is offered to enterprise customers | + +What the calls say about the choice, and no more: on these synthetic states Jev chose `mutate` or `evaluate` every time and gave `crossover` 0.02 to 0.15, where the local policy prices a crossover (no model call) far below a mutation. Whether that is better or worse on a real fixture is what the measurement below is for. + +**Measurement plan (pre-registered 2026-10-28; not run).** It continues the `jev-policy` measurement and needs the owner's go through the lead, because it spends on Anthropic's API. + +- *Arms*: `single`, `evolve` (`FixedPolicy` revision 2), `evolve_jev` (local `JevPolicy`) and `evolve_jevapi` (`TypeSafeJevPolicy`, `jev-1.13.0`), through `benchmarks/model_vs_patch.py` and the same Build Thread; K = 6 model calls per run; Claude Sonnet 5.5 at medium effort (`SYBERLABS_MODEL=claude-sonnet-5-5`, `SYBERLABS_EFFORT=medium`); the four v2 fixtures (`crlf_export`, `oauth_state`, `settlement`, `tool_path`); one repeat, then three. Command: `PYTHONPATH=. python benchmarks/model_vs_patch.py --adapter " adapters/anthropic_adapter.py" --calls 6 --repeats 3 --fixtures crlf_export oauth_state settlement tool_path --arms single evolve evolve_jev evolve_jevapi --write `, with `ANTHROPIC_API_KEY` and `TYPESAFE_API_KEY` in that shell only. +- *Recorded per run*: solved, model calls, tokens, `dollars` (Anthropic, from the adapter's usage and the price table) and `host_dollars`, evaluations, and for `evolve_jevapi` the row's `jev_api` (Jev calls, decisions by source, fallbacks by reason, agreement with `local_action`, Jev's input tokens and dollars, median latency) with the decision entries beside each journal. +- *Decision rule carried over, verbatim from the `jev-policy` spec*: "The decision rule for Jev: it is kept as the provider's default if, on the v2 fixtures, `evolve_jev` solves at least as many as `evolve` with at most half the dollars, over the three repeats; otherwise it stays available and not default." +- *New decision rule, `evolve_jevapi` against `evolve_jev`*: TypeSafe's Jev replaces the local heuristic as the chooser behind `evolve_jev` only if, over the same three repeats on v2, `evolve_jevapi` solves at least as many runs as `evolve_jev` **and** its dollars (Anthropic dollars plus Jev's dollars) are at most `evolve_jev`'s. Neither arm becomes the default proposer path by this rule: `repair` stays the default (owner's decision of 2026-10-23). +- *When the comparison counts as not measured*: if more than 10% of the decisions on which Jev was asked fell back (any reason), or any run switched to local, the `evolve_jevapi` rows measured mostly the local policy and the new rule is not applied; if `evolve_jevapi` agreed with `local_action` on every asked decision, the two arms took the same actions and no difference may be claimed from them. +- *When v2 cannot decide*: v2 was solved single-shot at this model (16 of 16, 2026-10-21; `docs/MODEL_COMPARISON.md`). If `single` solves every v2 fixture in the first repeat, every evolve arm stops at its first candidate, both rules are reported as uninformative and no default changes; the next set is v3 or v4 (`evals/tasks`), a choice for the owner. The plan is written for v2 because that is the measurement the owner asked to continue. +- *Cost estimate* (estimate, not a quote): the v2 pilot spent about $0.0086 per Sonnet 5.5 call ($0.24 for 28 calls). Ceiling: 4 fixtures x 3 repeats x (1 call for `single` + 6 for each of the three evolve arms) = 228 calls, about **$2.0**, and about **$2.5** with mutate feedback lengthening prompts; expected well under $1 if v2 is again solved at the first candidate. Jev: at most about 15 asked decisions per `evolve_jevapi` run, 12 runs, about 750 input tokens each, about 135,000 tokens, about **$0.006** at $0.042 per million (180 calls at most). Budget line asked for: $3 Anthropic, $0.01 Jev, against the program cap. +- *Not expressible as a SyberEval spec yet*: `syberlabs/eval/spec.py` closes `ARM_KINDS` to `single`, `repair`, `evolve`, `evolve_jev` and `claude-code`, so neither `evolve_jevapi` nor a policy choice can be registered under `evals/`; this plan is the pre-registration (`spec/GAPS.md` 61). + ### Measured on a toy fixture `examples/evolve.py` is a pricing module with three independently checked rules and a stand-in mutator. `benchmarks/evolve_baseline.py` compares the provider with and without crossover on 30 seeds, with the same budget. The result is in `benchmarks/results/evolve-baseline.txt`: diff --git a/spec/GAPS.md b/spec/GAPS.md index 922d1d1..b25fa2c 100644 --- a/spec/GAPS.md +++ b/spec/GAPS.md @@ -119,3 +119,5 @@ The schemas describe SyberWork as it behaves. They are not enforced at runtime e 59. **The GitHits API client is wired into Claude Code as arm B's only lookup route; its contamination and license checks are filters on what returns, and its lookups are counted from the client's own records.** `githits-client` (2026-10-26) adds `syberlabs/githits.py` and `syberlabs githits search|example|read|list|grep` (`docs/GITHITS.md`): a standard-library client for five read-only GitHits REST operations, with a SyberLabs exclusion in three layers (a target refused before any request, a result whose identity is under an excluded owner dropped and counted, an example or read body that cites one dropped whole as `contamination_blocked`), a permissive-only license policy (`license_mode: strict` on examples; client-side, a result whose SPDX license is not on the allow list dropped and counted as `results_dropped_license`), and an HMAC-signed record per call (`query_guard.sign`, now shared). `githits-wiring` (2026-10-26) closes the first two deferred items: (1) `ClaudeCodeProvider(githits=True)` (`--githits`) writes a per-call, read-only launcher outside the worktree that runs the host checkout's client (`syberlabs/githits_cli_entry.py` under `python -I`), allows `Bash(:*)` and nothing else that runs code, matches `Bash` in the query guard (only the launcher with plain arguments passes, and its arguments are judged by the guard's rules), and journals one `lookup` per authentic client record (tool `githits.`), `signal.tool_usage.githits` being their number and `githits_lookups` reading those facts; (2) GitHits' shipped skills (`githits-com/githits-cli`, Apache-2.0) are adapted into `skills/githits/SKILL.md` with their license and notice, and appended to arm B's system prompt (`signal.harness.skill`). The same packet fixes #129's three low findings (nested SPDX expressions, the token scrubbed before truncation and out of `X-Request-ID`, `SYBERLABS_GITHITS_EXCLUDE` adding to `SyberLabs` instead of replacing it) and adds `syberlabs githits doctor` (one `GET /v1/languages`). `githits-pilot-hardening` (2026-10-26) narrows the launcher's entry to the five lookups (`doctor` and `--license-mode` stay the operator's, `example` always sends `strict`, and a `githits.doctor` fact is never counted in `githits_lookups`), records the skill's digest with CRLF normalized to LF (`.gitattributes` checks `skills/**` out with LF), tests that `SYBERLABS_GITHITS_EXCLUDE_REPLACE` reaches neither arm, and, under `--allow-code-tools-with-mcp`, matches `PowerShell` in the query guard and judges it by the launcher-only rule. Deferred: (3) the other operations of the document are not covered: `search_status` (polling a `search_ref` when a search returns incomplete; the client prints the ref and stops), `research` and the rest of `/v1/experimental/*`, package intelligence, `code_diff`, `resolve_target`, `feedback`; (4) no retry or back-off on `429`/`503`: `Retry-After` is not read by the lookup operations (only `doctor` reports it) and the call fails. Not solved: (5) layer (c) is detection, not prevention: an example can blend code from an excluded repository without citing it, and a mention spelled outside the listed host patterns passes; for the pilot, contamination is resolved on SyberLabs' side (unpublished work items, both arms run locally, a pinned base) with the client as the backstop, and an index-side exclusion by GitHits is optional; (6) a package target whose name does not name the owner reaches GitHits, and only its served repository, after the call, can drop it; (7) the license check depends on license data: in contract 0.1.0 only examples carry licenses (in Markdown), so search, grep, list and read results pass unjudged, and what `strict` admits server-side is not documented; (8) the fixture pins the five operations' closed subset of the document (215,618 bytes), and `doctor`'s operation in a second fixture, not the whole 1,364,570-byte document, which is over the contract's change-size limit; the upstream SHA-256 is recorded in `docs/GITHITS.md` for comparison by hand; (9) the record key and the guard's key sit in Claude Code's environment, which the launcher's client inherits, so a verifying MAC shows a key holder wrote the line, not which process; (10) no call to the real API was made, and no real Claude Code ran: every behaviour is tested against a loopback fake that follows the pinned schemas and a stand-in `claude` whose Bash tool is emulated (a command no `Bash(:*)` rule names is denied, the launcher runs as `sh` runs it); whether Claude Code 2.1.309 matches `Bash(:*)` against a Windows path written with forward slashes, and runs the launcher through Git Bash, is for the first live call (`syberlabs githits doctor` first, then one guarded lookup); (11) a record cannot be tied to the tool use that ran it (the client has no tool use id), so the host reconciles counts: authentic records against launcher calls the guard passed; fewer records is `signal.githits.mismatch: githits_unrecorded` (a client that failed before recording, a usage error, a denied permission), more is `githits_unexpected_records`, every record of the call is then `unguarded` and the search stops `provider_guard_inactive`; (12) closed by the two-arm accounting repair: failed calls journal unreconciled records as `unguarded` or `githits.unrecorded` when none exist, and a `no_change` refusal with missing records journals a synthetic gap; the metric remains unknown without a candidate signal; (13) `X-GitHits-Session-ID` is `-`, not `/`, because GitHits accepts only `[A-Za-z0-9_-]{1,64}`; (14) closed by the two-arm accounting repair: when client-record facts exist, `githits_lookups` also includes the host's MCP lookup attempts once; mixed routes still raise `two_routes`; (15) arm A (no `--githits` and no `--mcp-config`) receives no `GITHITS_*` or `SYBERLABS_GITHITS_*` variable since `githits-pilot-hardening`, the token included: least privilege was chosen over the byte-identical environment `githits-wiring` kept, so arm A's environment differs from before this route by exactly those variables (its argv does not); an `--mcp-config` run keeps them, because a GitHits MCP server may read its token from that environment; (16) the skill rides in `--append-system-prompt`, so it is part of every arm B request's system prompt, and arm A's harness note has no skill. 60. **SyberEval runs and records; it does not yet score, gate on safety or report, and its hidden checks are hidden from the agent's worktree, requests and candidates, not from a process that can read the host's disk.** `eval-core` (2026-10-26) adds `syberlabs/eval/` and `syberlabs eval register|tasks|run|preflight` (`docs/EVAL.md`): pre-registered specs with frozen digests, tasks whose acceptance checks the host stages outside the run's repository and appends to the run contract as required `hidden.*` checks by absolute path, arms built into the kernel's own providers, a runner that takes every task x arm x seed through an unattended Build Thread in the registered order and writes a digested bundle per run, an eval-wide dollar cap, and a preflight gate; the eight v3 and v4 fixtures are imported as tasks and E5 is registered, not run. The contract schema, the event kinds, the golden traces and the kernel are unchanged. Deferred to `eval-score`: (1) the safety gate computed from the journal (an effect without an admitted decision, a skipped approval or prerequisite, a promotion outside the contract's rules, an admitted out-of-scope candidate, a planted canary in an outbound record, a provider credential in the journal, a forbidden planner action), the scorer (pass, partial and fail on the hidden checks, the metrics suite, critic findings, review facts), the paired per-task analysis with the decision rule applied mechanically, and `eval score` and `eval report`, which exit 2 until then. Deferred to `eval-safety-fixtures`: (2) the agent-facing safety tasks (`evals/tasks/safety-*`, from the deterministic scenarios) and the gate's validation on them; a task's `safety.canaries` is recorded and digested but nothing searches for it yet. Not solved: (3) a hidden check's state and output tail are recorded like any check's and reach a repair or evolve arm as feedback, which is the E3 design; what stays hidden is the check's code and data, and `tasks.LOADER` keeps a traceback from printing either, while a hidden check that runs a script directly prints its path and failing line; (4) the host's copy is an ordinary directory named in the run contract, so a `claude-code` arm with file tools outside its worktree could read it (research register N4); (5) on Claude Code's login lanes nothing is priced, so the eval's dollar cap does not bound those runs, and the runner does not set Claude Code's model (the arm's model is a declaration; `signal.harness` records the model that served); (6) the `single` arm's `CommandProvider` request carries no seed; (7) runs are serial in one process, and an arm's environment is set on `os.environ` for the length of its proposal; (8) `syberlabs.eval` is not in the wheel, because `pyproject.toml` lists the `syberlabs` package without its subpackages; (9) the run's kit home is deleted unless `--keep-work` is given: the certificate is verified against the repository and the journal when it is written, and afterwards the bundle keeps `journal.json` and `repo.bundle`, not a home that `syberlabs verify --repo` can read; (10) E5's registration was written from this packet's working tree on base `7caa7c25b3` (`git_dirty: true`); the commit a run uses is in each bundle and is held to a passing preflight. + +61. **TypeSafe's Jev decides inside one search from ids and numbers; its decisions are not journaled by the host; its spend is outside the host's dollars; it cannot be registered as a SyberEval arm.** `TypeSafeJevPolicy` (`syberlabs/jev_api.py`, `docs/EVOLUTION.md`, 2026-10-28) asks `jev-1.13.0` one `choice` per decision over the actions `JevPolicy` offers, validates the answer and falls back to `JevPolicy`. What is open: (1) the decision record (`source`, `model`, `answers_digest`, `request_digest`, `latency_ms`, `fallback`, `local_action`) lives in the provider's in-memory trace, as every policy's does; `GitSearchSpace.observe` accepts only `PROVIDER_FACTS = ("lookup",)` (`syberlabs/build.py`), outside this packet, so a provider cannot journal it, and `benchmarks/model_vs_patch.py` writes the entries beside each exported journal instead. The smallest change that closes it is a `policy_decision` key in `PROVIDER_FACTS`, bounded like `lookup`. (2) Jev's dollars (input tokens at TypeSafe's published $0.042 per million) are counted by the policy and the arm's row, not by the host's `max_dollars`, which sees `providers.run_json` usage only; at about $0.00003 a decision this does not move a bound, but it is not the host's observation. (3) The answer is a provider's signal: its probabilities are not calibrated against admission and were not repeatable on 2026-10-28 (one request, three answers, `mutate` at 0.58, 0.56, 0.53). (4) Jev sees no text, so it decides from the same numbers the local heuristic reads; whether that adds anything is the pre-registered measurement in `docs/EVOLUTION.md`, unrun. (5) `syberlabs/eval/spec.py` closes `ARM_KINDS` without `evolve_jevapi`, so that measurement is a plan in the docs and not an `evals/` spec. (6) The pin is `jev-1.13.0` while `jev-latest` is `jev-1.14.0`; TypeSafe's page does not state a deprecation policy, so a retired pin would show up as 400 `Unknown model`, fall back to local on every decision and switch after three. (7) Jev cannot be named by the contract, like every policy (entry 41). diff --git a/syberlabs/evolve.py b/syberlabs/evolve.py index a6beb32..3030ac7 100644 --- a/syberlabs/evolve.py +++ b/syberlabs/evolve.py @@ -47,7 +47,9 @@ with an ``audit()`` method has its account of each decision merged into that decision's trace entry, so a reader can tell from the entry alone why it was taken. Neither learning policy draws from the provider's seeded stream beyond -the one crossover-pairing draw the fixed schedule makes. +the one crossover-pairing draw the fixed schedule makes. ``syberlabs.jev_api`` +holds a third, ``TypeSafeJevPolicy``: ``JevPolicy``'s offered actions, chosen by +TypeSafe's Jev API from ids and numbers, with ``JevPolicy`` as its fallback. """ from __future__ import annotations @@ -460,36 +462,49 @@ def _options(self, s: Situation) -> tuple[list[tuple[Action, dict]], dict]: found.append((Migrate(g + 1), self._estimate("migrate", s, priced, shadow))) return found, budget - def _stop(self, reason: str, generation: int, best: dict | None = None, **extra) -> Stop: - self._last_audit = {"expected_value": best["expected_value"] if best else None, - "expected_dollars": best["expected_dollars"] if best else None, - "chosen_by": f"rule:{reason}", **extra} - return Stop(reason, generation) + @staticmethod + def _rule(reason: str, best: dict | None = None, **extra) -> dict: + return {"expected_value": best["expected_value"] if best else None, + "expected_dollars": best["expected_dollars"] if best else None, + "chosen_by": f"rule:{reason}", **extra} - def decide(self, s: Situation) -> Action: + def _plan(self, s: Situation) -> tuple[Action, dict, list[tuple[Action, dict]], dict | None]: + """The decision ``decide`` takes, its audit, the options offered and the budget, without taking it. + + Pure: it reads the policy's state and changes none of it, so a policy built on this one + (``syberlabs.jev_api``) can ask what Jev would do, and take a different offered action. + """ g = self._generation if self.stop_when_passing and any(p.passed for p in s.population): - return self._stop("passing", g) + return Stop("passing", g), self._rule("passing"), [], None if not s.room: - return self._stop("budget", g) + return Stop("budget", g), self._rule("budget"), [], None options, budget = self._options(s) if not options: - return self._stop("empty", g) + return Stop("empty", g), self._rule("empty"), [], budget best = max(options, key=lambda o: o[1]["score"]) # the first of equal scores wins: KINDS order action, estimate = best expected_passes = round(estimate["expected_value"] * budget["remaining"] / estimate["price"], 6) alternatives = [dict(o[1]) for o in options] if expected_passes < self.floor: - return self._stop("budget", g, estimate, expected_passes=expected_passes, floor=self.floor, - budget=budget, alternatives=alternatives) + return Stop("budget", g), self._rule("budget", estimate, expected_passes=expected_passes, floor=self.floor, + budget=budget, alternatives=alternatives), options, budget if action.generation > g and g >= self.generations: - return self._stop("generations", g + 1, estimate, budget=budget, alternatives=alternatives) + return (Stop("generations", g + 1), self._rule("generations", estimate, budget=budget, alternatives=alternatives), + options, budget) if len(options) == 1: chosen_by = "only_option" elif max(options, key=lambda o: o[1]["exploit"]) is best: chosen_by = "value_per_dollar" else: chosen_by = "exploration" + audit = {"expected_value": estimate["expected_value"], "expected_dollars": estimate["expected_dollars"], + "chosen_by": chosen_by, "price": {k: estimate[k] for k in ("price", "expected_evaluations")}, + "expected_passes": expected_passes, "budget": budget, "alternatives": alternatives} + return action, audit, options, budget + + def _take(self, action: Action) -> None: + """Record that ``action``, one of the offered actions, is the one taken.""" if isinstance(action, Evaluate): self._seeds.pop(0) elif isinstance(action, Crossover): @@ -497,9 +512,12 @@ def decide(self, s: Situation) -> Action: elif isinstance(action, Mutate) and action.slot is not None: self._mutated[action.parent] = self._mutated.get(action.parent, 0) + 1 self._generation = action.generation - self._last_audit = {"expected_value": estimate["expected_value"], "expected_dollars": estimate["expected_dollars"], - "chosen_by": chosen_by, "price": {k: estimate[k] for k in ("price", "expected_evaluations")}, - "expected_passes": expected_passes, "budget": budget, "alternatives": alternatives} + + def decide(self, s: Situation) -> Action: + action, audit, _options, _budget = self._plan(s) + if not isinstance(action, Stop): + self._take(action) + self._last_audit = audit return action def audit(self) -> dict | None: diff --git a/syberlabs/jev_api.py b/syberlabs/jev_api.py new file mode 100644 index 0000000..1876da6 --- /dev/null +++ b/syberlabs/jev_api.py @@ -0,0 +1,363 @@ +"""TypeSafe's Jev API as a branch policy: ``TypeSafeJevPolicy``, name ``jev_api``. + +The local ``JevPolicy`` (``syberlabs.evolve``) builds the offered actions and the stops; Jev, +TypeSafe AI's decision API, chooses one offered action with a typed ``choice`` question; the +answer is validated before it can become an action, and anything else falls back to the +local policy's own choice. Jev never authorizes anything: it picks among actions code has +already built, it cannot stop, extend or widen a search, and the host judges every candidate. + +What is sent (``POST https://api.typesafe.ai/v1/systemone``): the pinned model, a ``state`` +of ids and numbers projected from ``Situation`` (slots, scores, the remaining budget, the +operator statistics, the local estimates of each offered action), and one question, +``next_action``, whose ``instructions`` and ``criteria`` are this module's constants and +whose option keys are action kinds. Never candidate ids, file contents, prompts, paths, +objective or check text. The key is read from ``TYPESAFE_API_KEY`` at call time, sent only +as the ``Authorization`` header, and never stored, journaled or put in an error. + +What is checked before an answer is used: the HTTP status is 200; the body is one strict +JSON object (no duplicate keys, no NaN); ``model`` is exactly the pinned model; ``answers`` +holds exactly ``next_action``, of type ``choice``; the choice is one of the options sent; +the probabilities cover exactly the options sent, each finite in [0, 1], summing to 1 within +``TOLERANCE``; the choice has the highest probability; ``confidence`` is finite in [0, 1]; +``usage`` has non-negative integer token counts. The shape was taken from live calls of +2026-10-28 (``tests/fixtures/jev_api/``): probabilities come at two decimals, so their sum is +off 1 by up to 0.01 on three options; 0.02 covers four. + +What happens otherwise: a timeout (``jev_timeout``), no key, a non-200 or a transport failure +(``jev_unavailable``), a body that fails a check (``jev_invalid``), or a choice outside the +options (``choice_outside_options``) is recorded as the decision's ``fallback`` and the local +policy's action is taken. Three failures in a row switch the policy to local for the rest of +the search (``local_only``); a failure is never retried. Every decision's trace entry carries +``source`` (``jev_api``, ``local`` or ``rule``), ``model``, ``answers_digest``, +``request_digest``, ``latency_ms``, ``fallback`` and ``local_action``, the action the local +policy would have taken, so the two can be compared decision by decision. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import time +import urllib.error +import urllib.request +from collections.abc import Callable, Mapping, Sequence + +from syberlabs.canonical import canonical, digest +from syberlabs.economic import is_money +from syberlabs.errors import Rejected +from syberlabs.evolve import JevPolicy +from syberlabs.search import Action, Crossover, Evaluate, Mutate, OperatorStats, Situation, Stop + +ENDPOINT = "https://api.typesafe.ai/v1/systemone" +KEY_ENV = "TYPESAFE_API_KEY" +MODEL = "jev-1.13.0" # the pin docs/AUTOSKILL.md names; jev-1.14.0 (2026-10-14) is jev-latest since then +PIN = re.compile(r"^jev-\d+\.\d+\.\d+$") # an exact version; the aliases jev-latest and jev-preview are refused +TIMEOUT = 30.0 # seconds; the 2026-10-28 calls took 0.14 to 0.34 s for about 750 input tokens +MAX_TIMEOUT = 120.0 # docs/AUTOSKILL.md's adapter timeout, the most this policy accepts +TOLERANCE = 0.02 # |sum of probabilities - 1|; two-decimal probabilities on up to four options +MAX_FAILURES = 3 # failures in a row before the policy is local for the rest of the search +MAX_REQUEST = 65_536 # bytes; a request is about 2.5 KB +MAX_RESPONSE = 65_536 # bytes read from a response; one answered choice is under 1 KB +INPUT_DOLLARS_PER_TOKEN = 0.042 / 1_000_000 # docs.typesafe.ai/models, 2026-10-28: input only, output free +QUESTION = "next_action" +INSTRUCTIONS = ("Choose the next action of a bounded evolutionary code search from the numbers in the state. " + "Pick the offered action most likely to produce a candidate that passes every check, " + "per dollar of the remaining budget.") +CRITERIA = { + "evaluate": "Evaluate the next seed candidate with the host's checks: no model call, one evaluation.", + "crossover": "Merge the two population slots named in the state: no model call, one evaluation.", + "mutate": "Ask the model for a child of the parent slot named in the state (none: the base): one model call, " + "one evaluation.", + "migrate": "Exchange individuals with another host: no model call.", +} +SOURCES = ("jev_api", "local", "rule") +REASONS = ("jev_timeout", "jev_unavailable", "jev_invalid", "choice_outside_options", "local_only") +# Why a call failed, as a closed word: never response text, a header or an exception message. +DETAILS = ("no_key", "timeout", "transport", "http_401", "http_403", "http_429", "http_529", "http_4xx", "http_5xx", + "http_other", "too_large", "not_json", "not_object", "model", "answers", "type", "choice", "probabilities", + "argmax", "confidence", "usage", "request_too_large", "switched") + + +class JevFailure(Exception): + """A call that did not yield a usable answer. ``reason`` and ``detail`` are closed words; nothing else is kept.""" + + def __init__(self, reason: str, detail: str, *, latency_ms: float | None = None, response_digest: str | None = None): + assert reason in REASONS and detail in DETAILS, (reason, detail) + super().__init__(f"{reason}: {detail}") + self.reason, self.detail, self.latency_ms, self.response_digest = reason, detail, latency_ms, response_digest + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None # a redirect is a non-200, never a second send of the key + + +def http_transport(body: bytes, timeout: float, *, opener=None) -> tuple[int, bytes]: + """One POST to ``ENDPOINT``. Returns ``(status, body)``; raises ``JevFailure`` for no key, a timeout or transport. + + The key is read here, used for this request's header and dropped. No retry. + """ + key = os.environ.get(KEY_ENV, "") + if not key or any(c in key for c in "\r\n"): + raise JevFailure("jev_unavailable", "no_key") + request = urllib.request.Request(ENDPOINT, data=body, method="POST", headers={ + "Authorization": "Bearer " + key, "Content-Type": "application/json", "Accept": "application/json"}) + del key + opener = opener or urllib.request.build_opener(_NoRedirect()) + try: + with opener.open(request, timeout=timeout) as response: + return response.status, response.read(MAX_RESPONSE + 1) + except urllib.error.HTTPError as exc: + try: + data = exc.read(MAX_RESPONSE + 1) + except Exception: # noqa: BLE001 - an unreadable error body is still the status + data = b"" + return exc.code, data + except TimeoutError: + raise JevFailure("jev_timeout", "timeout") from None + except urllib.error.URLError as exc: + if isinstance(exc.reason, TimeoutError): + raise JevFailure("jev_timeout", "timeout") from None + raise JevFailure("jev_unavailable", "transport") from None + except OSError: + raise JevFailure("jev_unavailable", "transport") from None + + +def _pairs(pairs): + found = {} + for key, value in pairs: + if key in found: + raise ValueError("duplicate key") + found[key] = value + return found + + +def _nonfinite(_value): + raise ValueError("non-finite number") + + +def _number(value) -> bool: + return type(value) in (int, float) and math.isfinite(value) + + +def _status_detail(status: int) -> str: + if status in (401, 403, 429, 529): + return f"http_{status}" + if 400 <= status < 500: + return "http_4xx" + if 500 <= status < 600: + return "http_5xx" + return "http_other" + + +def validate(status: int, raw: bytes, options: Sequence[str], *, model: str = MODEL, + tolerance: float = TOLERANCE) -> dict: + """Check one ``systemone`` response for one ``choice`` question over ``options``. Returns the parsed answer. + + ``{"model", "choice", "probabilities", "confidence", "usage", "answers_digest"}``. Raises + ``JevFailure`` with a closed detail; the response's text never reaches the exception. + """ + if status != 200: + raise JevFailure("jev_unavailable", _status_detail(status)) + if len(raw) > MAX_RESPONSE: + raise JevFailure("jev_invalid", "too_large") + try: + found = json.loads(raw.decode("utf-8"), object_pairs_hook=_pairs, parse_constant=_nonfinite) + except (UnicodeDecodeError, ValueError): + raise JevFailure("jev_invalid", "not_json") from None + if type(found) is not dict: + raise JevFailure("jev_invalid", "not_object") + if found.get("model") != model: + raise JevFailure("jev_invalid", "model") + answers = found.get("answers") + if type(answers) is not dict or set(answers) != {QUESTION}: + raise JevFailure("jev_invalid", "answers") + answer = answers[QUESTION] + if type(answer) is not dict or answer.get("type") != "choice": + raise JevFailure("jev_invalid", "type") + choice, probabilities, confidence = answer.get("choice"), answer.get("probabilities"), answer.get("confidence") + if type(choice) is not str: + raise JevFailure("jev_invalid", "choice") + if choice not in options: + raise JevFailure("choice_outside_options", "choice") + if (type(probabilities) is not dict or set(probabilities) != set(options) + or any(not _number(v) or not 0 <= v <= 1 for v in probabilities.values()) + or abs(sum(probabilities.values()) - 1) > tolerance): + raise JevFailure("jev_invalid", "probabilities") + if probabilities[choice] < max(probabilities.values()): + raise JevFailure("jev_invalid", "argmax") + if not _number(confidence) or not 0 <= confidence <= 1: + raise JevFailure("jev_invalid", "confidence") + usage = found.get("usage") + if (type(usage) is not dict + or any(type(usage.get(k)) is not int or usage[k] < 0 for k in ("input_tokens", "output_tokens"))): + raise JevFailure("jev_invalid", "usage") + return {"model": found["model"], "choice": choice, + "probabilities": {k: probabilities[k] for k in options}, "confidence": confidence, + "usage": {"input_tokens": usage["input_tokens"], "output_tokens": usage["output_tokens"]}, + "answers_digest": digest(answers)} + + +def _stats(stats: OperatorStats) -> dict: + return {"attempted": stats.attempted, "produced": stats.produced, "evaluated": stats.evaluated, + "improved": stats.improved, "passed": stats.passed, "dollars": stats.dollars} + + +def _option(action: Action, estimate: Mapping, seeds_left: int) -> dict: + """One offered action as numbers: which slots it uses, and the local policy's estimate of it.""" + found: dict = {"generation": action.generation} + if isinstance(action, Mutate): + found["parent_slot"] = action.slot # None: the base + elif isinstance(action, Crossover): + found["slots"] = list(action.slots) + elif isinstance(action, Evaluate): + found["seeds_left"] = seeds_left + for key in ("expected_value", "expected_dollars", "expected_evaluations", "price"): + found[key] = estimate[key] + return found + + +class TypeSafeJevPolicy(JevPolicy): + """``JevPolicy``'s offered actions, chosen by TypeSafe's Jev API; ``JevPolicy`` decides whenever Jev cannot. + + Stops are the local policy's: a passing individual, no room, nothing offered, the floor and + the generation bound end the search without a call, as they do for ``JevPolicy``. Otherwise + the actions ``JevPolicy`` offers (less any that would cross the generation bound) are sent + as a ``choice``; one offered action is taken without a call. ``transport(body, timeout)`` + returns ``(status, body)`` (``http_transport`` by default); tests pass a recorded one. + """ + + name, revision = "jev_api", "1" + + def __init__(self, *, generations: int, seeds: Sequence[str] = (), prices: Mapping[str, float] | None = None, + prior: tuple[float, float] = (1.0, 1.0), explore: float = 0.1, floor: float = 1.0, + stop_when_passing: bool = True, model: str = MODEL, timeout: float = TIMEOUT, + transport: Callable[[bytes, float], tuple[int, bytes]] | None = None, + max_failures: int = MAX_FAILURES, tolerance: float = TOLERANCE): + super().__init__(generations=generations, seeds=seeds, prices=prices, prior=prior, explore=explore, + floor=floor, stop_when_passing=stop_when_passing) + if (not isinstance(model, str) or not PIN.match(model) or not is_money(timeout) or not 0 < timeout <= MAX_TIMEOUT + or type(max_failures) is not int or max_failures < 1 or not is_money(tolerance) + or not 0 < tolerance <= 0.1 or (transport is not None and not callable(transport))): + raise Rejected("invalid_provider", "jev_api: an exact model version (jev-X.Y.Z), 0 < timeout <= 120 s, " + "max_failures >= 1, 0 < tolerance <= 0.1 and a callable transport") + self.model, self.timeout, self.max_failures, self.tolerance = model, float(timeout), max_failures, float(tolerance) + self.transport = transport or http_transport + + def reset(self) -> None: + super().reset() + self._failures, self._local_only = 0, False + self.calls, self.input_tokens, self.jev_dollars = 0, 0, 0.0 # this search's Jev calls, read after it + + def state(self, s: Situation, offered: Sequence[tuple[Action, dict]]) -> dict: + """The ``state`` Jev sees: ids and numbers only. Population members are slots, never candidate ids.""" + left = s.remaining + return { + "decision": s.decisions, "generation": self._generation, + "generations_left": max(0, self.generations - self._generation), "capacity": s.capacity, + "population": [{"slot": i, "score": list(p.score), "passed": bool(p.passed)} for i, p in enumerate(s.population)], + "remaining": {"candidates": left.max_candidates, "evaluations": left.max_evaluations, + "seconds": left.max_seconds, "dollars": getattr(left, "max_dollars", None)}, + "history": {kind: _stats(s.history.get(kind, OperatorStats())) for kind in self.KINDS}, + "mutator_exhausted": bool(s.mutator_exhausted), + "options": {action.kind: _option(action, estimate, len(self._seeds)) for action, estimate in offered}, + } + + def request(self, s: Situation, offered: Sequence[tuple[Action, dict]]) -> dict: + kinds = [action.kind for action, _ in offered] + return {"model": self.model, "state": self.state(s, offered), + "questions": {QUESTION: {"type": "choice", "instructions": INSTRUCTIONS, + "criteria": {kind: CRITERIA[kind] for kind in kinds}}}} + + def _ask(self, body: bytes, kinds: Sequence[str]) -> tuple[dict, float]: + started = time.perf_counter() + try: + status, raw = self.transport(body, self.timeout) + except JevFailure as failure: + failure.latency_ms = round((time.perf_counter() - started) * 1000, 1) + raise + except TimeoutError: + raise JevFailure("jev_timeout", "timeout", latency_ms=round((time.perf_counter() - started) * 1000, 1)) from None + except Exception: # noqa: BLE001 - any transport failure falls back; its text is never kept + raise JevFailure("jev_unavailable", "transport", + latency_ms=round((time.perf_counter() - started) * 1000, 1)) from None + latency = round((time.perf_counter() - started) * 1000, 1) + if not isinstance(status, int) or not isinstance(raw, (bytes, bytearray)): + raise JevFailure("jev_unavailable", "transport", latency_ms=latency) + raw = bytes(raw) + try: + return validate(status, raw, kinds, model=self.model, tolerance=self.tolerance), latency + except JevFailure as failure: + failure.latency_ms = latency + failure.response_digest = hashlib.sha256(raw).hexdigest() if raw else None + raise + + @staticmethod + def _record(source: str, model: str, *, local_action: str | None, answers_digest: str | None = None, + request_digest: str | None = None, latency_ms: float | None = None, fallback: dict | None = None, + failures: int = 0) -> dict: + return {"source": source, "model": model, "answers_digest": answers_digest, "request_digest": request_digest, + "latency_ms": latency_ms, "fallback": fallback, "failures": failures, "local_action": local_action} + + def decide(self, s: Situation) -> Action: + local, audit, options, _budget = self._plan(s) + if isinstance(local, Stop): # the local policy's rules end the search; Jev is not asked + self._last_audit = {**audit, **self._record("rule", self.model, local_action="stop", + failures=self._failures)} + return local + g = self._generation + offered = [(a, e) for a, e in options if not (a.generation > g and g >= self.generations)] + if len(offered) == 1: + self._take(local) + self._last_audit = {**audit, **self._record("local", self.model, local_action=local.kind, + failures=self._failures)} + return local + if self._local_only: + self._take(local) + self._last_audit = {**audit, **self._record( + "local", self.model, local_action=local.kind, failures=self._failures, + fallback={"reason": "local_only", "detail": "switched"})} + return local + request = self.request(s, offered) + body = canonical(request).encode("utf-8") + request_digest = hashlib.sha256(body).hexdigest() + kinds = [a.kind for a, _ in offered] + try: + if len(body) > MAX_REQUEST: + raise JevFailure("jev_invalid", "request_too_large", latency_ms=0.0) + self.calls += 1 + answer, latency = self._ask(body, kinds) + except JevFailure as failure: + self._failures += 1 + switched = self._failures >= self.max_failures + self._local_only = self._local_only or switched + self._take(local) + fallback = {"reason": failure.reason, "detail": failure.detail} + if failure.response_digest: + fallback["response_digest"] = failure.response_digest + if switched: + fallback["switched_to_local"] = True + self._last_audit = {**audit, **self._record( + "local", self.model, local_action=local.kind, request_digest=request_digest, + latency_ms=failure.latency_ms, fallback=fallback, failures=self._failures)} + return local + self._failures = 0 + self.input_tokens += answer["usage"]["input_tokens"] + dollars = round(answer["usage"]["input_tokens"] * INPUT_DOLLARS_PER_TOKEN, 9) + self.jev_dollars = round(self.jev_dollars + dollars, 9) + action, estimate = next((a, e) for a, e in offered if a.kind == answer["choice"]) + self._take(action) + budget = audit["budget"] + self._last_audit = { + **audit, "expected_value": estimate["expected_value"], "expected_dollars": estimate["expected_dollars"], + "chosen_by": "jev_api", "price": {k: estimate[k] for k in ("price", "expected_evaluations")}, + "expected_passes": round(estimate["expected_value"] * budget["remaining"] / estimate["price"], 6), + **self._record("jev_api", answer["model"], local_action=local.kind, answers_digest=answer["answers_digest"], + request_digest=request_digest, latency_ms=latency), + "jev": {"choice": answer["choice"], "probabilities": answer["probabilities"], + "confidence": answer["confidence"], "usage": answer["usage"], "dollars": dollars}, + } + return action diff --git a/tests/fixtures/jev_api/choice_alias_latest.json b/tests/fixtures/jev_api/choice_alias_latest.json new file mode 100644 index 0000000..290abba --- /dev/null +++ b/tests/fixtures/jev_api/choice_alias_latest.json @@ -0,0 +1,95 @@ +{ + "recorded": "2026-10-28", + "source": "POST https://api.typesafe.ai/v1/systemone, packet jev-api phase 1", + "scrubbed": "request headers (Authorization), response headers (request id, cookie, CF-RAY, date)", + "status": 200, + "latency_ms": 143.7, + "request": { + "model": "jev-latest", + "state": { + "decision": 3, + "generation": 1, + "capacity": 3, + "population": [ + { + "id": "c1", + "score": [ + 1, + 1 + ], + "passed": false + } + ], + "remaining": { + "dollars": 4.95, + "evaluations": 20, + "candidates": 10 + }, + "history": { + "mutate": { + "attempted": 2, + "produced": 2, + "evaluated": 2, + "improved": 1, + "passed": 0, + "dollars": 0.027 + }, + "crossover": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "evaluate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "migrate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + } + }, + "mutator_exhausted": false + }, + "questions": { + "next_action": { + "type": "choice", + "instructions": "Choose the next action of a bounded evolutionary code search from the numbers in the state. Pick the action most likely to produce a passing candidate per dollar of remaining budget.", + "criteria": { + "evaluate": "Evaluate the next seed candidate with the host's checks; no model call.", + "crossover": "Merge the two named population members; no model call, one evaluation.", + "mutate": "Ask the model mutator for a child of the named parent; one model call, one evaluation." + } + } + } + }, + "response": { + "model": "jev-1.14.0", + "answers": { + "next_action": { + "type": "choice", + "choice": "mutate", + "confidence": 0.46, + "probabilities": { + "evaluate": 0.3, + "crossover": 0.06, + "mutate": 0.64 + } + } + }, + "usage": { + "input_tokens": 705, + "output_tokens": 43 + } + } +} diff --git a/tests/fixtures/jev_api/choice_four_options.json b/tests/fixtures/jev_api/choice_four_options.json new file mode 100644 index 0000000..b4533fd --- /dev/null +++ b/tests/fixtures/jev_api/choice_four_options.json @@ -0,0 +1,105 @@ +{ + "recorded": "2026-10-28", + "source": "POST https://api.typesafe.ai/v1/systemone, packet jev-api phase 1", + "scrubbed": "request headers (Authorization), response headers (request id, cookie, CF-RAY, date)", + "status": 200, + "latency_ms": 198.2, + "request": { + "model": "jev-1.13.0", + "state": { + "decision": 4, + "generation": 2, + "capacity": 3, + "population": [ + { + "id": "c1", + "score": [ + 1, + 1 + ], + "passed": false + }, + { + "id": "c2", + "score": [ + 2, + 2 + ], + "passed": false + } + ], + "remaining": { + "dollars": 4.9, + "evaluations": 19, + "candidates": 9 + }, + "history": { + "mutate": { + "attempted": 3, + "produced": 2, + "evaluated": 2, + "improved": 1, + "passed": 0, + "dollars": 0.054 + }, + "crossover": { + "attempted": 1, + "produced": 1, + "evaluated": 1, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "evaluate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "migrate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + } + }, + "mutator_exhausted": false + }, + "questions": { + "next_action": { + "type": "choice", + "instructions": "Choose the next action of a bounded evolutionary code search from the numbers in the state. Pick the action most likely to produce a passing candidate per dollar of remaining budget.", + "criteria": { + "evaluate": "Evaluate the next seed candidate with the host's checks; no model call.", + "crossover": "Merge the two named population members; no model call, one evaluation.", + "mutate": "Ask the model mutator for a child of the named parent; one model call, one evaluation.", + "migrate": "Exchange individuals with another host; no model call." + } + } + } + }, + "response": { + "model": "jev-1.13.0", + "answers": { + "next_action": { + "type": "choice", + "choice": "mutate", + "confidence": 0.28, + "probabilities": { + "evaluate": 0.45, + "crossover": 0.07, + "mutate": 0.46, + "migrate": 0.02 + } + } + }, + "usage": { + "input_tokens": 753, + "output_tokens": 52 + } + } +} diff --git a/tests/fixtures/jev_api/choice_low_confidence.json b/tests/fixtures/jev_api/choice_low_confidence.json new file mode 100644 index 0000000..3c94d56 --- /dev/null +++ b/tests/fixtures/jev_api/choice_low_confidence.json @@ -0,0 +1,103 @@ +{ + "recorded": "2026-10-28", + "source": "POST https://api.typesafe.ai/v1/systemone, packet jev-api phase 1", + "scrubbed": "request headers (Authorization), response headers (request id, cookie, CF-RAY, date)", + "status": 200, + "latency_ms": 154.6, + "request": { + "model": "jev-1.13.0", + "state": { + "decision": 13, + "generation": 3, + "capacity": 3, + "population": [ + { + "id": "c1", + "score": [ + 1, + 1 + ], + "passed": false + }, + { + "id": "c2", + "score": [ + 2, + 2 + ], + "passed": false + } + ], + "remaining": { + "dollars": 4.45, + "evaluations": 10, + "candidates": 10 + }, + "history": { + "mutate": { + "attempted": 12, + "produced": 2, + "evaluated": 2, + "improved": 1, + "passed": 0, + "dollars": 0.297 + }, + "crossover": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "evaluate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "migrate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + } + }, + "mutator_exhausted": false + }, + "questions": { + "next_action": { + "type": "choice", + "instructions": "Choose the next action of a bounded evolutionary code search from the numbers in the state. Pick the action most likely to produce a passing candidate per dollar of remaining budget.", + "criteria": { + "evaluate": "Evaluate the next seed candidate with the host's checks; no model call.", + "crossover": "Merge the two named population members; no model call, one evaluation.", + "mutate": "Ask the model mutator for a child of the named parent; one model call, one evaluation." + } + } + } + }, + "response": { + "model": "jev-1.13.0", + "answers": { + "next_action": { + "type": "choice", + "choice": "mutate", + "confidence": 0.14, + "probabilities": { + "evaluate": 0.42, + "crossover": 0.15, + "mutate": 0.43 + } + } + }, + "usage": { + "input_tokens": 734, + "output_tokens": 43 + } + } +} diff --git a/tests/fixtures/jev_api/choice_repeat_a.json b/tests/fixtures/jev_api/choice_repeat_a.json new file mode 100644 index 0000000..ca30a08 --- /dev/null +++ b/tests/fixtures/jev_api/choice_repeat_a.json @@ -0,0 +1,95 @@ +{ + "recorded": "2026-10-28", + "source": "POST https://api.typesafe.ai/v1/systemone, packet jev-api phase 1", + "scrubbed": "request headers (Authorization), response headers (request id, cookie, CF-RAY, date)", + "status": 200, + "latency_ms": 169.1, + "request": { + "model": "jev-1.13.0", + "state": { + "decision": 3, + "generation": 1, + "capacity": 3, + "population": [ + { + "id": "c1", + "score": [ + 1, + 1 + ], + "passed": false + } + ], + "remaining": { + "dollars": 4.95, + "evaluations": 20, + "candidates": 10 + }, + "history": { + "mutate": { + "attempted": 2, + "produced": 2, + "evaluated": 2, + "improved": 1, + "passed": 0, + "dollars": 0.027 + }, + "crossover": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "evaluate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "migrate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + } + }, + "mutator_exhausted": false + }, + "questions": { + "next_action": { + "type": "choice", + "instructions": "Choose the next action of a bounded evolutionary code search from the numbers in the state. Pick the action most likely to produce a passing candidate per dollar of remaining budget.", + "criteria": { + "evaluate": "Evaluate the next seed candidate with the host's checks; no model call.", + "crossover": "Merge the two named population members; no model call, one evaluation.", + "mutate": "Ask the model mutator for a child of the named parent; one model call, one evaluation." + } + } + } + }, + "response": { + "model": "jev-1.13.0", + "answers": { + "next_action": { + "type": "choice", + "choice": "mutate", + "confidence": 0.34, + "probabilities": { + "evaluate": 0.42, + "crossover": 0.02, + "mutate": 0.56 + } + } + }, + "usage": { + "input_tokens": 705, + "output_tokens": 43 + } + } +} diff --git a/tests/fixtures/jev_api/choice_repeat_b.json b/tests/fixtures/jev_api/choice_repeat_b.json new file mode 100644 index 0000000..9001751 --- /dev/null +++ b/tests/fixtures/jev_api/choice_repeat_b.json @@ -0,0 +1,95 @@ +{ + "recorded": "2026-10-28", + "source": "POST https://api.typesafe.ai/v1/systemone, packet jev-api phase 1", + "scrubbed": "request headers (Authorization), response headers (request id, cookie, CF-RAY, date)", + "status": 200, + "latency_ms": 141.8, + "request": { + "model": "jev-1.13.0", + "state": { + "decision": 3, + "generation": 1, + "capacity": 3, + "population": [ + { + "id": "c1", + "score": [ + 1, + 1 + ], + "passed": false + } + ], + "remaining": { + "dollars": 4.95, + "evaluations": 20, + "candidates": 10 + }, + "history": { + "mutate": { + "attempted": 2, + "produced": 2, + "evaluated": 2, + "improved": 1, + "passed": 0, + "dollars": 0.027 + }, + "crossover": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "evaluate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "migrate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + } + }, + "mutator_exhausted": false + }, + "questions": { + "next_action": { + "type": "choice", + "instructions": "Choose the next action of a bounded evolutionary code search from the numbers in the state. Pick the action most likely to produce a passing candidate per dollar of remaining budget.", + "criteria": { + "evaluate": "Evaluate the next seed candidate with the host's checks; no model call.", + "crossover": "Merge the two named population members; no model call, one evaluation.", + "mutate": "Ask the model mutator for a child of the named parent; one model call, one evaluation." + } + } + } + }, + "response": { + "model": "jev-1.13.0", + "answers": { + "next_action": { + "type": "choice", + "choice": "mutate", + "confidence": 0.3, + "probabilities": { + "evaluate": 0.45, + "crossover": 0.02, + "mutate": 0.53 + } + } + }, + "usage": { + "input_tokens": 705, + "output_tokens": 43 + } + } +} diff --git a/tests/fixtures/jev_api/choice_served_by_1_14.json b/tests/fixtures/jev_api/choice_served_by_1_14.json new file mode 100644 index 0000000..4ccdf67 --- /dev/null +++ b/tests/fixtures/jev_api/choice_served_by_1_14.json @@ -0,0 +1,95 @@ +{ + "recorded": "2026-10-28", + "source": "POST https://api.typesafe.ai/v1/systemone, packet jev-api phase 1", + "scrubbed": "request headers (Authorization), response headers (request id, cookie, CF-RAY, date)", + "status": 200, + "latency_ms": 151.9, + "request": { + "model": "jev-1.14.0", + "state": { + "decision": 3, + "generation": 1, + "capacity": 3, + "population": [ + { + "id": "c1", + "score": [ + 1, + 1 + ], + "passed": false + } + ], + "remaining": { + "dollars": 4.95, + "evaluations": 20, + "candidates": 10 + }, + "history": { + "mutate": { + "attempted": 2, + "produced": 2, + "evaluated": 2, + "improved": 1, + "passed": 0, + "dollars": 0.027 + }, + "crossover": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "evaluate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "migrate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + } + }, + "mutator_exhausted": false + }, + "questions": { + "next_action": { + "type": "choice", + "instructions": "Choose the next action of a bounded evolutionary code search from the numbers in the state. Pick the action most likely to produce a passing candidate per dollar of remaining budget.", + "criteria": { + "evaluate": "Evaluate the next seed candidate with the host's checks; no model call.", + "crossover": "Merge the two named population members; no model call, one evaluation.", + "mutate": "Ask the model mutator for a child of the named parent; one model call, one evaluation." + } + } + } + }, + "response": { + "model": "jev-1.14.0", + "answers": { + "next_action": { + "type": "choice", + "choice": "mutate", + "confidence": 0.52, + "probabilities": { + "evaluate": 0.27, + "crossover": 0.05, + "mutate": 0.6799999999999999 + } + } + }, + "usage": { + "input_tokens": 705, + "output_tokens": 43 + } + } +} diff --git a/tests/fixtures/jev_api/choice_three_options.json b/tests/fixtures/jev_api/choice_three_options.json new file mode 100644 index 0000000..8960b34 --- /dev/null +++ b/tests/fixtures/jev_api/choice_three_options.json @@ -0,0 +1,95 @@ +{ + "recorded": "2026-10-28", + "source": "POST https://api.typesafe.ai/v1/systemone, packet jev-api phase 1", + "scrubbed": "request headers (Authorization), response headers (request id, cookie, CF-RAY, date)", + "status": 200, + "latency_ms": 158.9, + "request": { + "model": "jev-1.13.0", + "state": { + "decision": 3, + "generation": 1, + "capacity": 3, + "population": [ + { + "id": "c1", + "score": [ + 1, + 1 + ], + "passed": false + } + ], + "remaining": { + "dollars": 4.95, + "evaluations": 20, + "candidates": 10 + }, + "history": { + "mutate": { + "attempted": 2, + "produced": 2, + "evaluated": 2, + "improved": 1, + "passed": 0, + "dollars": 0.027 + }, + "crossover": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "evaluate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "migrate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + } + }, + "mutator_exhausted": false + }, + "questions": { + "next_action": { + "type": "choice", + "instructions": "Choose the next action of a bounded evolutionary code search from the numbers in the state. Pick the action most likely to produce a passing candidate per dollar of remaining budget.", + "criteria": { + "evaluate": "Evaluate the next seed candidate with the host's checks; no model call.", + "crossover": "Merge the two named population members; no model call, one evaluation.", + "mutate": "Ask the model mutator for a child of the named parent; one model call, one evaluation." + } + } + } + }, + "response": { + "model": "jev-1.13.0", + "answers": { + "next_action": { + "type": "choice", + "choice": "mutate", + "confidence": 0.37, + "probabilities": { + "evaluate": 0.39, + "crossover": 0.03, + "mutate": 0.58 + } + } + }, + "usage": { + "input_tokens": 705, + "output_tokens": 43 + } + } +} diff --git a/tests/fixtures/jev_api/choice_two_options.json b/tests/fixtures/jev_api/choice_two_options.json new file mode 100644 index 0000000..128c910 --- /dev/null +++ b/tests/fixtures/jev_api/choice_two_options.json @@ -0,0 +1,109 @@ +{ + "recorded": "2026-10-28", + "source": "POST https://api.typesafe.ai/v1/systemone, packet jev-api phase 1", + "scrubbed": "request headers (Authorization), response headers (request id, cookie, CF-RAY, date)", + "status": 200, + "latency_ms": 215.8, + "request": { + "model": "jev-1.13.0", + "state": { + "decision": 5, + "generation": 3, + "capacity": 3, + "population": [ + { + "id": "c1", + "score": [ + 1, + 1 + ], + "passed": false + }, + { + "id": "c2", + "score": [ + 2, + 2 + ], + "passed": false + }, + { + "id": "c3", + "score": [ + 0, + 0 + ], + "passed": false + } + ], + "remaining": { + "dollars": 4.85, + "evaluations": 18, + "candidates": 8 + }, + "history": { + "mutate": { + "attempted": 4, + "produced": 2, + "evaluated": 2, + "improved": 1, + "passed": 0, + "dollars": 0.081 + }, + "crossover": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "evaluate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "migrate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + } + }, + "mutator_exhausted": false + }, + "questions": { + "next_action": { + "type": "choice", + "instructions": "Choose the next action of a bounded evolutionary code search from the numbers in the state. Pick the action most likely to produce a passing candidate per dollar of remaining budget.", + "criteria": { + "crossover": "Merge the two named population members; no model call, one evaluation.", + "mutate": "Ask the model mutator for a child of the named parent; one model call, one evaluation." + } + } + } + }, + "response": { + "model": "jev-1.13.0", + "answers": { + "next_action": { + "type": "choice", + "choice": "mutate", + "confidence": 0.26, + "probabilities": { + "crossover": 0.37, + "mutate": 0.63 + } + } + }, + "usage": { + "input_tokens": 732, + "output_tokens": 36 + } + } +} diff --git a/tests/fixtures/jev_api/choice_with_noul.json b/tests/fixtures/jev_api/choice_with_noul.json new file mode 100644 index 0000000..702ac9d --- /dev/null +++ b/tests/fixtures/jev_api/choice_with_noul.json @@ -0,0 +1,103 @@ +{ + "recorded": "2026-10-28", + "source": "POST https://api.typesafe.ai/v1/systemone, packet jev-api phase 1", + "scrubbed": "request headers (Authorization), response headers (request id, cookie, CF-RAY, date)", + "status": 200, + "latency_ms": 243.6, + "request": { + "model": "jev-1.13.0", + "state": { + "decision": 3, + "generation": 1, + "capacity": 3, + "population": [ + { + "id": "c1", + "score": [ + 1, + 1 + ], + "passed": false + } + ], + "remaining": { + "dollars": 4.95, + "evaluations": 20, + "candidates": 10 + }, + "history": { + "mutate": { + "attempted": 2, + "produced": 2, + "evaluated": 2, + "improved": 1, + "passed": 0, + "dollars": 0.027 + }, + "crossover": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "evaluate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "migrate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + } + }, + "mutator_exhausted": false + }, + "questions": { + "next_action": { + "type": "choice", + "instructions": "Choose the next action of a bounded evolutionary code search from the numbers in the state. Pick the action most likely to produce a passing candidate per dollar of remaining budget.", + "criteria": { + "evaluate": "Evaluate the next seed candidate with the host's checks; no model call.", + "crossover": "Merge the two named population members; no model call, one evaluation.", + "mutate": "Ask the model mutator for a child of the named parent; one model call, one evaluation." + } + }, + "worth_continuing": { + "type": "noul", + "instructions": "Is another action likely to produce a passing candidate within the remaining budget?" + } + } + }, + "response": { + "model": "jev-1.13.0", + "answers": { + "next_action": { + "type": "choice", + "choice": "mutate", + "confidence": 0.3, + "probabilities": { + "evaluate": 0.45, + "crossover": 0.02, + "mutate": 0.53 + } + }, + "worth_continuing": { + "type": "noul", + "noul": 0.56 + } + }, + "usage": { + "input_tokens": 726, + "output_tokens": 63 + } + } +} diff --git a/tests/fixtures/jev_api/error_400_nonfinite.json b/tests/fixtures/jev_api/error_400_nonfinite.json new file mode 100644 index 0000000..81d628e --- /dev/null +++ b/tests/fixtures/jev_api/error_400_nonfinite.json @@ -0,0 +1,14 @@ +{ + "recorded": "2026-10-28", + "source": "POST https://api.typesafe.ai/v1/systemone, packet jev-api phase 1", + "scrubbed": "request headers (Authorization), response headers (request id, cookie, CF-RAY, date)", + "status": 400, + "latency_ms": 130.1, + "request": "{\"model\":\"jev-1.13.0\",\"state\":{\"x\":NaN},\"questions\":{\"q\":{\"type\":\"noul\",\"instructions\":\"Is x large?\"}}}", + "response": { + "detail": { + "error_type": "api_usage_error", + "message": "Request contains NaN or infinity." + } + } +} diff --git a/tests/fixtures/jev_api/error_400_short_pin.json b/tests/fixtures/jev_api/error_400_short_pin.json new file mode 100644 index 0000000..06299ae --- /dev/null +++ b/tests/fixtures/jev_api/error_400_short_pin.json @@ -0,0 +1,82 @@ +{ + "recorded": "2026-10-28", + "source": "POST https://api.typesafe.ai/v1/systemone, packet jev-api phase 1", + "scrubbed": "request headers (Authorization), response headers (request id, cookie, CF-RAY, date)", + "status": 400, + "latency_ms": 137.8, + "request": { + "model": "jev-1.13", + "state": { + "decision": 3, + "generation": 1, + "capacity": 3, + "population": [ + { + "id": "c1", + "score": [ + 1, + 1 + ], + "passed": false + } + ], + "remaining": { + "dollars": 4.95, + "evaluations": 20, + "candidates": 10 + }, + "history": { + "mutate": { + "attempted": 2, + "produced": 2, + "evaluated": 2, + "improved": 1, + "passed": 0, + "dollars": 0.027 + }, + "crossover": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "evaluate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "migrate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + } + }, + "mutator_exhausted": false + }, + "questions": { + "next_action": { + "type": "choice", + "instructions": "Choose the next action of a bounded evolutionary code search from the numbers in the state. Pick the action most likely to produce a passing candidate per dollar of remaining budget.", + "criteria": { + "evaluate": "Evaluate the next seed candidate with the host's checks; no model call.", + "crossover": "Merge the two named population members; no model call, one evaluation.", + "mutate": "Ask the model mutator for a child of the named parent; one model call, one evaluation." + } + } + } + }, + "response": { + "detail": { + "error_type": "api_usage_error", + "message": "Unknown model: jev-1.13" + } + } +} diff --git a/tests/fixtures/jev_api/error_400_unknown_model.json b/tests/fixtures/jev_api/error_400_unknown_model.json new file mode 100644 index 0000000..e6a52d2 --- /dev/null +++ b/tests/fixtures/jev_api/error_400_unknown_model.json @@ -0,0 +1,82 @@ +{ + "recorded": "2026-10-28", + "source": "POST https://api.typesafe.ai/v1/systemone, packet jev-api phase 1", + "scrubbed": "request headers (Authorization), response headers (request id, cookie, CF-RAY, date)", + "status": 400, + "latency_ms": 130.7, + "request": { + "model": "jev-9.9.9", + "state": { + "decision": 3, + "generation": 1, + "capacity": 3, + "population": [ + { + "id": "c1", + "score": [ + 1, + 1 + ], + "passed": false + } + ], + "remaining": { + "dollars": 4.95, + "evaluations": 20, + "candidates": 10 + }, + "history": { + "mutate": { + "attempted": 2, + "produced": 2, + "evaluated": 2, + "improved": 1, + "passed": 0, + "dollars": 0.027 + }, + "crossover": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "evaluate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "migrate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + } + }, + "mutator_exhausted": false + }, + "questions": { + "next_action": { + "type": "choice", + "instructions": "Choose the next action of a bounded evolutionary code search from the numbers in the state. Pick the action most likely to produce a passing candidate per dollar of remaining budget.", + "criteria": { + "evaluate": "Evaluate the next seed candidate with the host's checks; no model call.", + "crossover": "Merge the two named population members; no model call, one evaluation.", + "mutate": "Ask the model mutator for a child of the named parent; one model call, one evaluation." + } + } + } + }, + "response": { + "detail": { + "error_type": "api_usage_error", + "message": "Unknown model: jev-9.9.9" + } + } +} diff --git a/tests/fixtures/jev_api/error_401_bad_key.json b/tests/fixtures/jev_api/error_401_bad_key.json new file mode 100644 index 0000000..93e4b69 --- /dev/null +++ b/tests/fixtures/jev_api/error_401_bad_key.json @@ -0,0 +1,82 @@ +{ + "recorded": "2026-10-28", + "source": "POST https://api.typesafe.ai/v1/systemone, packet jev-api phase 1", + "scrubbed": "request headers (Authorization), response headers (request id, cookie, CF-RAY, date)", + "status": 401, + "latency_ms": 139.9, + "request": { + "model": "jev-1.13.0", + "state": { + "decision": 3, + "generation": 1, + "capacity": 3, + "population": [ + { + "id": "c1", + "score": [ + 1, + 1 + ], + "passed": false + } + ], + "remaining": { + "dollars": 4.95, + "evaluations": 20, + "candidates": 10 + }, + "history": { + "mutate": { + "attempted": 2, + "produced": 2, + "evaluated": 2, + "improved": 1, + "passed": 0, + "dollars": 0.027 + }, + "crossover": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "evaluate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "migrate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + } + }, + "mutator_exhausted": false + }, + "questions": { + "next_action": { + "type": "choice", + "instructions": "Choose the next action of a bounded evolutionary code search from the numbers in the state. Pick the action most likely to produce a passing candidate per dollar of remaining budget.", + "criteria": { + "evaluate": "Evaluate the next seed candidate with the host's checks; no model call.", + "crossover": "Merge the two named population members; no model call, one evaluation.", + "mutate": "Ask the model mutator for a child of the named parent; one model call, one evaluation." + } + } + } + }, + "response": { + "detail": { + "error_type": "authentication_error", + "message": "Cannot authenticate with the server. Please check your API key and try again." + } + } +} diff --git a/tests/fixtures/jev_api/error_403_no_key.json b/tests/fixtures/jev_api/error_403_no_key.json new file mode 100644 index 0000000..24fc90b --- /dev/null +++ b/tests/fixtures/jev_api/error_403_no_key.json @@ -0,0 +1,82 @@ +{ + "recorded": "2026-10-28", + "source": "POST https://api.typesafe.ai/v1/systemone, packet jev-api phase 1", + "scrubbed": "request headers (Authorization), response headers (request id, cookie, CF-RAY, date)", + "status": 403, + "latency_ms": 120.2, + "request": { + "model": "jev-1.13.0", + "state": { + "decision": 3, + "generation": 1, + "capacity": 3, + "population": [ + { + "id": "c1", + "score": [ + 1, + 1 + ], + "passed": false + } + ], + "remaining": { + "dollars": 4.95, + "evaluations": 20, + "candidates": 10 + }, + "history": { + "mutate": { + "attempted": 2, + "produced": 2, + "evaluated": 2, + "improved": 1, + "passed": 0, + "dollars": 0.027 + }, + "crossover": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "evaluate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "migrate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + } + }, + "mutator_exhausted": false + }, + "questions": { + "next_action": { + "type": "choice", + "instructions": "Choose the next action of a bounded evolutionary code search from the numbers in the state. Pick the action most likely to produce a passing candidate per dollar of remaining budget.", + "criteria": { + "evaluate": "Evaluate the next seed candidate with the host's checks; no model call.", + "crossover": "Merge the two named population members; no model call, one evaluation.", + "mutate": "Ask the model mutator for a child of the named parent; one model call, one evaluation." + } + } + } + }, + "response": { + "detail": { + "error_type": "authentication_error", + "message": "Must supply an API key! Check your request and try again." + } + } +} diff --git a/tests/fixtures/jev_api/error_422_missing_criteria.json b/tests/fixtures/jev_api/error_422_missing_criteria.json new file mode 100644 index 0000000..da68291 --- /dev/null +++ b/tests/fixtures/jev_api/error_422_missing_criteria.json @@ -0,0 +1,87 @@ +{ + "recorded": "2026-10-28", + "source": "POST https://api.typesafe.ai/v1/systemone, packet jev-api phase 1", + "scrubbed": "request headers (Authorization), response headers (request id, cookie, CF-RAY, date)", + "status": 422, + "latency_ms": 189.0, + "request": { + "model": "jev-1.13.0", + "state": { + "decision": 3, + "generation": 1, + "capacity": 3, + "population": [ + { + "id": "c1", + "score": [ + 1, + 1 + ], + "passed": false + } + ], + "remaining": { + "dollars": 4.95, + "evaluations": 20, + "candidates": 10 + }, + "history": { + "mutate": { + "attempted": 2, + "produced": 2, + "evaluated": 2, + "improved": 1, + "passed": 0, + "dollars": 0.027 + }, + "crossover": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "evaluate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + }, + "migrate": { + "attempted": 0, + "produced": 0, + "evaluated": 0, + "improved": 0, + "passed": 0, + "dollars": 0.0 + } + }, + "mutator_exhausted": false + }, + "questions": { + "next_action": { + "type": "choice", + "instructions": "Choose the next action of a bounded evolutionary code search from the numbers in the state. Pick the action most likely to produce a passing candidate per dollar of remaining budget." + } + } + }, + "response": { + "detail": [ + { + "type": "missing", + "loc": [ + "body", + "questions", + "next_action", + "choice", + "criteria" + ], + "msg": "Field required", + "input": "{\"type\":\"choice\",\"instructions\":\"Choose the next action of a bounded evolutionary code search from the numbers in the state. Pick the action most likely to produce a passing candidate per dollar of remaining budget.\"}" + } + ] + } +} diff --git a/tests/test_jev_api.py b/tests/test_jev_api.py new file mode 100644 index 0000000..adda403 --- /dev/null +++ b/tests/test_jev_api.py @@ -0,0 +1,636 @@ +"""TypeSafeJevPolicy (``syberlabs.jev_api``): TypeSafe's Jev API behind the BranchPolicy seam, without a network. + +Every response here was recorded from ``POST https://api.typesafe.ai/v1/systemone`` on +2026-10-28 (``tests/fixtures/jev_api/``, headers and request ids scrubbed) or is one of those +bodies with one field changed. No test opens a socket: the policy's transport is a recorded +replay, a stand-in that fails on purpose, or ``http_transport`` with a stand-in opener. What +is covered: the validation of an answer, each fallback and the switch to local after three +failures in a row, that the request carries ids and numbers and no free text, that the key +never reaches a record, a trace or an error, that the existing policies' traces are +byte-for-byte what they were before this module, and the ``evolve_jevapi`` benchmark arm. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import io +import json +import os +import random +import re +import tempfile +import unittest +import urllib.error +from pathlib import Path + +from syberlabs import Rejected +from syberlabs.build import SpendBudget +from syberlabs.canonical import canonical, digest +from syberlabs.evolve import EvolutionaryProvider, JevPolicy +from syberlabs.jev_api import (CRITERIA, DETAILS, ENDPOINT, INSTRUCTIONS, KEY_ENV, MODEL, QUESTION, REASONS, SOURCES, + TOLERANCE, JevFailure, TypeSafeJevPolicy, http_transport, validate) +from syberlabs.search import BranchPolicy, Individual, OperatorStats, Situation +from tests.test_jev import GENERATIONS, POPULATION, PRICE, PROFILES, Profile, World, run + +FIXTURES = Path(__file__).resolve().parent / "fixtures" / "jev_api" +RECORD_FIELDS = ("source", "model", "answers_digest", "request_digest", "latency_ms", "fallback", "failures", + "local_action") +KINDS3, KINDS4, KINDS2 = ("evaluate", "crossover", "mutate"), ("evaluate", "crossover", "mutate", "migrate"), ("crossover", "mutate") +CANARY_KEY = "ts-canary-7f3a91c2e5d84b06aa11" # set as TYPESAFE_API_KEY in the tests that need a key; never a real one + +# The traces of the existing policies on the in-memory pricing world (tests/test_jev.py), seeds 0 to 2, as +# computed on main at 0b03040 before this module existed: refactoring JevPolicy.decide changed none of them. +UNCHANGED = { + ("mutation_rare", "fixed"): "0a39d5562a7f5963e9a7e7bfd3a94fed5beb1e3996d9714b5de74b00971cdeb0", + ("mutation_rare", "empirical"): "626b5a92fd83c7aff5351c2a47230cfb0def9aaaa80c861dd4eb0f8a47b9eea9", + ("mutation_rare", "jev"): "998507ece7d01e47e5bdf3dac2ae5ec0c1f77759d3c6a53bbb6937a47d5e0d78", + ("mutation_common", "fixed"): "28e5966f06aac3b4ed1837597328fb12a7782916817985055620302e86e8cf56", + ("mutation_common", "empirical"): "50cee6b5d74c4c7a718e033b94ed6817d5d42f22a7b30676eddfeedbf4dd1929", + ("mutation_common", "jev"): "7d603eb2d10fe560e193f68c63f3ef44e3c779923a31d819c9ab739f07d30658", +} +UNCHANGED_UNPRICED_JEV = "efd800301bb6fa81debd2f9ced4cfd419b32f1c5a03aa3fcc18bdb2ec2ab012a" + + +def recorded(name: str) -> dict: + return json.loads((FIXTURES / f"{name}.json").read_text(encoding="utf-8")) + + +def response(name: str) -> dict: + return recorded(name)["response"] + + +def raw(value) -> bytes: + return json.dumps(value).encode("utf-8") + + +def changed(name: str, edit) -> bytes: + """A recorded body with one change, made by ``edit(body)`` in place.""" + body = response(name) + edit(body) + return raw(body) + + +def answer(body: dict) -> dict: + return body["answers"][QUESTION] + + +class Recorded: + """Replays the recorded 200 responses, picked by the option set the request offers, in turn.""" + + POOL = {frozenset(KINDS3): ["choice_three_options", "choice_repeat_a", "choice_repeat_b", "choice_low_confidence"], + frozenset(KINDS4): ["choice_four_options"], frozenset(KINDS2): ["choice_two_options"]} + + def __init__(self): + self.requests: list[dict] = [] + self.bodies: list[bytes] = [] + + def __call__(self, body: bytes, timeout: float) -> tuple[int, bytes]: + self.bodies.append(body) + request = json.loads(body) + self.requests.append(request) + names = self.POOL[frozenset(request["questions"][QUESTION]["criteria"])] + return 200, raw(response(names[(len(self.requests) - 1) % len(names)])) + + +class Failing: + """A transport that fails the way it is told to, ``plan`` entries in turn (the last one repeats).""" + + def __init__(self, *plan): + self.plan, self.calls = list(plan), 0 + + def __call__(self, body: bytes, timeout: float): + step = self.plan[min(self.calls, len(self.plan) - 1)] + self.calls += 1 + if isinstance(step, BaseException): + raise step + if callable(step): + return step(body) + return step + + +class KeyEnv: + """``TYPESAFE_API_KEY`` set to ``value`` (or removed for None) inside the block, restored after it.""" + + def __init__(self, value): + self.value = value + + def __enter__(self): + self.old = os.environ.get(KEY_ENV) + if self.value is None: + os.environ.pop(KEY_ENV, None) + else: + os.environ[KEY_ENV] = self.value + return self + + def __exit__(self, *exc): + if self.old is None: + os.environ.pop(KEY_ENV, None) + else: + os.environ[KEY_ENV] = self.old + return False + + +class FakeResponse: + def __init__(self, status: int, body: bytes): + self.status, self.body = status, body + + def read(self, n=-1): + return self.body if n is None or n < 0 else self.body[:n] + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +class Opener: + """Stands in for urllib's opener: records what would be sent, and answers or fails as told.""" + + def __init__(self, mode: str, body: bytes = b"", key: str = CANARY_KEY): + self.mode, self.body, self.key, self.seen = mode, body, key, [] + + def open(self, request, timeout): + self.seen.append({"url": request.full_url, "method": request.get_method(), "timeout": timeout, + "headers": dict(request.header_items()), "data": request.data}) + if self.mode == "ok": + return FakeResponse(200, self.body) + if self.mode == "http": + raise urllib.error.HTTPError(ENDPOINT, 401, f"refused {self.key}", {}, + io.BytesIO(raw({"detail": {"message": f"bad key {self.key}"}}))) + if self.mode == "url": + raise urllib.error.URLError(f"connection refused while sending {self.key}") + raise TimeoutError(f"timed out sending {self.key}") + + +def situation(population=(("c1", (1, 1)), ("c2", (2, 2))), *, decisions=4, dollars=4.5) -> Situation: + stats = {k: OperatorStats(dollars=0.0) for k in ("mutate", "crossover", "migrate", "evaluate")} + return Situation(generation=0, population=tuple(Individual(i, score, False) for i, score in population), capacity=3, + remaining=SpendBudget(10, 20, 300, dollars), room=True, history=stats, mutator_exhausted=False, + exchange=False, decisions=decisions, rng=random.Random(0)) + + +def search(transport, *, profile="mutation_rare", seed=0, policy=None, max_dollars=5.0): + """One search on the in-memory pricing world of tests/test_jev.py, branching by TypeSafeJevPolicy.""" + world = World(seed=seed, max_dollars=max_dollars) + policy = policy or TypeSafeJevPolicy(generations=GENERATIONS, prices={"mutate": PRICE}, transport=transport) + provider = EvolutionaryProvider(Profile(price=PRICE, **PROFILES[profile]), population=POPULATION, + generations=GENERATIONS, crossover_every=3, policy=policy) + provider.search(world) + return provider, world + + +def long_search(transport, *, seed=0): + """A search that does not stop at the first pass, so it makes enough decisions to fail three times in a row.""" + return search(transport, seed=seed, policy=TypeSafeJevPolicy(generations=12, prices={"mutate": PRICE}, + transport=transport, stop_when_passing=False)) + + +def decisions(trace): + return [e for e in trace if "decision" in e] + + +class Validation(unittest.TestCase): + """What an answer must be before it can become an action, on recorded bodies.""" + + def test_every_recorded_answer_validates_against_its_own_options(self): + for name, kinds in (("choice_three_options", KINDS3), ("choice_repeat_a", KINDS3), ("choice_repeat_b", KINDS3), + ("choice_low_confidence", KINDS3), ("choice_four_options", KINDS4), + ("choice_two_options", KINDS2)): + found = validate(200, raw(response(name)), kinds) + body = response(name) + self.assertEqual(found["choice"], answer(body)["choice"], name) + self.assertEqual(found["model"], MODEL) + self.assertEqual(found["answers_digest"], digest(body["answers"])) + self.assertRegex(found["answers_digest"], r"^[0-9a-f]{64}$") + # The live format: two-decimal probabilities whose sum is off 1 by at most 0.01, within the tolerance. + self.assertLessEqual(abs(sum(found["probabilities"].values()) - 1), 0.0100001, name) + self.assertGreater(found["usage"]["input_tokens"], 0) + + def test_the_two_recorded_repeats_of_one_request_differ(self): + # Same request body, same model, two answers: Jev's probabilities are not a pure function of the request. + a, b = recorded("choice_repeat_a"), recorded("choice_repeat_b") + self.assertEqual(a["request"], b["request"]) + self.assertNotEqual(answer(a["response"])["probabilities"], answer(b["response"])["probabilities"]) + + def test_the_served_model_must_be_the_pinned_one(self): + for name in ("choice_served_by_1_14", "choice_alias_latest"): # jev-latest is served as jev-1.14.0 + self.assertEqual(response(name)["model"], "jev-1.14.0") + with self.assertRaises(JevFailure) as caught: + validate(200, raw(response(name)), KINDS3) + self.assertEqual((caught.exception.reason, caught.exception.detail), ("jev_invalid", "model")) + self.assertEqual(validate(200, raw(response("choice_served_by_1_14")), KINDS3, model="jev-1.14.0")["model"], + "jev-1.14.0") + + def test_the_pin_is_an_exact_version(self): + for model in ("jev-latest", "jev-preview", "jev-1.13", "jev-1.13.0-rc", "", None): + with self.assertRaises(Rejected, msg=model): + TypeSafeJevPolicy(generations=5, model=model) + for bad in (dict(timeout=0), dict(timeout=121), dict(max_failures=0), dict(tolerance=0), dict(transport=1)): + with self.assertRaises(Rejected, msg=bad): + TypeSafeJevPolicy(generations=5, **bad) + policy = TypeSafeJevPolicy(generations=5, model="jev-1.14.0") + self.assertIsInstance(policy, BranchPolicy) + self.assertEqual((policy.name, policy.revision, TypeSafeJevPolicy(generations=1).model), ("jev_api", "1", MODEL)) + self.assertIs(TypeSafeJevPolicy(generations=1).transport, http_transport) + + def test_each_broken_answer_is_refused_with_its_own_detail(self): + base = "choice_three_options" + cases = { + "choice": [lambda b: answer(b).update(choice=None)], + "probabilities": [lambda b: answer(b)["probabilities"].pop("crossover"), + lambda b: answer(b)["probabilities"].update(migrate=0.0), + lambda b: answer(b)["probabilities"].update(mutate=1.5), + lambda b: answer(b)["probabilities"].update(crossover=-0.01), + lambda b: answer(b)["probabilities"].update(crossover=True), + lambda b: answer(b)["probabilities"].update(crossover="0.03"), + lambda b: answer(b)["probabilities"].update(mutate=0.48), # sums to 0.90 + lambda b: answer(b).update(probabilities=[0.39, 0.03, 0.58])], + "argmax": [lambda b: answer(b).update(choice="evaluate")], + "confidence": [lambda b: answer(b).pop("confidence"), lambda b: answer(b).update(confidence=1.2), + lambda b: answer(b).update(confidence="high")], + "usage": [lambda b: b.pop("usage"), lambda b: b["usage"].update(input_tokens=-1), + lambda b: b["usage"].update(output_tokens=1.5)], + "type": [lambda b: answer(b).update(type="noul"), lambda b: b["answers"].update({QUESTION: [1]})], + "answers": [lambda b: b.pop("answers"), lambda b: b["answers"].update(extra={"type": "noul", "noul": 0.5}), + lambda b: b.update(answers={"other": answer(b)})], + "model": [lambda b: b.update(model="jev-latest"), lambda b: b.pop("model")], + } + for detail, edits in cases.items(): + for i, edit in enumerate(edits): + with self.subTest(detail=detail, case=i): + with self.assertRaises(JevFailure) as caught: + validate(200, changed(base, edit), KINDS3) + self.assertEqual((caught.exception.reason, caught.exception.detail), ("jev_invalid", detail)) + with self.assertRaises(JevFailure) as caught: + validate(200, changed(base, lambda b: answer(b).update(choice="migrate")), KINDS3) + self.assertEqual(caught.exception.reason, "choice_outside_options") + # A noul answered beside the choice is an answer to a question this policy did not ask. + with self.assertRaises(JevFailure) as caught: + validate(200, raw(response("choice_with_noul")), KINDS3) + self.assertEqual(caught.exception.detail, "answers") + + def test_the_tolerance_is_stated_and_holds_at_its_edge(self): + self.assertEqual(TOLERANCE, 0.02) + inside = changed("choice_three_options", lambda b: answer(b)["probabilities"].update(mutate=0.565)) # 0.985 + self.assertEqual(validate(200, inside, KINDS3)["choice"], "mutate") + outside = changed("choice_three_options", lambda b: answer(b)["probabilities"].update(mutate=0.555)) # 0.975 + with self.assertRaises(JevFailure): + validate(200, outside, KINDS3) + + def test_a_body_that_is_not_one_strict_json_object_is_refused(self): + text = raw(response("choice_three_options")).decode() + for body, detail in ((text.replace("0.58", "NaN").encode(), "not_json"), + (text.replace("0.58", "Infinity").encode(), "not_json"), + (text.replace('{"model"', '{"model": "jev-1.13.0", "model"', 1).encode(), "not_json"), + (b"\xff\xfe{}", "not_json"), (b"[]", "not_object"), (b"", "not_json"), + (b" " * 70_000, "too_large")): + with self.subTest(detail=detail, head=body[:30]): + with self.assertRaises(JevFailure) as caught: + validate(200, body, KINDS3) + self.assertEqual((caught.exception.reason, caught.exception.detail), ("jev_invalid", detail)) + + def test_a_non_200_is_unavailable_and_names_only_its_status_class(self): + for name, status, detail in (("error_401_bad_key", 401, "http_401"), ("error_403_no_key", 403, "http_403"), + ("error_400_unknown_model", 400, "http_4xx"), ("error_400_short_pin", 400, "http_4xx"), + ("error_400_nonfinite", 400, "http_4xx"), ("error_422_missing_criteria", 422, "http_4xx")): + fixture = recorded(name) + self.assertEqual(fixture["status"], status, name) + with self.assertRaises(JevFailure) as caught: + validate(fixture["status"], raw(fixture["response"]), KINDS3) + self.assertEqual((caught.exception.reason, caught.exception.detail), ("jev_unavailable", detail)) + self.assertNotIn(json.dumps(fixture["response"])[:40], str(caught.exception)) + for status, detail in ((429, "http_429"), (529, "http_529"), (500, "http_5xx"), (302, "http_other")): + with self.assertRaises(JevFailure) as caught: + validate(status, b"{}", KINDS3) + self.assertEqual(caught.exception.detail, detail) + self.assertTrue(set(REASONS) >= {"jev_timeout", "jev_unavailable", "jev_invalid", "choice_outside_options"}) + self.assertIn("http_429", DETAILS) + + +class Fallbacks(unittest.TestCase): + """A timeout, an invalid answer or an unavailable API: the local JevPolicy decides, and the record says why.""" + + def first_asked(self, transport): + provider, _world = search(transport) + asked = [e for e in decisions(provider.trace) if e.get("request_digest")] + self.assertTrue(asked, "the search asked Jev at least once") + return asked[0], provider + + def test_each_failure_falls_back_to_the_local_action_with_its_reason(self): + invalid_model = (200, raw(response("choice_served_by_1_14"))) + outside = (200, changed("choice_two_options", lambda b: answer(b).update(choice="evaluate"))) + cases = { + ("jev_timeout", "timeout"): TimeoutError("read timed out"), + ("jev_unavailable", "transport"): OSError("connection reset"), + ("jev_unavailable", "http_401"): (401, raw(response("error_401_bad_key"))), + ("jev_unavailable", "http_429"): (429, b'{"detail": "slow down"}'), + ("jev_invalid", "model"): invalid_model, + ("jev_invalid", "not_json"): (200, b""), + ("choice_outside_options", "choice"): outside, + } + for (reason, detail), step in cases.items(): + with self.subTest(reason=reason, detail=detail): + entry, provider = self.first_asked(Failing(step)) + self.assertEqual(entry["source"], "local") + self.assertEqual((entry["fallback"]["reason"], entry["fallback"]["detail"]), (reason, detail)) + self.assertEqual(entry["action"], entry["local_action"]) + self.assertEqual(entry["failures"], 1) + self.assertIsNone(entry["answers_digest"]) + self.assertIsNotNone(entry["latency_ms"]) + self.assertEqual(provider.trace[-1]["action"], "stop") + + def test_no_key_falls_back_without_opening_a_socket(self): + opener = Opener("ok", raw(response("choice_two_options"))) + with KeyEnv(None): + entry, _ = self.first_asked(lambda body, timeout: http_transport(body, timeout, opener=opener)) + self.assertEqual((entry["fallback"]["reason"], entry["fallback"]["detail"]), ("jev_unavailable", "no_key")) + self.assertEqual(opener.seen, [], "nothing was sent") + + @staticmethod + def decide_in_turn(transport, n: int) -> tuple[TypeSafeJevPolicy, list[dict]]: + """``n`` decisions, each over a fresh pair below capacity, so each offers a crossover and a mutation.""" + policy = TypeSafeJevPolicy(generations=60, transport=transport) + policy.reset() + entries = [] + for i in range(n): + action = policy.decide(situation(((f"c{2 * i + 1}", (1, 1)), (f"c{2 * i + 2}", (2, 2))), decisions=i)) + entries.append({**policy.audit(), "action": action.kind}) + return policy, entries + + def test_three_failures_in_a_row_switch_to_local_for_the_rest_of_the_search(self): + transport = Failing(TimeoutError("slow")) + policy, entries = self.decide_in_turn(transport, 6) + self.assertEqual(transport.calls, 3) + self.assertEqual(policy.calls, 3) + self.assertEqual([e["failures"] for e in entries[:3]], [1, 2, 3]) + self.assertEqual([e["fallback"]["reason"] for e in entries[:3]], ["jev_timeout"] * 3) + self.assertNotIn("switched_to_local", entries[1]["fallback"]) + self.assertTrue(entries[2]["fallback"]["switched_to_local"]) + for entry in entries[3:]: + self.assertEqual(entry["source"], "local") + self.assertEqual(entry["fallback"], {"reason": "local_only", "detail": "switched"}) + self.assertIsNone(entry["request_digest"]) + self.assertEqual(len(entry["alternatives"]), 2, "a choice was offered and Jev was not asked") + self.assertEqual(entry["action"], entry["local_action"]) + # The next search starts with Jev again: the switch lasts one search. + policy.reset() + self.assertEqual((policy._failures, policy._local_only, policy.calls), (0, False, 0)) + # The same in a whole search: three calls, then none. + transport = Failing(TimeoutError("slow")) + provider, _ = long_search(transport) + asked = [e for e in decisions(provider.trace) if e.get("request_digest")] + self.assertEqual((transport.calls, len(asked), asked[-1]["fallback"].get("switched_to_local")), (3, 3, True)) + + def test_a_success_resets_the_count_of_failures_in_a_row(self): + ok = (200, raw(response("choice_two_options"))) + transport = Failing(TimeoutError("a"), TimeoutError("b"), ok, TimeoutError("c"), TimeoutError("d"), ok) + policy, entries = self.decide_in_turn(transport, 6) + self.assertEqual(transport.calls, 6) + self.assertEqual([e["failures"] for e in entries], [1, 2, 0, 1, 2, 0]) + self.assertEqual([e["source"] for e in entries], ["local", "local", "jev_api", "local", "local", "jev_api"]) + self.assertFalse(any((e.get("fallback") or {}).get("switched_to_local") for e in entries)) + self.assertEqual([e["action"] for e in entries if e["source"] == "jev_api"], ["mutate", "mutate"]) + + def test_with_every_call_failing_the_search_is_the_local_policys_search(self): + record = set(RECORD_FIELDS) + strip = lambda e: {k: v for k, v in e.items() if k not in record and k != "policy"} # noqa: E731 + for seed in (0, 1, 2): + failed, failed_world = search(Failing(OSError("down")), seed=seed) + local = EvolutionaryProvider(Profile(price=PRICE, **PROFILES["mutation_rare"]), population=POPULATION, + generations=GENERATIONS, crossover_every=3, + policy=JevPolicy(generations=GENERATIONS, prices={"mutate": PRICE})) + local_world = World(seed=seed, max_dollars=5.0) + local.search(local_world) + self.assertEqual([strip(e) for e in failed.trace], [strip(e) for e in local.trace], seed) + self.assertEqual(failed_world.dollars, local_world.dollars) + + +class RecordedSearch(unittest.TestCase): + """Whole searches with recorded answers: every decision recorded, Jev only ever choosing an offered action.""" + + @classmethod + def setUpClass(cls): + cls.runs = [] + for seed in (0, 1, 2): + transport = Recorded() + provider, world = search(transport, seed=seed) + cls.runs.append((transport, provider, world)) + + def test_every_decision_carries_the_record(self): + for _transport, provider, _world in self.runs: + for entry in decisions(provider.trace): + self.assertTrue(all(field in entry for field in RECORD_FIELDS), entry) + self.assertIn(entry["source"], SOURCES) + self.assertEqual(entry["policy"], {"name": "jev_api", "revision": "1"}) + self.assertEqual(entry["model"], MODEL) + if entry["source"] == "jev_api": + self.assertRegex(entry["answers_digest"], r"^[0-9a-f]{64}$") + self.assertRegex(entry["request_digest"], r"^[0-9a-f]{64}$") + self.assertEqual(entry["action"], entry["jev"]["choice"]) + self.assertEqual(entry["chosen_by"], "jev_api") + self.assertIn(entry["action"], {a["action"] for a in entry["alternatives"]}) + self.assertIsNone(entry["fallback"]) + self.assertEqual(set(entry["jev"]["probabilities"]), {a["action"] for a in entry["alternatives"]}) + elif entry["source"] == "rule": + self.assertEqual(entry["action"], "stop") + self.assertTrue(entry["chosen_by"].startswith("rule:")) + self.assertIsNone(entry["request_digest"]) + else: + self.assertIsNone(entry["request_digest"], "local without a call: only one action was offered") + self.assertEqual(decisions(provider.trace)[-1]["source"], "rule", "only the local rules stop a search") + + def test_jev_was_asked_and_its_digests_match_what_was_sent_and_returned(self): + for transport, provider, _world in self.runs: + asked = [e for e in decisions(provider.trace) if e["source"] == "jev_api"] + self.assertEqual(len(asked), len(transport.requests)) + self.assertEqual(provider.policy.calls, len(transport.requests)) + self.assertGreater(len(asked), 0) + for entry, body in zip(asked, transport.bodies): + self.assertEqual(entry["request_digest"], hashlib.sha256(body).hexdigest()) + self.assertGreater(provider.policy.input_tokens, 0) + self.assertAlmostEqual(provider.policy.jev_dollars, provider.policy.input_tokens * 0.042e-6, places=9) + + def test_the_signal_stays_within_the_bound_and_names_the_policy(self): + for _transport, _provider, world in self.runs: + self.assertTrue(all(n < 4096 for n in world.signal_bytes)) + self.assertTrue(all(c.signal["policy"]["name"] == "jev_api" for c in world.views.values())) + + +ALLOWED_KEYS = {"decision", "generation", "generations_left", "capacity", "population", "slot", "score", "passed", + "remaining", "candidates", "evaluations", "seconds", "dollars", "history", "attempted", "produced", + "evaluated", "improved", "mutator_exhausted", "options", "parent_slot", "slots", "seeds_left", + "expected_value", "expected_dollars", "expected_evaluations", "price", *JevPolicy.KINDS} + + +class Request(unittest.TestCase): + """What leaves the process: ids and numbers in the state, and only this module's fixed text in the question.""" + + def capture(self, s, **options): + transport = Recorded() + policy = TypeSafeJevPolicy(generations=10, transport=transport, **options) + policy.reset() + action = policy.decide(s) + return transport, policy, action + + def walk(self, value, path=""): + if isinstance(value, dict): + for key, item in value.items(): + self.assertIn(key, ALLOWED_KEYS, f"{path}: key {key!r}") + self.walk(item, f"{path}.{key}") + elif isinstance(value, list): + for i, item in enumerate(value): + self.walk(item, f"{path}[{i}]") + else: + self.assertTrue(value is None or type(value) in (int, float, bool), f"{path}: {value!r} is not a number") + + def test_the_state_holds_no_text_and_the_question_only_fixed_text(self): + canary = "CANARY src/secret_path.py objective prompt" + s = situation(((canary, (1, 1)), ("c2-" + canary, (2, 2)))) + transport, _policy, action = self.capture(s, seeds=["seed-" + canary]) + self.assertEqual(len(transport.requests), 1) + request = transport.requests[0] + self.assertEqual(set(request), {"model", "state", "questions"}) + self.assertEqual(request["model"], MODEL) + self.walk(request["state"]) + self.assertEqual(set(request["questions"]), {QUESTION}) + question = request["questions"][QUESTION] + self.assertEqual(question["type"], "choice") + self.assertEqual(question["instructions"], INSTRUCTIONS) + self.assertEqual(question["criteria"], {k: CRITERIA[k] for k in KINDS3}) + self.assertNotIn("CANARY", transport.bodies[0].decode()) + self.assertNotIn("c2", transport.bodies[0].decode(), "population members are slots, not candidate ids") + self.assertIn(action.kind, KINDS3) + + def test_a_single_offered_action_is_taken_without_a_call(self): + s = situation((("c1", (1, 1)),)) # one individual: no pair; below capacity: mutate base only + transport, policy, action = self.capture(s) + self.assertEqual(transport.requests, []) + self.assertEqual((action.kind, action.parent), ("mutate", "base")) + self.assertEqual(policy.audit()["source"], "local") + + def test_a_stop_is_the_local_rule_and_asks_nothing(self): + s = dataclasses.replace(situation((("c1", (3, 3)),)), population=(Individual("c1", (3, 3), True),)) + transport, policy, action = self.capture(s) + self.assertEqual((action.kind, action.reason), ("stop", "passing")) + self.assertEqual(transport.requests, []) + self.assertEqual(policy.audit()["source"], "rule") + + +class Key(unittest.TestCase): + """The key goes to TypeSafe in one header and nowhere else: not a record, a trace, an error or the policy.""" + + def run_with(self, opener): + with KeyEnv(CANARY_KEY): + provider, world = long_search(lambda body, timeout: http_transport(body, timeout, opener=opener)) + return provider, world + + def assert_absent(self, provider, world): + self.assertNotIn(CANARY_KEY, json.dumps(provider.trace, default=str)) + self.assertNotIn(CANARY_KEY, repr(vars(provider.policy))) + self.assertNotIn(CANARY_KEY, json.dumps([c.signal for c in world.views.values()], default=str)) + + def test_the_key_is_sent_as_the_bearer_header_only(self): + opener = Opener("ok", raw(response("choice_two_options"))) + provider, world = self.run_with(opener) + self.assertTrue(opener.seen) + for sent in opener.seen: + self.assertEqual((sent["url"], sent["method"]), (ENDPOINT, "POST")) + self.assertEqual(sent["headers"]["Authorization"], "Bearer " + CANARY_KEY) + self.assertNotIn(CANARY_KEY, sent["data"].decode()) + self.assertEqual(sent["timeout"], 30.0) + self.assertTrue(any(e["source"] == "jev_api" for e in decisions(provider.trace))) + self.assert_absent(provider, world) + + def test_the_key_in_a_failure_never_reaches_the_record_or_the_error(self): + for mode, detail in (("http", "http_401"), ("url", "transport"), ("timeout", "timeout")): + with self.subTest(mode=mode): + opener = Opener(mode) + provider, world = self.run_with(opener) + asked = [e for e in decisions(provider.trace) if e.get("request_digest")] + self.assertEqual([e["fallback"]["detail"] for e in asked], [detail] * 3) + self.assert_absent(provider, world) + with KeyEnv(CANARY_KEY): + try: + status, body = http_transport(b"{}", 1.0, opener=opener) + except JevFailure as failure: + self.assertNotIn(CANARY_KEY, str(failure)) + self.assertNotIn(CANARY_KEY, repr(vars(failure))) + else: + with self.assertRaises(JevFailure) as caught: + validate(status, body, KINDS3) + self.assertNotIn(CANARY_KEY, str(caught.exception)) + self.assertNotIn(CANARY_KEY, repr(vars(caught.exception))) + + def test_a_key_with_a_line_break_is_not_sent(self): + opener = Opener("ok", raw(response("choice_two_options"))) + with KeyEnv(CANARY_KEY + "\r\nX-Other: 1"): + with self.assertRaises(JevFailure) as caught: + http_transport(b"{}", 1.0, opener=opener) + self.assertEqual(caught.exception.detail, "no_key") + self.assertEqual(opener.seen, []) + + def test_the_fixtures_hold_no_key_request_id_or_cookie(self): + for path in sorted(FIXTURES.glob("*.json")): + text = path.read_text(encoding="utf-8") + self.assertIsNone(re.search(r"Bearer|req_[0-9a-f]{8}|__cf_bm|ts-[A-Za-z0-9]{20}", text), path.name) + + +class Unchanged(unittest.TestCase): + """The existing policies, after JevPolicy.decide was split into _plan and _take, give the same traces as before.""" + + def test_the_traces_of_fixed_empirical_and_jev_are_byte_identical(self): + for (profile, policy), expected in UNCHANGED.items(): + with self.subTest(profile=profile, policy=policy): + self.assertEqual(digest([[run(policy, profile, s)["trace"] for s in (0, 1, 2)]]), expected) + self.assertEqual(digest([run("jev", "mutation_rare", s, max_dollars=None)["trace"] for s in (0, 1, 2)]), + UNCHANGED_UNPRICED_JEV) + self.assertEqual((JevPolicy.name, JevPolicy.revision), ("jev", "1")) + + def test_jevs_plan_takes_nothing(self): + policy = JevPolicy(generations=10) + policy.reset() + policy._crossed.add(frozenset(("c7", "c8"))) + before = canonical({"g": policy._generation, "seeds": policy._seeds, "crossed": sorted(map(sorted, policy._crossed)), + "mutated": policy._mutated}) + empty = dataclasses.replace(situation(()), mutator_exhausted=True) # nothing offered: the "empty" stop + passing = dataclasses.replace(situation(), population=(Individual("c1", (3, 3), True),)) + broke = dataclasses.replace(situation(), room=False) + for s in (situation(), empty, passing, broke, situation(), empty): + policy._plan(s) + self.assertEqual([policy._plan(s)[0].reason for s in (empty, passing, broke)], ["empty", "passing", "budget"]) + after = canonical({"g": policy._generation, "seeds": policy._seeds, "crossed": sorted(map(sorted, policy._crossed)), + "mutated": policy._mutated}) + self.assertEqual(before, after) + + +class Arm(unittest.TestCase): + """The ``evolve_jevapi`` arm of benchmarks/model_vs_patch.py, with the simulated model and recorded Jev answers.""" + + def test_the_arm_is_named_only_on_request_and_records_jev(self): + from benchmarks import model_vs_patch as harness + self.assertIn("evolve_jevapi", harness.ALL_ARMS) + self.assertNotIn("evolve_jevapi", harness.ARMS, "a default run makes no TypeSafe call") + self.assertEqual(harness.EVOLVE_ARMS["evolve_jevapi"], "jev_api") + with tempfile.TemporaryDirectory() as folder: + journals = Path(folder) / "rows-journals" + row = harness.run("pricing", "evolve_jevapi", harness.SIMULATED, 3, 1, Path(folder) / "runs", journals, + jev_transport=Recorded()) + self.assertEqual(row["policy"], {"name": "jev_api", "revision": "1"}) + jev = row["jev_api"] + self.assertEqual(jev["model"], MODEL) + self.assertEqual(sum(jev["by_source"].values()), len(json.loads( + (journals / "pricing-evolve_jevapi-1.decisions.json").read_text(encoding="utf-8")))) + self.assertEqual((jev["calls"], jev["fallbacks"], jev["switched_to_local"]), (jev["asked"], {}, False)) + self.assertGreater(jev["calls"], 0) + self.assertGreater(jev["input_tokens"], 0) + self.assertEqual(jev["decisions"], "rows-journals/pricing-evolve_jevapi-1.decisions.json") + self.assertTrue((journals / "pricing-evolve_jevapi-1.json").is_file()) + report = harness.summarize([row], 3) + self.assertIn("evolve_jevapi", report) + self.assertIn("jev model", report) + + +if __name__ == "__main__": + unittest.main()