feat(probably-decisions): add Probably-judged Sentinel decision benchmark

src/benchmarks/benchmark-config.ts

@@ -22,6 +22,10 @@ import {
 } from "./benchmark-meta";
 import { DEFAULT_STEP_LIMIT as DEEP_SWE_DEFAULT_STEP_LIMIT } from "./deep-swe/schema";
 import { DracoPanelConfigSchema } from "./draco/schemas";
+import {
+  PROBABLY_DECISIONS_ID,
+  ProbablyDecisionsOptionsSchema,
+} from "./probably-decisions/schema";
 import { SearchLaneConfigSchema } from "./search/core/config";
 import { DEFAULT_JUDGE_MODEL, DEFAULT_STEP_LIMIT } from "./swe-atlas/schema";
 import { BankingRetrievalConfigSchema } from "./tau3-bench-banking/retrieval-config";
@@ -190,6 +194,16 @@ export type IfStructBenchmarkConfig = z.infer<
   typeof IfStructBenchmarkConfigSchema
 >;
 
+export const ProbablyDecisionsConfigSchema = z.object({
+  benchmarkId: z.literal(PROBABLY_DECISIONS_ID),
+  ...FixedTemperatureBenchmarkBaseSchema.shape,
+  ...ProbablyDecisionsOptionsSchema.shape,
+});
+
+export type ProbablyDecisionsConfig = z.infer<
+  typeof ProbablyDecisionsConfigSchema
+>;
+
 const AgenticOptionsSchema = z.object({
   taskSubset: z.array(z.string()).optional(),
   maxAgentTimeoutSec: z.number().positive().optional(),
@@ -334,6 +348,7 @@ export const NativeBenchmarkRunConfigSchema = z.discriminatedUnion(
     TerminalBenchConfigSchema,
     DracoBenchmarkConfigSchema,
     IfStructBenchmarkConfigSchema,
+    ProbablyDecisionsConfigSchema,
     SweAtlasQaConfigSchema,
     SweAtlasTwConfigSchema,
     SweAtlasRfConfigSchema,
@@ -368,6 +383,7 @@ export const BENCHMARK_OPTIONS_SCHEMAS = {
   mmmu_pro_vision: MmmuProVisionOptionsSchema,
   terminal_bench: TerminalBenchOptionsSchema,
   ifstruct: IfStructOptionsSchema,
+  [PROBABLY_DECISIONS_ID]: ProbablyDecisionsOptionsSchema,
   swe_atlas_qa: SweAtlasOptionsSchema,
   swe_atlas_tw: SweAtlasOptionsSchema,
   swe_atlas_rf: SweAtlasOptionsSchema,

src/benchmarks/benchmark-meta.ts

@@ -49,6 +49,12 @@ export const IFSTRUCT_META = {
   defaultEpochs: 1,
 } as const satisfies BenchmarkMeta;
 
+export const PROBABLY_DECISIONS_META = {
+  id: "probably_decisions",
+  defaultEpochs: 1,
+  temperature: 0,
+} as const satisfies BenchmarkMeta;
+
 export const SWE_ATLAS_QA_META = {
   id: "swe_atlas_qa",
   defaultEpochs: 3,
@@ -108,6 +114,7 @@ const BENCHMARK_META: Readonly<Record<string, BenchmarkMeta>> = {
   [TERMINAL_BENCH_META.id]: TERMINAL_BENCH_META,
   [DRACO_META.id]: DRACO_META,
   [IFSTRUCT_META.id]: IFSTRUCT_META,
+  [PROBABLY_DECISIONS_META.id]: PROBABLY_DECISIONS_META,
   [SWE_ATLAS_QA_META.id]: SWE_ATLAS_QA_META,
   [SWE_ATLAS_TW_META.id]: SWE_ATLAS_TW_META,
   [SWE_ATLAS_RF_META.id]: SWE_ATLAS_RF_META,

src/benchmarks/probably-decisions/README.md

@@ -0,0 +1,50 @@
+# probably_decisions
+
+Evaluates a model as the **judgment step** of a Trust and Safety decision program written in [Probably](https://probably-lang.southpolesteve.workers.dev/), and optionally as the **research agent** that assembles the dossier the program judges.
+
+## How a sample runs
+
+1. `programs.ts` holds the fixed Probably program (`sentinel_case_v1`). It encodes the Sentinel scanner authority rules as `feels` / `match` judgments: compromised-key gate, static-attribute-only leads hold, load versus abuse, two-signal corroboration, remedy selection and the `account_ban` escalation bar. The program is the benchmark contract. A change to it is a benchmark behaviour change.
+2. `probably/runtime.ts` interprets the program. Every semantic judgment is delegated to a `Provider.judge(value, labels)` call, which `judge.ts` satisfies in one of two ways. Chat models are prompted for a JSON probability distribution over the lettered labels. Decisions models (`typesafe/*`, e.g. `~typesafe/jev-latest`) are called natively through `POST /api/alpha/decisions` with a single `choice` question whose criteria are the labels, and the returned `probabilities` are mapped back to labels. Routing is by model id (`isDecisionsModel` in `src/providers/decisions-client.ts`), no extra flag.
+3. **judgment mode** feeds the redacted dossier plus evidence sections straight into `input()`. **research mode** first runs a tool loop (`list_evidence_sections`, `read_evidence_section`, `submit_dossier`) with the model under test, then judges the submitted dossier with the configured `judgeModel` (default: the same model). Holding the program fixed lets a wrong action be attributed to research or to judgment.
+4. `scorer.ts` compares the program's final `print` to the gold action and records per-decision branch agreement, a Brier score on the chosen-branch distributions, enactment agreement (`hold` versus any restriction), evidence coverage and gold-fact recall (research mode only).
+
+The primary score is `action_accuracy`. Run-level metrics also include `enactment_agreement`, `macro_recall`, `hold_precision`, `brier`, `branch_agreement`, `evidence_coverage`, `fact_recall`, `run_failure_rate` and `recall_<action>`.
+
+## Dataset
+
+`__fixtures__/sentinel-sample.jsonl` is a 102-record stratified sample (every denied and reverted case, plus 12 approved cases per action) built from OpenRouter's internal Sentinel ban-candidate queue. It ships in the repository so the benchmark runs without network access. A larger build of the same generator can be pointed at with `datasetUrl` (`file:` or `https:`).
+
+**Provenance.** Each record is one Sentinel suggestion (a scanner-proposed restriction on one or more accounts) read through the Sentinel ban-candidates CLI, joined with the human review outcome. The gold action is derived from the reviewed target state:
+
+- `approved` targets keep the proposed remedy (`frontier_block`, `inference_block`, `throttle`, `key_revocation`, `account_ban`).
+- `denied` and `reverted` suggestions map to `hold` (file for review, do not enact).
+
+`gold.facts` are short evidence tokens (client fingerprints, model slugs, bucketed counts) that a research agent should surface in its dossier. They are matched by case-insensitive substring.
+
+**Redaction.** The generator is field-aware and deterministic. Emails, names, Clerk IDs, API keys, IPv4 addresses, Slack references, internal links and suggestion UUIDs are removed or pseudonymised (`<account>`, `<ip>`, `case-<sha256 prefix>`), numeric identifiers become `<id>`, and record ids are a SHA-256 prefix of the original suggestion id. Gold facts that are numeric-only or contain an identifier are dropped. `dataset.test.ts` re-asserts these invariants over every string field of the bundled fixture.
+
+**Licensing.** Internal OpenRouter data. Not for redistribution outside the organisation. Generated Slack summaries were used only as leads, never as labels. Nothing in this benchmark files, approves, enacts or reverts a restriction. Those paths stay behind the Sentinel ban-candidates API.
+
+## Config
+
+```jsonc
+{
+  "benchmarkId": "probably_decisions",
+  "model": "openai/gpt-5",
+  "reasoningEffort": "medium",
+  "mode": "judgment", // or "research"
+  "judgeModel": "openai/gpt-5-mini", // research mode, optional
+  "maxResearchSteps": 16,
+  "program": "sentinel_case_v1",
+  "datasetUrl": "file:///path/to/probably-decisions.jsonl", // optional
+}
+```
+
+Temperature is fixed at 0. `maxTokens` is intentionally never set. Use `chunkSize: 1` for research mode.
+
+### Jev and other Decisions models
+
+- **judgment mode**: set `model` to `~typesafe/jev-latest`. Every `feels` / `match` is one Decisions request. Temperature and reasoning settings do not apply to the Decisions endpoint. Provider preferences (`providerOnly`, `providerIgnore`, `allowFallbacks`, `sort`) are forwarded as `provider`.
+- **research mode**: the tool loop needs a chat model, so `model` must be a chat model and Jev goes in `judgeModel`. A Decisions model as `model` in research mode fails at startup with an explanatory error.
+- Judge transcripts still record the lettered prompt and the returned distribution as JSON, so traces from chat and Decisions judges are comparable.