diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 67bf72a9b..b7f173887 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -501,6 +501,80 @@ export const removeClipArgs = z.object({ clipId: z.string().min(1), }); +/** + * Every tool the model is handed, in the order `buildTools` builds them. + * + * The roster lives here, beside `MUTATING_TOOL_NAMES`, rather than in + * `deep-agent/service.ts` where `buildTools` is: the workbench needs to name the + * surface from its L0 layer, and importing the service would drag LangChain into + * a layer that deliberately runs on zod and pure document helpers alone. + * + * It is hand-written — the schemas differ per tool, so nothing can generate it — + * but it is not free-floating: `deep-agent/service.test.ts` asserts it equals + * `buildTools(...).map(t => t.name)`, and that test runs in CI. Adding a tool + * without adding it here fails the suite. + * + * ponytail: there used to be two more copies of this list, one in that test and + * one in `workbench/lib/prompts.ts`, neither derived from anything. The + * workbench's copy sat at 19 entries from the day it was written while the agent + * grew to 21 (`addTrims`/`addZooms`, commit 560d368e). Nothing caught it, + * because `npm run wb` is not part of CI — so the bench asserted a surface the + * product had not had for some time. + */ +export const OPENSCREEN_TOOL_NAMES = [ + "getCurrentDocument", + "getTranscript", + "getCursorTrack", + "addTrim", + "addTrims", + "setTrim", + "setClipRange", + "moveClip", + "replaceTimeline", + "addZoom", + "addZooms", + "setZoom", + "addSpeed", + "setSpeed", + "addAnnotation", + "setAnnotation", + "addCameraFullscreen", + "setCameraFullscreen", + "removeTrim", + "removeModifier", + "removeClip", +] as const; + +/** + * The tools `createDeepAgent` used to inject on top of ours, over an in-memory + * backend that was EMPTY and that the model was not told was empty — the + * mechanical cause of D1, where the agent ran `ls`/`glob` against that sandbox + * and reported in good faith that the project held no cursor telemetry. + * + * The surface is gone, so this is no longer "tools we also get": it is the list + * of names that must never appear again. A call to one of them now means the + * model is hallucinating a filesystem it was never offered, which is a rarer but + * still exact D1 tell — which is why the workbench scores it as well as pinning + * it here. + * + * `execute` is included even though it vanished at runtime: it is in the + * middleware's list too and only disappeared because the default backend is not + * a sandbox. A sandbox backend would have made it a 26th tool. The workbench's + * own copy of this list omitted it, so `isPhantomTool` could not flag the one + * name a sandbox backend would have brought back. + */ +export const PHANTOM_TOOL_NAMES = [ + "ls", + "read_file", + "write_file", + "edit_file", + "glob", + "grep", + "execute", + "write_todos", + "task", +] as const; + /** * The tools that change the document. A LIST, not an inference: it gates the * checkpoint the chat-service takes before a write and the mutating/non-mutating diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index 43e963c22..f567da0ee 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -27,7 +27,12 @@ import { ZOOM_DEPTH_LEGEND, ZOOM_DEPTH_SCALES, } from "../../../src/lib/ai-edition/timeline/zoom-scale"; -import { executeAgentTool, isMutatingTool } from "../agent-tools"; +import { + executeAgentTool, + isMutatingTool, + OPENSCREEN_TOOL_NAMES, + PHANTOM_TOOL_NAMES, +} from "../agent-tools"; import { anthropicCachingMiddleware, buildSystemPrompt, @@ -37,45 +42,13 @@ import { TOOL_DESCRIPTIONS, } from "./service"; -const OPENSCREEN_TOOLS = [ - "getCurrentDocument", - "getTranscript", - "getCursorTrack", - "addTrim", - "addTrims", - "setTrim", - "setClipRange", - "moveClip", - "replaceTimeline", - "addZoom", - "addZooms", - "setZoom", - "addSpeed", - "setSpeed", - "addAnnotation", - "setAnnotation", - "addCameraFullscreen", - "setCameraFullscreen", - "removeTrim", - "removeModifier", - "removeClip", -]; - -/** The tools `createDeepAgent` used to add. None of them may ever be built here - * again: `execute` is in the middleware's list too and only disappeared at - * runtime because the default backend is not a sandbox, so it is listed as - * well — a sandbox backend would have made it a 26th tool. */ -const PHANTOM_TOOLS = [ - "ls", - "read_file", - "write_file", - "edit_file", - "glob", - "grep", - "execute", - "write_todos", - "task", -]; +// Both rosters used to be re-typed here, and a third time in the workbench. This +// file is the one that runs in CI, so its copy stayed right and the bench's went +// stale at 19 tools — asserting a surface the product had outgrown. One list now, +// in `agent-tools.ts`; this suite is what pins it to what `buildTools` actually +// builds, and the bench reads the same array. +const OPENSCREEN_TOOLS: readonly string[] = OPENSCREEN_TOOL_NAMES; +const PHANTOM_TOOLS: readonly string[] = PHANTOM_TOOL_NAMES; /** Valid arguments for every tool, chosen so the executor's verdict is split * across the table: some succeed, some are refused for an unknown id, and @@ -191,7 +164,9 @@ function toolsFor(document: AxcutDocument) { } describe("the tool surface handed to the model", () => { - it("is exactly OpenScreen's 21 tools", () => { + // No count in the title: the number moved twice without either copy of the + // roster following, and a title is the one place a stale number cannot fail. + it("is exactly the tools OpenScreen declares, in that order", () => { const { tools } = toolsFor(fixtureDocument()); expect(tools.map((t) => t.name)).toEqual(OPENSCREEN_TOOLS); }); @@ -419,7 +394,7 @@ describe("the prompt when the user has turned project edits off", () => { }); describe("the tools when the user has turned project edits off", () => { - it("still builds all 21 — the model has to be able to NAME the edit", () => { + it("still builds every one — the model has to be able to NAME the edit", () => { const { sink } = recordingSink(); const tools: BuiltTool[] = buildTools({ current: fixtureDocument() }, sink, false); expect(tools.map((t) => t.name)).toEqual(OPENSCREEN_TOOLS); diff --git a/workbench/README.md b/workbench/README.md index d21987580..bffad2c58 100644 --- a/workbench/README.md +++ b/workbench/README.md @@ -77,14 +77,19 @@ Les rapports vont dans `workbench/reports/` (gitignoré), en JSON et en Markdown seule une différence énorme est lisible. Un check qui passe de 2/3 à 3/3 n'est pas une amélioration, c'est du bruit. 2. **L'empreinte du run** : `systemSha256` (le message système réellement envoyé), `toolsSha256`, - `toolNames[]` (**19 attendus**), l'id du modèle, le sha git. Deux rapports d'empreintes - différentes ne sont pas comparables. C'est arrivé le jour où `createAgent` a remplacé - `createDeepAgent` : le message système est passé de ~8 700 à 2 968 caractères et la surface - d'outils de 25 à 17. Elle a **rebougé deux fois depuis** — `moveClip` (18ᵉ outil), les - descriptions de `replaceTimeline`/zoom/caméra, deux règles de sélection d'outil et le bloc de - consentement ajouté au prompt quand `allowAgentEdits` est faux ; puis `getCursorTrack` - (19ᵉ), deux lignes de prompt sur la télémétrie et sur la cécité, et `cursorNote` plus - `assets[].hasCursorTelemetry` dans le snapshot. **Tous les rapports antérieurs sont des + `toolNames[]` (exactement `OPENSCREEN_TOOL_NAMES` — aucun compte n'est écrit ici : le roster + est épinglé en CI par `deep-agent/service.test.ts` contre ce que `buildTools` construit + vraiment, et un nombre recopié en prose est précisément ce qui a laissé ce banc en annoncer + 19 pendant que le produit en livrait 21), l'id du modèle, le sha git. Deux rapports + d'empreintes différentes ne sont pas comparables. C'est arrivé le jour où `createAgent` a + remplacé `createDeepAgent` : le message système est passé de ~8 700 à 2 968 caractères et la + surface d'outils de 25 à 17. Elle a **rebougé trois fois depuis** — `moveClip` (18ᵉ outil), + les descriptions de `replaceTimeline`/zoom/caméra, deux règles de sélection d'outil et le + bloc de consentement ajouté au prompt quand `allowAgentEdits` est faux ; puis + `getCursorTrack` (19ᵉ), deux lignes de prompt sur la télémétrie et sur la cécité, et + `cursorNote` plus `assets[].hasCursorTelemetry` dans le snapshot ; puis `addTrims` et + `addZooms` (560d368e), les variantes par lot qui émettent toute la passe en un aller-retour. + **Tous les rapports antérieurs sont des archives**, pas des références — `baseline-full-2026-07-31T17-33-19-798Z` compris, et les trois fichiers de `baselines/` avec. Il faut re-mesurer une ligne de base live avant de prétendre comparer quoi que ce soit. diff --git a/workbench/l0/scenario-pack.wb.ts b/workbench/l0/scenario-pack.wb.ts index 2cd60eaac..8b2d26e8d 100644 --- a/workbench/l0/scenario-pack.wb.ts +++ b/workbench/l0/scenario-pack.wb.ts @@ -625,9 +625,13 @@ describe("les demoScripts ne peuvent nommer qu'un outil qui existe", () => { // tours où rien n'avait jamais été lu. Un scénario dont le seul objet est // « a-t-il regardé ? » certifiait un modèle aveugle. // - // Les 8 fantômes restent autorisés : `cursor-question` et `wizard-enhance-bare` + // Les fantômes restent autorisés : `cursor-question` et `wizard-enhance-bare` // rejouent des tours live de 2026-07-31 où le modèle appelait `ls`/`glob`, et - // c'est précisément ce que ces demos doivent continuer à exercer. + // c'est précisément ce que ces demos doivent continuer à exercer. Leur nombre + // n'est pas écrit ici : `PHANTOM_TOOL_NAMES` en compte un de plus que la liste + // que ce banc tenait à la main (`execute`, que seul un backend sandbox ferait + // réapparaître), et une exemption dont la taille est recopiée en prose dérive + // exactement comme le roster a dérivé. const KNOWN = new Set([...OPENSCREEN_TOOLS, ...PHANTOM_TOOLS]); for (const scenario of allScenarios()) { diff --git a/workbench/l1/end-to-end.wb.ts b/workbench/l1/end-to-end.wb.ts index 1ccfe7779..04021881b 100644 --- a/workbench/l1/end-to-end.wb.ts +++ b/workbench/l1/end-to-end.wb.ts @@ -19,7 +19,10 @@ import { runRepetition, runScenarioReps } from "../lib/runner"; import { allScenarios, getScenario } from "../scenarios/registry"; describe("the context the model actually receives", () => { - it("is our 19 tools and nothing else", async () => { + // The title used to say "19" while the agent shipped 21, and a title cannot + // fail. The count lives in the roster now, and the roster is pinned in CI by + // `deep-agent/service.test.ts` against what `buildTools` actually builds. + it("is our tool surface and nothing else", async () => { const run = await runScenario({ label: "l1-surface", prompt: "hello", @@ -122,7 +125,7 @@ describe("no scoring without evidence", () => { }); describe("a name the model was never given", () => { - it("comes back as a tool result naming the 19, and the turn survives", async () => { + it("comes back as a tool result naming the real ones, and the turn survives", async () => { // The demoScripts of `cursor-question` and `wizard-enhance-bare` still // replay the live turns of 2026-07-31, when the model had `ls`/`glob`/ // `grep` and used them. Now that the surface is gone those calls are diff --git a/workbench/lib/prompts.ts b/workbench/lib/prompts.ts index 60038cacc..f4b6612bb 100644 --- a/workbench/lib/prompts.ts +++ b/workbench/lib/prompts.ts @@ -1,81 +1,65 @@ // ponytail: prompts and tool-surface constants, kept apart from the scenarios -// so a wording change is one edit and so the wizard prompt stays byte-identical -// to production. +// so a wording change is one edit and so the wizard prompt is stated once, with +// its provenance, rather than paraphrased per scenario. + +import { OPENSCREEN_TOOL_NAMES, PHANTOM_TOOL_NAMES } from "../../electron/ai-edition/agent-tools"; /** - * VERBATIM copy of `AI_ENHANCE_PROMPT` from - * `src/components/ai-edition/v4/V4Timeline.tsx:57-58` — the string the - * Auto-enhance button sends through the prompt bus into the same `runChat`. - * A workbench that paraphrases it measures a prompt the product never sends. + * The Auto-enhance prompt the wizard scenarios send. It is NOT the string the + * button sends any more: `src/components/ai-edition/v4/V4Timeline.tsx:77` + * narrowed to cuts only in `7e6439ad`, because the model places zooms from what + * the transcript SAYS rather than from where the pointer WAS — on a real 66s + * screencast, 7 of its 9 focus points missed the cursor in their own window. + * + * This copy stays WIDE on purpose. That same commit names `real-wizard-enhance` + * as the scenario that would show the model can read the track and justify + * re-widening the product, and no scenario can measure zoom placement under a + * prompt that never asks for a zoom: `dsl.zoom.placement` (weight 3) opens with + * `fail("aucun zoom émis")`. The `wizard-enhance` pair scores the D1 fabricated + * focus and D2 multiplier tells the same way — under the narrow prompt those + * checks go green without anything having been fixed. + * + * So the divergence is deliberate, but it IS a divergence: re-narrowing this + * string means reworking the zoom checks in all three scenarios and re-recording + * their baselines, which is why it is not a one-line edit. + * + * ponytail: this said "VERBATIM copy … the string the Auto-enhance button sends" + * from `7e6439ad` until now, pointing at `V4Timeline.tsx:57-58` where the + * constant no longer lives. Nothing said so — `npm run wb` is not part of CI. */ export const AI_ENHANCE_PROMPT = "Automatically enhance this recording: (1) add smart zoom-ins on the moments where the cursor dwells or interacts with the UI, each focused on the cursor's location; and (2) cut the dead time — long pauses, silences, and idle stretches where nothing happens — to keep the pacing tight and natural. Apply the edits directly to the timeline."; -/** The 19 tools OpenScreen builds in `deep-agent/service.ts` (`buildTools`). - * `moveClip` reordering a clip had NO tool while the system prompt promised one, - * which is what pushed the model onto `replaceTimeline` (D-DESTRUCT). - * `getCursorTrack` is the newest: the app records pointer telemetry and loads it - * in the compositor, but NOTHING carried a single sample to the model, so asked - * what cursor data the project held it had to answer from nothing (D-TELEM). - * - * ponytail: the name is `getCursorTrack`, not `getCursorTrack` — the tool - * returns the TRACK (positions over time) and no longer the stillness detector's - * digest, and this list said otherwise for a while. That is not a cosmetic - * drift: a scenario check written as `calls("getCursorTrack").length > 0` - * counts a call LangChain refused, so `cursor-question` and `cursor-blind` both - * scored 1.0 on turns where nothing was ever read. Every name here is frozen - * against the real surface by `l1/end-to-end.wb.ts`. */ -export const OPENSCREEN_TOOLS = [ - "getCurrentDocument", - "getTranscript", - "getCursorTrack", - "addTrim", - "setTrim", - "setClipRange", - "moveClip", - "replaceTimeline", - "addZoom", - "setZoom", - "addSpeed", - "setSpeed", - "addAnnotation", - "setAnnotation", - "addCameraFullscreen", - "setCameraFullscreen", - "removeTrim", - "removeModifier", - "removeClip", -] as const; - /** - * The 8 filesystem/todo/sub-agent tools the `deepagents` middlewares used to - * inject on top of ours. They operated on an in-memory `StateBackend` that is - * EMPTY, and the model was not told so — the mechanical cause of D1: asked - * about cursor telemetry, the model ran `ls`/`glob` against that sandbox and - * reported, in good faith, that the project contains no pointer-tracking data. + * The tools OpenScreen builds in `deep-agent/service.ts` (`buildTools`), and the + * filesystem/todo/sub-agent tools that must never appear beside them again. + * + * Both are RE-EXPORTS, not copies. The bench used to keep its own hand-written + * pair, and both went stale without anything noticing: the tool roster sat at 19 + * from the day it was written while the agent grew to 21 (`addTrims`/`addZooms`), + * and the phantom list was missing `execute`, so `isPhantomTool` could not flag + * the one middleware tool a sandbox backend would have re-introduced. Nothing + * caught either, because `npm run wb` is not part of CI — the bench was checking + * the product against a surface the product had outgrown. + * + * `agent-tools.ts` is the right home for them: it already owns + * `MUTATING_TOOL_NAMES` and runs on zod plus pure document helpers, so importing + * it costs L0 nothing (four L0 files already pull `executeAgentTool` from it). + * Importing `deep-agent/service.ts` instead would drag LangChain down here. * - * The surface is gone (`deep-agent/service.ts` now calls LangChain's - * `createAgent` with our own tools and our prompt alone), so this list changed - * meaning rather than becoming dead: it is now the list of names that must - * NEVER appear on the wire again. A call to one of them is no longer the model - * using a tool it was handed — it is the model hallucinating a filesystem it - * was never offered, which is a rarer but still exact D1 tell. `l1` freezes the - * surface directly; the scenarios keep scoring the calls. + * What keeps the roster honest is `deep-agent/service.test.ts`, which asserts it + * equals `buildTools(...).map(t => t.name)` — and that suite runs in CI. A tool + * added without updating the roster now fails the build rather than a bench + * nobody runs. */ -export const PHANTOM_TOOLS = [ - "write_todos", - "ls", - "read_file", - "write_file", - "edit_file", - "glob", - "grep", - "task", -] as const; +export const OPENSCREEN_TOOLS = OPENSCREEN_TOOL_NAMES; +export const PHANTOM_TOOLS = PHANTOM_TOOL_NAMES; -/** Exactly our 19, and nothing else. A change here means the agent's context - * changed shape — which is the one thing a report cannot be compared across. */ -export const EXPECTED_TOOL_COUNT = OPENSCREEN_TOOLS.length; +/** Our whole surface, and nothing else. A change here means the agent's context + * changed shape — which is the one thing a report cannot be compared across, so + * `fingerprintOf` records the wire's own `toolNames`/`toolsSha256` in every + * report rather than trusting this to have been noticed. */ +export const EXPECTED_TOOL_COUNT = OPENSCREEN_TOOL_NAMES.length; const PHANTOM_SET: ReadonlySet = new Set(PHANTOM_TOOLS); diff --git a/workbench/scenarios/wizard-enhance.scn.ts b/workbench/scenarios/wizard-enhance.scn.ts index 547e53b86..f9e19b43b 100644 --- a/workbench/scenarios/wizard-enhance.scn.ts +++ b/workbench/scenarios/wizard-enhance.scn.ts @@ -202,7 +202,7 @@ export default defineScenario({ "live dira si le modèle y cède encore. Reste listé pour cette raison, pas par " + "habitude.", }, - // beh.sandbox retiré : les 8 outils fantômes ne sont plus sur la surface. + // beh.sandbox retiré : les outils fantômes ne sont plus sur la surface. // INTERMITTENTES, mesuré en live sur deepseek-v4-flash : ces deux checks // passent certains runs entiers. Le modèle omet parfois tout multiplicateur // (silence = honnête, donc `beh.multiplier` passe) et centre parfois ses