From a47efc8550515daf1b5a34490ac82105c692b85a Mon Sep 17 00:00:00 2001 From: trick77 Date: Tue, 29 Sep 2026 23:16:34 +0200 Subject: [PATCH 1/2] Eval: ask the two follow-up rubrics as answer-only questions #261 added rubrics for "draw the flow as a diagram" and "In that flow, where is the shipping cost added?" to flow-rubrics.json only; the answer arm asks what flow-questions.json lists, so neither ever ran. Both are follow-ups whose words mean nothing outside their thread, so they are marked answers_only: the answer arm asks them after the turn they follow, the search arms and the expansion freeze leave them out. The well-formedness gate refuses an answer-only question whose rubric follows no turn. --- .../internal/retrieve/eval/edgereach_test.go | 2 +- .../retrieve/eval/flow-questions.json | 48 +++++++++++++++++++ .../internal/retrieve/eval/flowgather_test.go | 4 +- .../internal/retrieve/eval/flowloop_test.go | 32 ++++++++++++- 4 files changed, 82 insertions(+), 4 deletions(-) diff --git a/backend/internal/retrieve/eval/edgereach_test.go b/backend/internal/retrieve/eval/edgereach_test.go index 699cba52..4c429616 100644 --- a/backend/internal/retrieve/eval/edgereach_test.go +++ b/backend/internal/retrieve/eval/edgereach_test.go @@ -22,7 +22,7 @@ func TestFlowEdgeReach(t *testing.T) { db := evalDB(t, embedDim(t)) ctx := context.Background() - questions := loadFlowQuestions(t) + questions := searchedFlowQuestions(t) var totalPairs, totalReached, totalRepoReached, totalComposed int for _, q := range questions { diff --git a/backend/internal/retrieve/eval/flow-questions.json b/backend/internal/retrieve/eval/flow-questions.json index 4e798509..0ef4ec04 100644 --- a/backend/internal/retrieve/eval/flow-questions.json +++ b/backend/internal/retrieve/eval/flow-questions.json @@ -251,5 +251,53 @@ } ], "note": "Follow-up. \"the old one\" is the guest's session cart from the preceding turn. A turn that lost the subject answers about carts in general and never reaches the delete." + }, + { + "question": "draw the flow as a diagram", + "kind": "how", + "resolution": "composition", + "answers_only": true, + "candidates": [ + { + "repo": "front-end", + "paths": ["api/orders/index.js"], + "verified": "index.js:48 app.post(\"/orders\"), :122 forwards to endpoints.ordersUrl + '/orders'" + }, + { + "repo": "orders", + "paths": ["src/main/java/works/weave/socks/orders/controllers/OrdersController.java"], + "verified": "OrdersController.java:50 @RequestMapping POST /orders, :61-73 fetches address, customer, card and items, :78-96 payment, :99-102 shipping, :115 customerOrderRepository.save" + }, + { + "repo": "payment", + "paths": ["transport.go", "service.go"], + "verified": "transport.go:29 POST /paymentAuth, service.go:40-58 Authorise" + }, + { + "repo": "shipping", + "paths": ["src/main/java/works/weave/socks/shipping/controllers/ShippingController.java"], + "verified": "ShippingController.java:36 POST /shipping, :40 rabbitTemplate.convertAndSend(\"shipping-task\", shipment)" + }, + { + "repo": "queue-master", + "paths": ["src/main/java/works/weave/socks/queuemaster/ShippingTaskHandler.java"], + "verified": "ShippingTaskHandler.java:13 handleMessage(Shipment)" + } + ], + "note": "Follow-up to the order flow. A diagram of the previous answer is a rework, drawn from that answer's own sources, never a standing rule. Answer-only: it searches nothing of its own." + }, + { + "question": "In that flow, where is the shipping cost added?", + "kind": "how", + "resolution": "unique", + "answers_only": true, + "candidates": [ + { + "repo": "orders", + "paths": ["src/main/java/works/weave/socks/orders/controllers/OrdersController.java"], + "verified": "OrdersController.java:156 float shipping = 4.99F, :158 added to the item total in calculateTotal" + } + ], + "note": "Follow-up to the order flow; \"that flow\" is the preceding answer's. Answer-only: its words mean nothing outside the thread. Four runs on 2026-09-29, before and after #261, never reached :156, and three said no shipping cost exists." } ] diff --git a/backend/internal/retrieve/eval/flowgather_test.go b/backend/internal/retrieve/eval/flowgather_test.go index 6c744ff6..64bd6cba 100644 --- a/backend/internal/retrieve/eval/flowgather_test.go +++ b/backend/internal/retrieve/eval/flowgather_test.go @@ -55,7 +55,7 @@ func TestExpandFlowQuestions(t *testing.T) { } var out []expansion - for _, q := range loadFlowQuestions(t) { + for _, q := range searchedFlowQuestions(t) { if old, ok := previous[q.Text]; ok && expandOnlyMissing() { out = append(out, old) continue @@ -140,7 +140,7 @@ func TestFlowGathered(t *testing.T) { ctx := context.Background() retriever := evalRetriever(t, db) expansions := loadFlowExpansions(t) - questions := loadFlowQuestions(t) + questions := searchedFlowQuestions(t) deployed := gatherOpts(t) arms := []flowGatherArm{ diff --git a/backend/internal/retrieve/eval/flowloop_test.go b/backend/internal/retrieve/eval/flowloop_test.go index 8441897d..0452016a 100644 --- a/backend/internal/retrieve/eval/flowloop_test.go +++ b/backend/internal/retrieve/eval/flowloop_test.go @@ -80,6 +80,11 @@ type flowQuestion struct { Resolution Resolution `json:"resolution"` Candidates []flowCandidate `json:"candidates"` Note string `json:"note"` + // AnswersOnly is a follow-up whose words mean nothing outside its + // thread — "draw the flow as a diagram". The answer arm asks it after + // the turn it follows; the search arms, which ask every question cold, + // leave it out rather than measure a search nobody would run. + AnswersOnly bool `json:"answers_only"` } type flowCandidate struct { @@ -127,6 +132,19 @@ func loadFlowQuestions(t *testing.T) []flowQuestion { return qs } +// searchedFlowQuestions is loadFlowQuestions for the arms that ask every +// question cold, without a thread: the answer-only follow-ups left out. +func searchedFlowQuestions(t *testing.T) []flowQuestion { + t.Helper() + var out []flowQuestion + for _, q := range loadFlowQuestions(t) { + if !q.AnswersOnly { + out = append(out, q) + } + } + return out +} + // TestFlowQuestionsAreWellFormed is the cheap gate, and it runs without // BACKEND_EVAL: a candidate without a verified string is a claim nobody // checked, and the whole point of this corpus is that every part was read. @@ -151,6 +169,18 @@ func TestFlowQuestionsAreWellFormed(t *testing.T) { } } } + // An answer-only question is asked only as the second turn of a + // thread, so its rubric has to name the turn it follows; without one + // it would be asked cold after all, and graded as a first question. + rubrics := loadRubrics(t) + for _, q := range qs { + if !q.AnswersOnly { + continue + } + if r, ok := rubrics[q.Text]; !ok || r.Follows == "" { + t.Errorf("%q is answer-only, but its rubric follows no turn", q.Text) + } + } } // --- the tools ------------------------------------------------------------- @@ -513,7 +543,7 @@ func TestFlowLoopDiagnostic(t *testing.T) { seen: map[flowPart]bool{}, } - questions := loadFlowQuestions(t) + questions := searchedFlowQuestions(t) for _, deployment := range flowModels(t) { t.Run(deployment, func(t *testing.T) { wire := flowWire(t, deployment) From b1b2eb8b7ab595e981fb432d078552fb8af52a7e Mon Sep 17 00:00:00 2001 From: trick77 Date: Wed, 30 Sep 2026 06:53:22 +0200 Subject: [PATCH 2/2] Eval: cite target a diagram rework can reach Without a cite list the diagram follow-up was scored against all six candidate files of the order flow; a mermaid diagram cites nothing and a rework cites only what the previous answer did, so every run lost parts no correct answer could earn. --- backend/internal/retrieve/eval/flow-rubrics.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/internal/retrieve/eval/flow-rubrics.json b/backend/internal/retrieve/eval/flow-rubrics.json index ac7e0fb2..5728ab46 100644 --- a/backend/internal/retrieve/eval/flow-rubrics.json +++ b/backend/internal/retrieve/eval/flow-rubrics.json @@ -193,6 +193,12 @@ "must_not": [ "The diagram saves or ships the order before payment is authorised.", "The diagram adds a step the previous answer did not describe, such as an e-mail to the customer." + ], + "cite": [ + { + "repo": "orders", + "path": "src/main/java/works/weave/socks/orders/controllers/OrdersController.java" + } ] }, {