Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion backend/internal/retrieve/eval/edgereach_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ func TestFlowEdgeReach(t *testing.T) {
db := evalDB(t, embedDim(t))
ctx := context.Background()

questions := loadFlowQuestions(t)
questions := searchedFlowQuestions(t)
var totalPairs, totalReached, totalRepoReached, totalComposed int

for _, q := range questions {
Expand Down
48 changes: 48 additions & 0 deletions backend/internal/retrieve/eval/flow-questions.json
Original file line number Diff line number Diff line change
Expand Up @@ -251,5 +251,53 @@
}
],
"note": "Follow-up. \"the old one\" is the guest's session cart from the preceding turn. A turn that lost the subject answers about carts in general and never reaches the delete."
},
{
"question": "draw the flow as a diagram",
"kind": "how",
"resolution": "composition",
"answers_only": true,
"candidates": [
{
"repo": "front-end",
"paths": ["api/orders/index.js"],
"verified": "index.js:48 app.post(\"/orders\"), :122 forwards to endpoints.ordersUrl + '/orders'"
},
{
"repo": "orders",
"paths": ["src/main/java/works/weave/socks/orders/controllers/OrdersController.java"],
"verified": "OrdersController.java:50 @RequestMapping POST /orders, :61-73 fetches address, customer, card and items, :78-96 payment, :99-102 shipping, :115 customerOrderRepository.save"
},
{
"repo": "payment",
"paths": ["transport.go", "service.go"],
"verified": "transport.go:29 POST /paymentAuth, service.go:40-58 Authorise"
},
{
"repo": "shipping",
"paths": ["src/main/java/works/weave/socks/shipping/controllers/ShippingController.java"],
"verified": "ShippingController.java:36 POST /shipping, :40 rabbitTemplate.convertAndSend(\"shipping-task\", shipment)"
},
{
"repo": "queue-master",
"paths": ["src/main/java/works/weave/socks/queuemaster/ShippingTaskHandler.java"],
"verified": "ShippingTaskHandler.java:13 handleMessage(Shipment)"
}
],
"note": "Follow-up to the order flow. A diagram of the previous answer is a rework, drawn from that answer's own sources, never a standing rule. Answer-only: it searches nothing of its own."
},
{
"question": "In that flow, where is the shipping cost added?",
"kind": "how",
"resolution": "unique",
"answers_only": true,
"candidates": [
{
"repo": "orders",
"paths": ["src/main/java/works/weave/socks/orders/controllers/OrdersController.java"],
"verified": "OrdersController.java:156 float shipping = 4.99F, :158 added to the item total in calculateTotal"
}
],
"note": "Follow-up to the order flow; \"that flow\" is the preceding answer's. Answer-only: its words mean nothing outside the thread. Four runs on 2026-09-29, before and after #261, never reached :156, and three said no shipping cost exists."
}
]
6 changes: 6 additions & 0 deletions backend/internal/retrieve/eval/flow-rubrics.json
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,12 @@
"must_not": [
"The diagram saves or ships the order before payment is authorised.",
"The diagram adds a step the previous answer did not describe, such as an e-mail to the customer."
],
"cite": [
{
"repo": "orders",
"path": "src/main/java/works/weave/socks/orders/controllers/OrdersController.java"
}
]
},
{
Expand Down
4 changes: 2 additions & 2 deletions backend/internal/retrieve/eval/flowgather_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func TestExpandFlowQuestions(t *testing.T) {
}

var out []expansion
for _, q := range loadFlowQuestions(t) {
for _, q := range searchedFlowQuestions(t) {
if old, ok := previous[q.Text]; ok && expandOnlyMissing() {
out = append(out, old)
continue
Expand Down Expand Up @@ -140,7 +140,7 @@ func TestFlowGathered(t *testing.T) {
ctx := context.Background()
retriever := evalRetriever(t, db)
expansions := loadFlowExpansions(t)
questions := loadFlowQuestions(t)
questions := searchedFlowQuestions(t)
deployed := gatherOpts(t)

arms := []flowGatherArm{
Expand Down
32 changes: 31 additions & 1 deletion backend/internal/retrieve/eval/flowloop_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ type flowQuestion struct {
Resolution Resolution `json:"resolution"`
Candidates []flowCandidate `json:"candidates"`
Note string `json:"note"`
// AnswersOnly is a follow-up whose words mean nothing outside its
// thread — "draw the flow as a diagram". The answer arm asks it after
// the turn it follows; the search arms, which ask every question cold,
// leave it out rather than measure a search nobody would run.
AnswersOnly bool `json:"answers_only"`
}

type flowCandidate struct {
Expand Down Expand Up @@ -127,6 +132,19 @@ func loadFlowQuestions(t *testing.T) []flowQuestion {
return qs
}

// searchedFlowQuestions is loadFlowQuestions for the arms that ask every
// question cold, without a thread: the answer-only follow-ups left out.
func searchedFlowQuestions(t *testing.T) []flowQuestion {
t.Helper()
var out []flowQuestion
for _, q := range loadFlowQuestions(t) {
if !q.AnswersOnly {
out = append(out, q)
}
}
return out
}

// TestFlowQuestionsAreWellFormed is the cheap gate, and it runs without
// BACKEND_EVAL: a candidate without a verified string is a claim nobody
// checked, and the whole point of this corpus is that every part was read.
Expand All @@ -151,6 +169,18 @@ func TestFlowQuestionsAreWellFormed(t *testing.T) {
}
}
}
// An answer-only question is asked only as the second turn of a
// thread, so its rubric has to name the turn it follows; without one
// it would be asked cold after all, and graded as a first question.
rubrics := loadRubrics(t)
for _, q := range qs {
if !q.AnswersOnly {
continue
}
if r, ok := rubrics[q.Text]; !ok || r.Follows == "" {
t.Errorf("%q is answer-only, but its rubric follows no turn", q.Text)
}
}
}

// --- the tools -------------------------------------------------------------
Expand Down Expand Up @@ -513,7 +543,7 @@ func TestFlowLoopDiagnostic(t *testing.T) {
seen: map[flowPart]bool{},
}

questions := loadFlowQuestions(t)
questions := searchedFlowQuestions(t)
for _, deployment := range flowModels(t) {
t.Run(deployment, func(t *testing.T) {
wire := flowWire(t, deployment)
Expand Down
Loading