From 2b4518b9c3e568529932e1502efcb7d39bd60fb8 Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Mon, 28 Sep 2026 00:27:14 -0600 Subject: [PATCH 1/5] =?UTF-8?q?feat(controlplane):=20quality=20report=20ov?= =?UTF-8?q?er=20the=20ledger=20=E2=80=94=20GET=20/api/report=20(U2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Store.Report aggregates one [since, until) window of terminal jobs: jobs by state, clean vs person-fixed accepts from the publish ledger, publish outcomes and held rate, CI repair entry/success/rounds/stop codes, flaky re-runs, KTD1's two revisit rates, and agent_end spend per outcome bucket with spend per clean accept. It reads publish_history (U8), ci_reruns (U4) and unmetered_sends (U1) when present and tolerates their absence. Read-only, no migration. Co-Authored-By: Claude Opus 5.5 --- internal/controlplane/http.go | 5 +- internal/controlplane/report.go | 694 +++++++++++++++++++++++++++ internal/controlplane/report_test.go | 451 +++++++++++++++++ 3 files changed, 1148 insertions(+), 2 deletions(-) create mode 100644 internal/controlplane/report.go create mode 100644 internal/controlplane/report_test.go diff --git a/internal/controlplane/http.go b/internal/controlplane/http.go index 41491c5..014fd3d 100644 --- a/internal/controlplane/http.go +++ b/internal/controlplane/http.go @@ -64,11 +64,12 @@ func NewHandler(store *Store, uiToken string, logger *slog.Logger) http.Handler mux.HandleFunc("POST /api/worktrees/{attempt_id}/release", api.releaseWorktree) // Publish routes register themselves from publish_ledger.go (U7), the // admission-trigger routes from schedule.go (U6), and trace ingestion plus - // the read-only surface from ingest.go (U8), where their handlers live - // beside the store methods they call. + // the read-only surface from ingest.go (U8), and the quality report from + // report.go, where their handlers live beside the store methods they call. api.registerPublishRoutes(mux) api.registerTriggerRoutes(mux) api.registerIngestRoutes(mux) + api.registerReportRoutes(mux) return mux } diff --git a/internal/controlplane/report.go b/internal/controlplane/report.go new file mode 100644 index 0000000..230ffbd --- /dev/null +++ b/internal/controlplane/report.go @@ -0,0 +1,694 @@ +// report.go — the factory quality report (plan 2026-09-28-001, U2, KTD3, +// KTD4). One read-only aggregate over a time window, computed from the +// ledger the control plane already keeps: job states, publish records, CI +// repair rounds, attempt results, and agent_end spend events. Nothing here +// writes, and nothing needs a migration. +// +// What the report can and cannot see (R4): +// - A clean accept is judged from the publish ledger, never from the +// worker's result: green CI recorded on a head jig pushed (the proof +// ref or a repair round's head_after), or proof under a definition that +// does not wait for CI. Green CI on any other head is a person's fix. +// - Accepts under a definition that does not wait for CI are clean by +// definition and counted separately, because jig never saw CI for them. +// - Nothing here observes the base branch after merge, so the report makes +// no claim about post-merge CI. +// +// Publish summaries are read from the latest attempt's result: `publish` +// plus the `publish_history` a publish-only retry keeps (R15), oldest first. +// Anything it needs from a result and cannot read is counted, never guessed +// (`unreadable_results`). +package controlplane + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/StructuPath/jig/internal/protocol" +) + +// defaultReportWindow is the window a report covers when the caller names no +// `since`. +const defaultReportWindow = 7 * 24 * time.Hour + +// Report is the quality report over one window. Counts are always numbers; +// a rate or a per-accept figure is null when its denominator is zero, because +// a zero there would read as a measurement. +type Report struct { + Since time.Time `json:"since"` + Until time.Time `json:"until"` + ObservedAt time.Time `json:"observed_at"` + + Jobs ReportJobs `json:"jobs"` + Accepts ReportAccepts `json:"accepts"` + Publish ReportPublish `json:"publish"` + CI ReportCI `json:"ci"` + Spend ReportSpend `json:"spend"` + + // UnreadableResults counts publish-eligible jobs whose latest attempt + // result could not be read in full: not JSON, a missing or unknown + // `publish`, or a malformed history, repair, or re-run entry. + UnreadableResults int `json:"unreadable_results"` +} + +// ReportJobs counts the jobs in the window by terminal state. A job is in +// the window when it is terminal and its updated_at falls in [since, until). +type ReportJobs struct { + Total int `json:"total"` + Accepted int `json:"accepted"` + AcceptedUnpublished int `json:"accepted_unpublished"` + Failed int `json:"failed"` + Cancelled int `json:"cancelled"` +} + +// ReportAccepts splits accepted jobs by what the ledger proves (KTD4, R4). +// Clean = CIGreen + NoCIWait. PersonFixed and Unverified are never clean. +type ReportAccepts struct { + Clean int `json:"clean"` + // CIGreen: green CI recorded on a head jig pushed. + CIGreen int `json:"clean_ci_green"` + // NoCIWait: proof under a definition that does not wait for CI; jig + // never saw CI for these. + NoCIWait int `json:"clean_no_ci_wait"` + // PersonFixed: green CI recorded on a head jig did not push. + PersonFixed int `json:"person_fixed"` + // Unverified: accepted, but the ledger cannot say which (the frozen + // definition no longer parses, or a CI-waiting accept has no ci record). + Unverified int `json:"unverified"` +} + +// ReportPublish classifies the latest publish outcome of every accepted or +// accepted_unpublished job. Accepted jobs are published by the ledger's own +// rule (R12); accepted_unpublished jobs are classified from their result. +type ReportPublish struct { + Eligible int `json:"eligible"` + Published int `json:"published"` + Held int `json:"held"` + NotAttempted int `json:"not_attempted"` + Failed int `json:"failed"` + Unreadable int `json:"unreadable"` + // FailedCodes counts failed publishes by their final code. + FailedCodes map[string]int `json:"failed_codes"` + // HeldRate is Held over Eligible. + HeldRate *float64 `json:"held_rate"` +} + +// ReportCI covers attempts that waited for CI: the latest attempt of a +// publish-eligible job whose definition waits for CI and whose publish +// reached the CI wait (a proof record exists). Every stop code, round, and +// re-run counts once per attempt, across `publish` and `publish_history`. +type ReportCI struct { + Waited int `json:"waited"` + // FirstPassGreen: clean CI-green accepts with no repair round and no + // re-run. A pass after a re-run is flaky, never counted here (R6). + FirstPassGreen int `json:"first_pass_green"` + Repair ReportCIRepair `json:"repair"` + Reruns ReportCIReruns `json:"reruns"` + Revisit ReportCIRevisit `json:"revisit"` +} + +// ReportCIRepair is CI repair (publish.ci.on_fail) over waited attempts. +type ReportCIRepair struct { + // Entered: attempts with a pushed round on the ledger or any round in a + // publish summary. + Entered int `json:"entered"` + // EntryRate is Entered over Waited. + EntryRate *float64 `json:"entry_rate"` + // Succeeded: entered attempts whose repair ended with CI green — the + // summary that carries the rounds is published. A later publish-only + // retry that goes green does not make a failed repair a success. + Succeeded int `json:"succeeded"` + // SuccessRate is Succeeded over Entered. + SuccessRate *float64 `json:"success_rate"` + // Rounds counts pushed rounds on the ledger (publish_ci_repairs). + Rounds int `json:"rounds"` + // StopCodes counts the code each unsuccessful repair stopped on. + StopCodes map[string]int `json:"stop_codes"` +} + +// ReportCIReruns reads the `ci_reruns` entries of publish summaries. +type ReportCIReruns struct { + // Attempts: waited attempts with at least one re-run. + Attempts int `json:"attempts"` + // Reruns: re-run entries, deduplicated per attempt by their number. + Reruns int `json:"reruns"` + // FlakyPasses: attempts with a re-run whose outcome is `passed`. + FlakyPasses int `json:"flaky_passes"` +} + +// ReportCIRevisit carries KTD1's two revisit signals, both over Waited. +type ReportCIRevisit struct { + // CITimeouts: attempts with any publish summary ending ci_timeout. + CITimeouts int `json:"ci_timeouts"` + CITimeoutRate *float64 `json:"ci_timeout_rate"` + // RetryStillRed: attempts whose publish summaries, oldest first, show a + // red summary directly followed by a publish-only retry that was still + // red. Red means CI failed: code ci_failed or a failed check recorded. + RetryStillRed int `json:"retry_still_red"` + RetryStillRedRate *float64 `json:"retry_still_red_rate"` +} + +// ReportSpend sums agent_end cost over every attempt of every job in the +// window. Every job lands in exactly one bucket, so the buckets add up to +// the totals. +type ReportSpend struct { + TotalUSD float64 `json:"total_usd"` + UnmeteredSends int `json:"unmetered_sends"` + // PerCleanAcceptUSD is TotalUSD over Accepts.Clean: what one clean + // accept costs once failures, holds, and person-fixed work are paid for. + // Those never enter the denominator. + PerCleanAcceptUSD *float64 `json:"per_clean_accept_usd"` + ByOutcome map[string]ReportSpendBucket `json:"by_outcome"` +} + +// ReportSpendBucket is one outcome's share of the spend. +type ReportSpendBucket struct { + Jobs int `json:"jobs"` + CostUSD float64 `json:"cost_usd"` + UnmeteredSends int `json:"unmetered_sends"` +} + +// Spend buckets. Every job in the window is in exactly one. +const ( + spendClean = "clean_accept" + spendPersonFixed = "person_fixed" + spendUnverified = "accepted_unverified" + spendHeld = "held" + spendUnpublished = "unpublished" + spendFailed = "failed" + spendCancelled = "cancelled" +) + +var spendBuckets = []string{spendClean, spendPersonFixed, spendUnverified, + spendHeld, spendUnpublished, spendFailed, spendCancelled} + +// reportJob is one job in the window with everything the report reads. +type reportJob struct { + id string + state string + runID string + snapshot string + attemptID string + result string + proofRef string + ciRef string + hasCI bool + // pushedHeads are the heads jig pushed: the proof ref and every round's + // head_after. + pushedHeads map[string]bool + rounds int + cost float64 + unmetered int +} + +// reportSummary is the part of a publish summary the report reads. Field +// names are the worker's (PublishSummary, CIRepairSummary) plus U4's pinned +// `ci_reruns`. +type reportSummary struct { + State string `json:"state"` + Code string `json:"code"` + CIFailures []struct { + Verdict string `json:"verdict"` + } `json:"ci_failures"` + CIRepairs []struct { + Round int `json:"round"` + Outcome string `json:"outcome"` + } `json:"ci_repairs"` + CIReruns []struct { + Attempt int `json:"attempt"` + Outcome string `json:"outcome"` + } `json:"ci_reruns"` +} + +// red reports whether this summary ended with CI red. +func (s reportSummary) red() bool { + if s.State == "published" { + return false + } + if s.Code == "ci_failed" { + return true + } + for _, failure := range s.CIFailures { + if failure.Verdict == "fail" { + return true + } + } + return false +} + +// reportResult is one latest-attempt result as the report reads it. +type reportResult struct { + // publish is the current value: a summary, or one of the engine's + // markers. + marker string + current *reportSummary + // summaries are publish_history (oldest first) then the current summary. + summaries []reportSummary + unreadable bool +} + +func readReportResult(raw string) reportResult { + var read reportResult + var document struct { + Publish json.RawMessage `json:"publish"` + PublishHistory []json.RawMessage `json:"publish_history"` + } + if err := json.Unmarshal([]byte(raw), &document); err != nil { + read.unreadable = true + return read + } + for _, entry := range document.PublishHistory { + summary, ok := readSummary(entry) + if !ok { + read.unreadable = true + continue + } + read.summaries = append(read.summaries, summary) + } + var marker string + switch { + case len(document.Publish) == 0: + read.unreadable = true + case json.Unmarshal(document.Publish, &marker) == nil: + if marker != "not_attempted" && marker != "held" { + read.unreadable = true + break + } + read.marker = marker + default: + summary, ok := readSummary(document.Publish) + if !ok { + read.unreadable = true + break + } + read.current = &summary + read.summaries = append(read.summaries, summary) + } + return read +} + +func readSummary(raw json.RawMessage) (reportSummary, bool) { + var summary reportSummary + if err := json.Unmarshal(raw, &summary); err != nil { + return summary, false + } + switch summary.State { + case "published", "held", "failed": + return summary, true + } + return summary, false +} + +// Report computes the quality report over [since, until). +func (s *Store) Report(ctx context.Context, since, until time.Time) (Report, error) { + report := Report{ + Since: since.UTC(), Until: until.UTC(), ObservedAt: s.now().UTC(), + Publish: ReportPublish{FailedCodes: map[string]int{}}, + CI: ReportCI{Repair: ReportCIRepair{StopCodes: map[string]int{}}}, + Spend: ReportSpend{ByOutcome: map[string]ReportSpendBucket{}}, + } + for _, bucket := range spendBuckets { + report.Spend.ByOutcome[bucket] = ReportSpendBucket{} + } + if !since.Before(until) { + return report, invalid("invalid_report_window", "since must be before until") + } + jobs, err := s.reportJobs(ctx, since.UnixMilli(), until.UnixMilli()) + if err != nil { + return report, err + } + specs := map[string]*protocol.DefinitionSpec{} + for _, job := range jobs { + spec, parsed := specs[job.runID] + if !parsed { + spec, _ = protocol.ParseDefinition([]byte(job.snapshot)) + specs[job.runID] = spec + } + report.add(job, spec) + } + report.finish() + return report, nil +} + +// add folds one job into the report. +func (r *Report) add(job *reportJob, spec *protocol.DefinitionSpec) { + r.Jobs.Total++ + r.Spend.TotalUSD += job.cost + r.Spend.UnmeteredSends += job.unmetered + bucket := "" + switch job.state { + case protocol.JobFailed: + r.Jobs.Failed++ + bucket = spendFailed + case protocol.JobCancelled: + r.Jobs.Cancelled++ + bucket = spendCancelled + case protocol.JobAccepted, protocol.JobAcceptedUnpublished: + bucket = r.addPublished(job, spec) + } + spent := r.Spend.ByOutcome[bucket] + spent.Jobs++ + spent.CostUSD += job.cost + spent.UnmeteredSends += job.unmetered + r.Spend.ByOutcome[bucket] = spent +} + +// addPublished classifies one accepted or accepted_unpublished job and +// returns its spend bucket. +func (r *Report) addPublished(job *reportJob, spec *protocol.DefinitionSpec) string { + r.Publish.Eligible++ + result := readReportResult(job.result) + if result.unreadable { + r.UnreadableResults++ + } + waits := spec != nil && spec.WaitsForCI() + bucket := "" + if job.state == protocol.JobAccepted { + r.Jobs.Accepted++ + r.Publish.Published++ + switch { + case spec == nil || (waits && !job.hasCI): + r.Accepts.Unverified++ + bucket = spendUnverified + case !waits: + r.Accepts.Clean++ + r.Accepts.NoCIWait++ + bucket = spendClean + case job.pushedHeads[job.ciRef]: + r.Accepts.Clean++ + r.Accepts.CIGreen++ + bucket = spendClean + default: + r.Accepts.PersonFixed++ + bucket = spendPersonFixed + } + } else { + r.Jobs.AcceptedUnpublished++ + bucket = spendUnpublished + switch { + case result.marker == "held" || (result.current != nil && result.current.State == "held"): + r.Publish.Held++ + bucket = spendHeld + case result.marker == "not_attempted": + r.Publish.NotAttempted++ + case result.current != nil && result.current.State == "failed": + r.Publish.Failed++ + code := result.current.Code + if code == "" { + code = "unknown" + } + r.Publish.FailedCodes[code]++ + default: + r.Publish.Unreadable++ + } + } + if waits && job.proofRef != "" { + r.addWaited(job, result, bucket == spendClean) + } + return bucket +} + +// addWaited folds one attempt that waited for CI into the CI section. +func (r *Report) addWaited(job *reportJob, result reportResult, cleanGreen bool) { + r.CI.Waited++ + + // Repair: the last summary that carries rounds is the repair's own + // outcome; a later retry's summary carries none. + var repair *reportSummary + for index := range result.summaries { + if len(result.summaries[index].CIRepairs) > 0 { + repair = &result.summaries[index] + } + } + entered := job.rounds > 0 || repair != nil + if entered { + r.CI.Repair.Entered++ + r.CI.Repair.Rounds += job.rounds + switch { + case repair != nil && repair.State == "published": + r.CI.Repair.Succeeded++ + case repair != nil && repair.Code != "": + r.CI.Repair.StopCodes[repair.Code]++ + default: + r.CI.Repair.StopCodes["unknown"]++ + } + } + + reruns := map[int]string{} + timedOut := false + stillRed := false + for index, summary := range result.summaries { + for _, rerun := range summary.CIReruns { + reruns[rerun.Attempt] = rerun.Outcome + } + if summary.Code == "ci_timeout" { + timedOut = true + } + if index > 0 && summary.red() && result.summaries[index-1].red() { + stillRed = true + } + } + flaky := false + for _, outcome := range reruns { + if outcome == "passed" { + flaky = true + } + } + if len(reruns) > 0 { + r.CI.Reruns.Attempts++ + r.CI.Reruns.Reruns += len(reruns) + } + if flaky { + r.CI.Reruns.FlakyPasses++ + } + if timedOut { + r.CI.Revisit.CITimeouts++ + } + if stillRed { + r.CI.Revisit.RetryStillRed++ + } + if cleanGreen && job.state == protocol.JobAccepted && !entered && len(reruns) == 0 { + r.CI.FirstPassGreen++ + } +} + +func (r *Report) finish() { + r.Publish.HeldRate = ratio(r.Publish.Held, r.Publish.Eligible) + r.CI.Repair.EntryRate = ratio(r.CI.Repair.Entered, r.CI.Waited) + r.CI.Repair.SuccessRate = ratio(r.CI.Repair.Succeeded, r.CI.Repair.Entered) + r.CI.Revisit.CITimeoutRate = ratio(r.CI.Revisit.CITimeouts, r.CI.Waited) + r.CI.Revisit.RetryStillRedRate = ratio(r.CI.Revisit.RetryStillRed, r.CI.Waited) + if r.Accepts.Clean > 0 { + per := r.Spend.TotalUSD / float64(r.Accepts.Clean) + r.Spend.PerCleanAcceptUSD = &per + } +} + +func ratio(part, whole int) *float64 { + if whole == 0 { + return nil + } + value := float64(part) / float64(whole) + return &value +} + +// reportWindowJobs selects the terminal jobs in the window; every report +// query joins it so all of them read the same set. +const reportWindowJobs = ` + SELECT id FROM jobs + WHERE state IN ('accepted', 'accepted_unpublished', 'failed', 'cancelled') + AND updated_at >= ? AND updated_at < ?` + +// reportLatestAttempts is each window job's latest attempt. +const reportLatestAttempts = ` + SELECT a.id FROM attempts a + WHERE a.job_id IN (` + reportWindowJobs + `) + AND a.attempt_number = (SELECT MAX(attempt_number) FROM attempts WHERE job_id = a.job_id)` + +// reportJobs reads every job in the window with its latest attempt, its +// publish ledger, its rounds, and its spend over all attempts. +func (s *Store) reportJobs(ctx context.Context, since, until int64) ([]*reportJob, error) { + rows, err := s.db.QueryContext(ctx, ` + SELECT j.id, j.state, r.id, r.snapshot, COALESCE(a.id, ''), COALESCE(a.result, '') + FROM jobs j + JOIN runs r ON r.id = j.run_id + LEFT JOIN attempts a ON a.job_id = j.id + AND a.attempt_number = (SELECT MAX(attempt_number) FROM attempts WHERE job_id = j.id) + WHERE j.id IN (`+reportWindowJobs+`) + ORDER BY j.id + `, since, until) + if err != nil { + return nil, unavailable(err) + } + jobs := []*reportJob{} + byID := map[string]*reportJob{} + byAttempt := map[string]*reportJob{} + for rows.Next() { + job := &reportJob{pushedHeads: map[string]bool{}} + if err := rows.Scan(&job.id, &job.state, &job.runID, &job.snapshot, + &job.attemptID, &job.result); err != nil { + rows.Close() + return nil, unavailable(err) + } + jobs = append(jobs, job) + byID[job.id] = job + if job.attemptID != "" { + byAttempt[job.attemptID] = job + } + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, unavailable(err) + } + rows.Close() + + if err := s.reportLedger(ctx, since, until, byAttempt); err != nil { + return nil, err + } + if err := s.reportSpend(ctx, since, until, byID); err != nil { + return nil, err + } + return jobs, nil +} + +// reportLedger reads the proof and ci records and the pushed rounds of each +// latest attempt. +func (s *Store) reportLedger(ctx context.Context, since, until int64, byAttempt map[string]*reportJob) error { + rows, err := s.db.QueryContext(ctx, ` + SELECT attempt_id, step, remote_ref FROM publish_records + WHERE step IN (?, ?) AND attempt_id IN (`+reportLatestAttempts+`) + `, protocol.PublishStepProof, protocol.PublishStepCI, since, until) + if err != nil { + return unavailable(err) + } + for rows.Next() { + var attemptID, step, ref string + if err := rows.Scan(&attemptID, &step, &ref); err != nil { + rows.Close() + return unavailable(err) + } + job := byAttempt[attemptID] + if job == nil { + continue + } + if step == protocol.PublishStepProof { + job.proofRef = ref + job.pushedHeads[ref] = true + } else { + job.ciRef = ref + job.hasCI = true + } + } + if err := rows.Err(); err != nil { + rows.Close() + return unavailable(err) + } + rows.Close() + + rows, err = s.db.QueryContext(ctx, ` + SELECT attempt_id, head_after FROM publish_ci_repairs + WHERE attempt_id IN (`+reportLatestAttempts+`) + `, since, until) + if err != nil { + return unavailable(err) + } + defer rows.Close() + for rows.Next() { + var attemptID, head string + if err := rows.Scan(&attemptID, &head); err != nil { + return unavailable(err) + } + if job := byAttempt[attemptID]; job != nil { + job.rounds++ + job.pushedHeads[head] = true + } + } + if err := rows.Err(); err != nil { + return unavailable(err) + } + return nil +} + +// reportSpend sums agent_end cost and unmetered sends over every attempt of +// each window job. The stored payload is the whole marshalled event, so the +// agent's numbers sit under $.payload. A missing field (an event from before +// the field existed) adds nothing. +func (s *Store) reportSpend(ctx context.Context, since, until int64, byID map[string]*reportJob) error { + rows, err := s.db.QueryContext(ctx, ` + SELECT a.job_id, + COALESCE(SUM(json_extract(CAST(e.payload AS TEXT), '$.payload.cost')), 0), + COALESCE(SUM(json_extract(CAST(e.payload AS TEXT), '$.payload.unmetered_sends')), 0) + FROM events e JOIN attempts a ON a.id = e.attempt_id + WHERE e.type = ? AND json_valid(CAST(e.payload AS TEXT)) + AND a.job_id IN (`+reportWindowJobs+`) + GROUP BY a.job_id + `, protocol.EventAgentEnd, since, until) + if err != nil { + return unavailable(err) + } + defer rows.Close() + for rows.Next() { + var jobID string + var cost float64 + var unmetered int + if err := rows.Scan(&jobID, &cost, &unmetered); err != nil { + return unavailable(err) + } + if job := byID[jobID]; job != nil { + job.cost = cost + job.unmetered = unmetered + } + } + if err := rows.Err(); err != nil { + return unavailable(err) + } + return nil +} + +// ---- HTTP surface ---------------------------------------------------------- + +// registerReportRoutes attaches the report. It is a GET with no side +// effects, so it takes no mutation gate (the read-surface rule in ingest.go). +func (a *API) registerReportRoutes(mux *http.ServeMux) { + mux.HandleFunc("GET /api/report", a.report) +} + +// report answers GET /api/report?since=&until=. until +// defaults to now and since to seven days before until. +func (a *API) report(w http.ResponseWriter, r *http.Request) { + until, err := queryTime(r, "until", a.store.now()) + if err != nil { + writeError(w, err) + return + } + since, err := queryTime(r, "since", until.Add(-defaultReportWindow)) + if err != nil { + writeError(w, err) + return + } + report, err := a.store.Report(r.Context(), since, until) + if err != nil { + writeError(w, err) + return + } + writeJSON(w, http.StatusOK, report) +} + +// queryTime reads one RFC3339 query parameter. A malformed value is a 400 +// naming the parameter, never a silent default (the queryInt rule). +func queryTime(r *http.Request, name string, fallback time.Time) (time.Time, error) { + raw := strings.TrimSpace(r.URL.Query().Get(name)) + if raw == "" { + return fallback, nil + } + value, err := time.Parse(time.RFC3339, raw) + if err != nil { + return time.Time{}, invalid("invalid_query_parameter", + fmt.Sprintf("%s must be an RFC3339 timestamp", name)) + } + return value, nil +} diff --git a/internal/controlplane/report_test.go b/internal/controlplane/report_test.go new file mode 100644 index 0000000..ebb39c2 --- /dev/null +++ b/internal/controlplane/report_test.go @@ -0,0 +1,451 @@ +// report_test.go — the quality report (plan 2026-09-28-001, U2). The ledger +// is seeded row by row, including result fields other units add in parallel +// (`publish_history`, `ci_reruns`, `unmetered_sends`), so every count below +// is exact and known. +package controlplane + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "testing" + "time" + + "github.com/StructuPath/jig/internal/protocol" +) + +// reportFixture seeds report rows straight into the ledger. +type reportFixture struct { + t *testing.T + store *Store + seq map[string]int64 +} + +const ( + reportRunCI = "run-ci" + reportRunNoCI = "run-noci" +) + +func newReportFixture(t *testing.T) (*reportFixture, *testClock) { + t.Helper() + store, clock := newTestStore(t) + seedRun(t, store, reportRunCI) + useSnapshot(t, store, reportRunCI, repairFixtureSnapshot) + seedRun(t, store, reportRunNoCI) + return &reportFixture{t: t, store: store, seq: map[string]int64{}}, clock +} + +func (f *reportFixture) exec(query string, args ...any) { + f.t.Helper() + if _, err := f.store.db.Exec(query, args...); err != nil { + f.t.Fatalf("seed %q: %v", query, err) + } +} + +// job inserts a job in state, last updated at updated. +func (f *reportFixture) job(id, runID, state string, updated time.Time) { + f.t.Helper() + f.exec(`INSERT INTO jobs(id, run_id, repository, base_sha, state, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, id, runID, "github.com/example/"+id, shaA, state, + updated.UnixMilli(), updated.UnixMilli()) +} + +// attempt inserts attempt number n of jobID and returns its id. +func (f *reportFixture) attempt(jobID string, n int, state, result string) string { + f.t.Helper() + id := jobID + "-attempt-" + string(rune('0'+n)) + var stored any + if result != "" { + stored = result + } + f.exec(`INSERT INTO attempts(id, job_id, attempt_number, state, result, created_at) + VALUES (?, ?, ?, ?, ?, ?)`, id, jobID, n, state, stored, f.store.now().UnixMilli()) + return id +} + +// event stores one event the way ingestion does: the whole marshalled event. +func (f *reportFixture) event(attemptID, eventType string, payload map[string]any) { + f.t.Helper() + body, err := json.Marshal(payload) + if err != nil { + f.t.Fatal(err) + } + stored, err := json.Marshal(protocol.Event{ + Seq: f.seq[attemptID], Type: eventType, Phase: "build", Name: "builder", Payload: body, + }) + if err != nil { + f.t.Fatal(err) + } + f.exec(`INSERT INTO events(attempt_id, seq, type, phase, payload, payload_bytes, server_time) + VALUES (?, ?, ?, 'build', ?, ?, ?)`, attemptID, f.seq[attemptID], eventType, stored, + len(stored), f.store.now().UnixMilli()) + f.seq[attemptID]++ +} + +// spend records one agent_end; unmetered < 0 leaves the field out, as every +// event from before the field existed does. +func (f *reportFixture) spend(attemptID string, cost float64, unmetered int) { + f.t.Helper() + payload := map[string]any{"cost": cost, "tokens": 100, "sends": 1} + if unmetered >= 0 { + payload["unmetered_sends"] = unmetered + } + f.event(attemptID, protocol.EventAgentEnd, payload) +} + +func (f *reportFixture) step(attemptID, step, ref string) { + f.t.Helper() + f.exec(`INSERT INTO publish_records(attempt_id, step, branch, remote_ref, pr_url, completed_at) + VALUES (?, ?, 'jig/branch', ?, '', ?)`, attemptID, step, ref, f.store.now().UnixMilli()) +} + +// rounds records pushed repair rounds chaining through heads. +func (f *reportFixture) rounds(attemptID string, heads ...string) { + f.t.Helper() + for index := 1; index < len(heads); index++ { + f.exec(`INSERT INTO publish_ci_repairs(attempt_id, round, branch, head_before, head_after, failed_checks, completed_at) + VALUES (?, ?, 'jig/branch', ?, ?, '["test"]', ?)`, attemptID, index, heads[index-1], heads[index], + f.store.now().UnixMilli()) + } +} + +// result builds an attempt result from a publish value and optional history. +func result(t *testing.T, publish any, history ...any) string { + t.Helper() + document := map[string]any{"changed_paths": []string{"a.go"}, "publish": publish} + if history != nil { + document["publish_history"] = history + } + body, err := json.Marshal(document) + if err != nil { + t.Fatal(err) + } + return string(body) +} + +func summary(state, code string, extra map[string]any) map[string]any { + value := map[string]any{"state": state, "branch": "jig/branch"} + if code != "" { + value["code"] = code + } + for key, item := range extra { + value[key] = item + } + return value +} + +var ( + redChecks = []map[string]any{{"name": "test", "verdict": "fail"}} + pendingChecks = []map[string]any{{"name": "test", "verdict": "pending"}} + twoRounds = []map[string]any{ + {"round": 1, "head_before": shaA, "head_after": shaB, "outcome": "pushed"}, + {"round": 2, "head_before": shaB, "head_after": shaC, "outcome": "pushed"}, + } +) + +func floatPointer(value float64) *float64 { return &value } + +func mustReport(t *testing.T, store *Store, since, until time.Time) Report { + t.Helper() + report, err := store.Report(context.Background(), since, until) + if err != nil { + t.Fatalf("report: %v", err) + } + return report +} + +// seedEveryOutcome seeds one job per outcome the report distinguishes, plus +// jobs just outside the window. The window is [since, until). +func seedEveryOutcome(t *testing.T) (*Store, time.Time, time.Time) { + f, clock := newReportFixture(t) + now := clock.Now() + since, until := now.Add(-time.Hour), now.Add(time.Hour) + + // Clean on the first try, over two attempts: spend counts both. + f.job("j01-clean", reportRunCI, protocol.JobAccepted, now) + failedFirst := f.attempt("j01-clean", 1, protocol.AttemptFailed, "") + f.spend(failedFirst, 0.25, -1) + f.event(failedFirst, protocol.EventLog, map[string]any{"cost": 50.0}) + clean := f.attempt("j01-clean", 2, protocol.AttemptAccepted, result(t, summary("published", "", nil))) + f.spend(clean, 0.25, 0) + f.spend(clean, 0.25, -1) + f.step(clean, protocol.PublishStepProof, shaA) + f.step(clean, protocol.PublishStepCI, shaA) + + // Two pushed rounds that ended green. + f.job("j02-repaired", reportRunCI, protocol.JobAccepted, now) + repaired := f.attempt("j02-repaired", 1, protocol.AttemptAccepted, + result(t, summary("published", "", map[string]any{"ci_repairs": twoRounds}))) + f.spend(repaired, 0.5, -1) + f.step(repaired, protocol.PublishStepProof, shaA) + f.rounds(repaired, shaA, shaB, shaC) + f.step(repaired, protocol.PublishStepCI, shaC) + + // Repair exhausted, then a person pushed and the publish-only retry went + // green on their head: repaired, unsuccessful, and person-fixed. + f.job("j03-exhausted", reportRunCI, protocol.JobAccepted, now) + exhausted := f.attempt("j03-exhausted", 1, protocol.AttemptAccepted, result(t, + summary("published", "", nil), + summary("failed", "ci_repair_exhausted", map[string]any{"ci_repairs": twoRounds, "ci_failures": redChecks}))) + f.spend(exhausted, 0.25, -1) + f.step(exhausted, protocol.PublishStepProof, shaA) + f.rounds(exhausted, shaA, shaB, shaC) + f.step(exhausted, protocol.PublishStepCI, shaD) + + // Red, retried, still red: the retry-repair signal. + f.job("j04-still-red", reportRunCI, protocol.JobAcceptedUnpublished, now) + stillRed := f.attempt("j04-still-red", 1, protocol.AttemptAcceptedUnpublished, result(t, + summary("failed", "ci_failed", map[string]any{"ci_failures": redChecks}), + summary("failed", "ci_failed", map[string]any{"ci_failures": redChecks}))) + f.spend(stillRed, 0.25, -1) + f.step(stillRed, protocol.PublishStepProof, shaA) + + // CI timed out. + f.job("j05-timeout", reportRunCI, protocol.JobAcceptedUnpublished, now) + timeout := f.attempt("j05-timeout", 1, protocol.AttemptAcceptedUnpublished, + result(t, summary("failed", "ci_timeout", map[string]any{"ci_failures": pendingChecks}))) + f.step(timeout, protocol.PublishStepProof, shaA) + + // A pass after a re-run: flaky. Re-run 1 appears in history and current, + // and counts once. + f.job("j06-flaky", reportRunCI, protocol.JobAccepted, now) + flaky := f.attempt("j06-flaky", 1, protocol.AttemptAccepted, result(t, + summary("published", "", map[string]any{"ci_reruns": []map[string]any{ + {"attempt": 1, "jobs": []int64{11}, "outcome": "failed"}, + {"attempt": 2, "jobs": []int64{11}, "outcome": "passed"}, + }}), + summary("failed", "publish_unavailable", map[string]any{"ci_reruns": []map[string]any{ + {"attempt": 1, "jobs": []int64{11}, "outcome": "failed"}, + }}))) + f.spend(flaky, 0.5, -1) + f.step(flaky, protocol.PublishStepProof, shaA) + f.step(flaky, protocol.PublishStepCI, shaA) + + // Held by the engine marker, before any publish step. + f.job("j07-held-marker", reportRunCI, protocol.JobAcceptedUnpublished, now) + f.spend(f.attempt("j07-held-marker", 1, protocol.AttemptAcceptedUnpublished, result(t, "held")), 0.25, -1) + + // Held by the publishing runner's summary. + f.job("j08-held-summary", reportRunNoCI, protocol.JobAcceptedUnpublished, now) + f.attempt("j08-held-summary", 1, protocol.AttemptAcceptedUnpublished, + result(t, summary("held", "publish_held", nil))) + + // Publish never ran. + f.job("j09-not-attempted", reportRunNoCI, protocol.JobAcceptedUnpublished, now) + f.attempt("j09-not-attempted", 1, protocol.AttemptAcceptedUnpublished, result(t, "not_attempted")) + + // A result that is not JSON at all. + f.job("j10-malformed", reportRunNoCI, protocol.JobAcceptedUnpublished, now) + f.spend(f.attempt("j10-malformed", 1, protocol.AttemptAcceptedUnpublished, `{"publish": {"state"`), 0.125, -1) + + // Accepted under a definition that does not wait for CI. + f.job("j11-no-ci", reportRunNoCI, protocol.JobAccepted, now) + noCI := f.attempt("j11-no-ci", 1, protocol.AttemptAccepted, result(t, summary("published", "", nil))) + f.spend(noCI, 0.25, -1) + f.step(noCI, protocol.PublishStepProof, shaA) + + // Failed, updated exactly at since: the lower bound is inclusive. + f.job("j12-failed", reportRunCI, protocol.JobFailed, since) + f.spend(f.attempt("j12-failed", 1, protocol.AttemptFailed, ""), 1.0, 2) + + f.job("j13-cancelled", reportRunCI, protocol.JobCancelled, now) + f.spend(f.attempt("j13-cancelled", 1, protocol.AttemptCancelled, ""), 0.125, 1) + + // Clean on the first try, with a history entry that is not a summary. + f.job("j14-bad-history", reportRunCI, protocol.JobAccepted, now) + badHistory := f.attempt("j14-bad-history", 1, protocol.AttemptAccepted, + result(t, summary("published", "", nil), "garbage")) + f.step(badHistory, protocol.PublishStepProof, shaA) + f.step(badHistory, protocol.PublishStepCI, shaA) + + // Outside the window: before since, exactly at until, and not terminal. + for _, outside := range []struct { + id, state string + updated time.Time + }{ + {"x-before", protocol.JobAccepted, since.Add(-time.Millisecond)}, + {"x-at-until", protocol.JobFailed, until}, + {"x-active", protocol.JobActive, now}, + } { + f.job(outside.id, reportRunCI, outside.state, outside.updated) + attempt := f.attempt(outside.id, 1, protocol.AttemptRunning, result(t, "held")) + f.spend(attempt, 100, 7) + f.step(attempt, protocol.PublishStepProof, shaA) + f.step(attempt, protocol.PublishStepCI, shaE) + } + return f.store, since, until +} + +// Seeded jobs across every terminal state and publish outcome produce exact +// counts; jobs outside [since, until) are excluded. +func TestReportCountsEveryOutcomeExactlyWithinTheWindow(t *testing.T) { + store, since, until := seedEveryOutcome(t) + got := mustReport(t, store, since, until) + + want := Report{ + Since: since, Until: until, ObservedAt: store.now().UTC(), + Jobs: ReportJobs{Total: 14, Accepted: 6, AcceptedUnpublished: 6, Failed: 1, Cancelled: 1}, + Accepts: ReportAccepts{Clean: 5, CIGreen: 4, NoCIWait: 1, PersonFixed: 1}, + Publish: ReportPublish{ + Eligible: 12, Published: 6, Held: 2, NotAttempted: 1, Failed: 2, Unreadable: 1, + FailedCodes: map[string]int{"ci_failed": 1, "ci_timeout": 1}, + HeldRate: floatPointer(2.0 / 12), + }, + CI: ReportCI{ + Waited: 7, + FirstPassGreen: 2, + Repair: ReportCIRepair{ + Entered: 2, EntryRate: floatPointer(2.0 / 7), + Succeeded: 1, SuccessRate: floatPointer(0.5), + Rounds: 4, + StopCodes: map[string]int{"ci_repair_exhausted": 1}, + }, + Reruns: ReportCIReruns{Attempts: 1, Reruns: 2, FlakyPasses: 1}, + Revisit: ReportCIRevisit{ + CITimeouts: 1, CITimeoutRate: floatPointer(1.0 / 7), + RetryStillRed: 1, RetryStillRedRate: floatPointer(1.0 / 7), + }, + }, + Spend: ReportSpend{ + TotalUSD: 4.0, UnmeteredSends: 3, PerCleanAcceptUSD: floatPointer(0.8), + ByOutcome: map[string]ReportSpendBucket{ + spendClean: {Jobs: 5, CostUSD: 2.0}, + spendPersonFixed: {Jobs: 1, CostUSD: 0.25}, + spendUnverified: {}, + spendHeld: {Jobs: 2, CostUSD: 0.25}, + spendUnpublished: {Jobs: 4, CostUSD: 0.375}, + spendFailed: {Jobs: 1, CostUSD: 1.0, UnmeteredSends: 2}, + spendCancelled: {Jobs: 1, CostUSD: 0.125, UnmeteredSends: 1}, + }, + }, + UnreadableResults: 2, + } + if !reflect.DeepEqual(got, want) { + gotJSON, _ := json.MarshalIndent(got, "", " ") + wantJSON, _ := json.MarshalIndent(want, "", " ") + t.Fatalf("report mismatch\n got: %s\nwant: %s", gotJSON, wantJSON) + } +} + +// An accept the ledger cannot vouch for is never clean: a CI-waiting accept +// with no ci record, and one whose frozen definition no longer parses. +func TestReportNeverCountsAnUnprovableAcceptAsClean(t *testing.T) { + f, clock := newReportFixture(t) + now := clock.Now() + f.job("no-ci-record", reportRunCI, protocol.JobAccepted, now) + missing := f.attempt("no-ci-record", 1, protocol.AttemptAccepted, result(t, summary("published", "", nil))) + f.spend(missing, 0.5, -1) + f.step(missing, protocol.PublishStepProof, shaA) + + seedRun(t, f.store, "run-broken") + useSnapshot(t, f.store, "run-broken", "not: [a definition") + f.job("broken-snapshot", "run-broken", protocol.JobAccepted, now) + broken := f.attempt("broken-snapshot", 1, protocol.AttemptAccepted, result(t, summary("published", "", nil))) + f.step(broken, protocol.PublishStepProof, shaA) + + got := mustReport(t, f.store, now.Add(-time.Hour), now.Add(time.Hour)) + if got.Accepts != (ReportAccepts{Unverified: 2}) { + t.Fatalf("accepts = %+v, want 2 unverified and nothing clean", got.Accepts) + } + if got.Spend.PerCleanAcceptUSD != nil { + t.Fatalf("per clean accept = %v, want null with no clean accept", *got.Spend.PerCleanAcceptUSD) + } + if bucket := got.Spend.ByOutcome[spendUnverified]; bucket.Jobs != 2 || bucket.CostUSD != 0.5 { + t.Fatalf("unverified spend = %+v, want 2 jobs, 0.5", bucket) + } +} + +// The empty window is zeros and nulls, never an error, and every spend +// bucket is present. +func TestReportOverAnEmptyWindowIsZeros(t *testing.T) { + store, clock := newTestStore(t) + now := clock.Now() + got := mustReport(t, store, now.Add(-time.Hour), now) + if got.Jobs != (ReportJobs{}) || got.Accepts != (ReportAccepts{}) || got.UnreadableResults != 0 || + got.Spend.TotalUSD != 0 || got.CI.Waited != 0 { + t.Fatalf("empty report = %+v, want zeros", got) + } + if got.Publish.HeldRate != nil || got.CI.Repair.EntryRate != nil || got.CI.Repair.SuccessRate != nil || + got.CI.Revisit.CITimeoutRate != nil || got.CI.Revisit.RetryStillRedRate != nil || + got.Spend.PerCleanAcceptUSD != nil { + t.Fatalf("empty report rates = %+v, want every rate null", got) + } + if len(got.Spend.ByOutcome) != len(spendBuckets) { + t.Fatalf("spend buckets = %v, want all %d present", got.Spend.ByOutcome, len(spendBuckets)) + } +} + +func getReport(t *testing.T, handler http.Handler, query string) *httptest.ResponseRecorder { + t.Helper() + request := httptest.NewRequest("GET", "http://127.0.0.1:8383/api/report"+query, nil) + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, request) + return recorder +} + +// GET /api/report returns the store's object for the window it names, and +// defaults to the seven days before now. +func TestReportRouteReturnsTheStoreReport(t *testing.T) { + store, since, until := seedEveryOutcome(t) + handler := NewHandler(store, "", discardLogger()) + + recorder := getReport(t, handler, "?since="+since.Format(time.RFC3339)+"&until="+until.Format(time.RFC3339)) + if recorder.Code != http.StatusOK { + t.Fatalf("GET /api/report = %d %s", recorder.Code, recorder.Body.String()) + } + var got Report + if err := json.Unmarshal(recorder.Body.Bytes(), &got); err != nil { + t.Fatalf("decode report: %v", err) + } + want := mustReport(t, store, since, until) + if !reflect.DeepEqual(got, want) { + t.Fatalf("route report = %+v\nwant %+v", got, want) + } + + // A minute later, so the fixture's jobs (updated at the old now) are + // inside a window whose end is exclusive. + store.now = func() time.Time { return since.Add(time.Hour + time.Minute) } + recorder = getReport(t, handler, "") + if recorder.Code != http.StatusOK { + t.Fatalf("GET /api/report (default window) = %d %s", recorder.Code, recorder.Body.String()) + } + var defaulted Report + if err := json.Unmarshal(recorder.Body.Bytes(), &defaulted); err != nil { + t.Fatalf("decode report: %v", err) + } + now := store.now().UTC() + if !defaulted.Until.Equal(now) || !defaulted.Since.Equal(now.Add(-7*24*time.Hour)) { + t.Fatalf("default window = [%s, %s), want the seven days before %s", defaulted.Since, defaulted.Until, now) + } + // The seven-day window takes in x-before (an hour and a millisecond ago) + // and still leaves out x-at-until (an hour from now) and x-active. + if defaulted.Jobs.Total != 15 { + t.Fatalf("default window jobs = %d, want 15", defaulted.Jobs.Total) + } +} + +// A malformed or inverted window is a 400 naming the fault. +func TestReportRouteRejectsABadWindow(t *testing.T) { + store, clock := newTestStore(t) + handler := NewHandler(store, "", discardLogger()) + now := clock.Now() + for _, test := range []struct{ query, code string }{ + {"?since=yesterday", "invalid_query_parameter"}, + {"?until=1754380800000", "invalid_query_parameter"}, + {"?since=" + now.Format(time.RFC3339) + "&until=" + now.Format(time.RFC3339), "invalid_report_window"}, + {"?since=" + now.Format(time.RFC3339) + "&until=" + now.Add(-time.Hour).Format(time.RFC3339), "invalid_report_window"}, + } { + recorder := getReport(t, handler, test.query) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("GET /api/report%s = %d, want 400", test.query, recorder.Code) + } + var body struct { + Error struct{ Code string } `json:"error"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil || body.Error.Code != test.code { + t.Fatalf("GET /api/report%s error = %s, want %s", test.query, recorder.Body.String(), test.code) + } + } +} From 9104effbef3291c8535d7225faad72eaaf9a1031 Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Mon, 28 Sep 2026 00:28:05 -0600 Subject: [PATCH 2/5] test(controlplane): report reads unreadable results and history-only signals Co-Authored-By: Claude Opus 5.5 --- internal/controlplane/report_test.go | 51 ++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/internal/controlplane/report_test.go b/internal/controlplane/report_test.go index ebb39c2..96cd90b 100644 --- a/internal/controlplane/report_test.go +++ b/internal/controlplane/report_test.go @@ -449,3 +449,54 @@ func TestReportRouteRejectsABadWindow(t *testing.T) { } } } + +// What the report cannot read is counted, never guessed; and a signal that +// only history carries — a timeout, a red summary before a retry, a repair +// with no pushed round — is still read, once. +func TestReportClassifiesUnreadableResultsAndReadsSignalsFromHistory(t *testing.T) { + f, clock := newReportFixture(t) + now := clock.Now() + seed := func(id, stored string) { + f.job(id, reportRunCI, protocol.JobAcceptedUnpublished, now) + f.step(f.attempt(id, 1, protocol.AttemptAcceptedUnpublished, stored), protocol.PublishStepProof, shaA) + } + seed("u1-unknown-marker", result(t, "maybe_later")) + seed("u2-number", result(t, 42)) + seed("u3-unknown-state", result(t, summary("exploded", "", nil))) + seed("u4-history-object", `{"publish":{"state":"failed","code":"ci_failed"},"publish_history":{"state":"failed"}}`) + seed("u5-no-publish", `{"changed_paths":[]}`) + // A timeout, then a retry that went red: the timeout counts, and a + // timeout is not red, so this is not the retry-repair signal. + seed("s1-timeout-then-red", result(t, + summary("failed", "ci_failed", map[string]any{"ci_failures": redChecks}), + summary("failed", "ci_timeout", map[string]any{"ci_failures": pendingChecks}))) + // A repair that stopped before pushing anything, red by its recorded + // failures alone, then a retry still red. + seed("s2-unpushed-repair", result(t, + summary("failed", "ci_failed", map[string]any{"ci_failures": redChecks}), + summary("failed", "ci_repair_no_change", map[string]any{ + "ci_failures": redChecks, + "ci_repairs": []map[string]any{{"round": 1, "head_before": shaA, "outcome": "ci_repair_no_change"}}, + }))) + + got := mustReport(t, f.store, now.Add(-time.Hour), now.Add(time.Hour)) + if got.UnreadableResults != 5 || got.Publish.Unreadable != 5 { + t.Fatalf("unreadable = %d results, %d publish; want 5 and 5", got.UnreadableResults, got.Publish.Unreadable) + } + if got.Publish.Failed != 2 || !reflect.DeepEqual(got.Publish.FailedCodes, map[string]int{"ci_failed": 2}) { + t.Fatalf("publish failed = %d %v, want 2 ci_failed", got.Publish.Failed, got.Publish.FailedCodes) + } + if got.CI.Waited != 7 { + t.Fatalf("waited = %d, want 7", got.CI.Waited) + } + if got.CI.Revisit.CITimeouts != 1 || got.CI.Revisit.RetryStillRed != 1 { + t.Fatalf("revisit = %+v, want 1 timeout and 1 retry still red", got.CI.Revisit) + } + wantRepair := ReportCIRepair{ + Entered: 1, EntryRate: floatPointer(1.0 / 7), SuccessRate: floatPointer(0), + StopCodes: map[string]int{"ci_repair_no_change": 1}, + } + if !reflect.DeepEqual(got.CI.Repair, wantRepair) { + t.Fatalf("repair = %+v, want %+v", got.CI.Repair, wantRepair) + } +} From 460c0183cd6197d87f37b3e627f7573d71810596 Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Mon, 28 Sep 2026 00:29:08 -0600 Subject: [PATCH 3/5] test(controlplane): a re-run that fails again is not a flaky pass Co-Authored-By: Claude Opus 5.5 --- internal/controlplane/report_test.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/internal/controlplane/report_test.go b/internal/controlplane/report_test.go index 96cd90b..b3666d2 100644 --- a/internal/controlplane/report_test.go +++ b/internal/controlplane/report_test.go @@ -478,22 +478,30 @@ func TestReportClassifiesUnreadableResultsAndReadsSignalsFromHistory(t *testing. "ci_failures": redChecks, "ci_repairs": []map[string]any{{"round": 1, "head_before": shaA, "outcome": "ci_repair_no_change"}}, }))) + // A re-run that failed again is a re-run, not a flaky pass. + seed("s3-rerun-still-red", result(t, summary("failed", "ci_failed", map[string]any{ + "ci_failures": redChecks, + "ci_reruns": []map[string]any{{"attempt": 1, "jobs": []int64{11}, "outcome": "failed"}}, + }))) got := mustReport(t, f.store, now.Add(-time.Hour), now.Add(time.Hour)) + if got.CI.Reruns != (ReportCIReruns{Attempts: 1, Reruns: 1}) { + t.Fatalf("reruns = %+v, want one re-run and no flaky pass", got.CI.Reruns) + } if got.UnreadableResults != 5 || got.Publish.Unreadable != 5 { t.Fatalf("unreadable = %d results, %d publish; want 5 and 5", got.UnreadableResults, got.Publish.Unreadable) } - if got.Publish.Failed != 2 || !reflect.DeepEqual(got.Publish.FailedCodes, map[string]int{"ci_failed": 2}) { - t.Fatalf("publish failed = %d %v, want 2 ci_failed", got.Publish.Failed, got.Publish.FailedCodes) + if got.Publish.Failed != 3 || !reflect.DeepEqual(got.Publish.FailedCodes, map[string]int{"ci_failed": 3}) { + t.Fatalf("publish failed = %d %v, want 3 ci_failed", got.Publish.Failed, got.Publish.FailedCodes) } - if got.CI.Waited != 7 { - t.Fatalf("waited = %d, want 7", got.CI.Waited) + if got.CI.Waited != 8 { + t.Fatalf("waited = %d, want 8", got.CI.Waited) } if got.CI.Revisit.CITimeouts != 1 || got.CI.Revisit.RetryStillRed != 1 { t.Fatalf("revisit = %+v, want 1 timeout and 1 retry still red", got.CI.Revisit) } wantRepair := ReportCIRepair{ - Entered: 1, EntryRate: floatPointer(1.0 / 7), SuccessRate: floatPointer(0), + Entered: 1, EntryRate: floatPointer(1.0 / 8), SuccessRate: floatPointer(0), StopCodes: map[string]int{"ci_repair_no_change": 1}, } if !reflect.DeepEqual(got.CI.Repair, wantRepair) { From c33cac77c478f878b750b4236de6322845b5a82d Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Mon, 28 Sep 2026 00:30:05 -0600 Subject: [PATCH 4/5] fix(controlplane): read unmetered_sends as an integer whatever SQLite sums Co-Authored-By: Claude Opus 5.5 --- internal/controlplane/report.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controlplane/report.go b/internal/controlplane/report.go index 230ffbd..0b92124 100644 --- a/internal/controlplane/report.go +++ b/internal/controlplane/report.go @@ -621,7 +621,7 @@ func (s *Store) reportSpend(ctx context.Context, since, until int64, byID map[st rows, err := s.db.QueryContext(ctx, ` SELECT a.job_id, COALESCE(SUM(json_extract(CAST(e.payload AS TEXT), '$.payload.cost')), 0), - COALESCE(SUM(json_extract(CAST(e.payload AS TEXT), '$.payload.unmetered_sends')), 0) + CAST(COALESCE(SUM(json_extract(CAST(e.payload AS TEXT), '$.payload.unmetered_sends')), 0) AS INTEGER) FROM events e JOIN attempts a ON a.id = e.attempt_id WHERE e.type = ? AND json_valid(CAST(e.payload AS TEXT)) AND a.job_id IN (`+reportWindowJobs+`) From f5db08f0c3e6c829741773865da4c8250327cc52 Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Mon, 28 Sep 2026 00:42:15 -0600 Subject: [PATCH 5/5] =?UTF-8?q?feat(cli):=20jig=20report=20=E2=80=94=20the?= =?UTF-8?q?=20quality=20report=20over=20a=20window=20(U3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `jig report [--server] [--since 7d|] [--until ] [--json]` reads GET /api/report. A duration --since converts to RFC3339 against now; --json prints the API object verbatim; the prose form is one short table per section (jobs, accepts, publish, CI repair, re-runs, spend), prints null rates as n/a, and states in one line each what jig cannot see (R4) and the unmetered-send count. Exit codes follow def.go: 0, 1 when the control plane is unreachable or rejects, 2 on usage. Co-Authored-By: Claude Opus 5.5 --- README.md | 39 +++++- cmd/jig/main.go | 10 +- cmd/jig/report.go | 270 +++++++++++++++++++++++++++++++++++++++++ cmd/jig/report_test.go | 256 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 571 insertions(+), 4 deletions(-) create mode 100644 cmd/jig/report.go create mode 100644 cmd/jig/report_test.go diff --git a/README.md b/README.md index 9c47fb5..b897d55 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ the whole path from here to a browsable accepted run. --- -## The three surfaces +## The four surfaces **`jig run`** — the direct harness. Loads a definition file, freezes a run against the local repository at its current HEAD, executes the chain @@ -108,6 +108,9 @@ and publishes accepted work as a branch and pull request. definition offline, save it, invoke it, and stand up the schedules and GitHub polls that invoke it unattended. Every read command takes `--json`. +**`jig report`** — measurement. Jobs, accepts, publish outcomes, CI repair, +re-runs, and spend over a time window, read from the control plane's ledger. + --- ## Authoring a definition @@ -301,6 +304,40 @@ their terminal state, and destroys their ephemeral scratch. --- +## Measuring the factory + +```sh +jig report # the last seven days +jig report --since 24h +jig report --since 2026-09-01T00:00:00Z --until 2026-09-15T00:00:00Z +jig report --json # the GET /api/report object, verbatim +``` + +`jig report` reads the control plane's ledger over a window and changes +nothing. A job is in the window when it is terminal and was last updated in +`[since, until)`. `--since` takes a duration back from now (`7d`, `24h`, +`90m`) or an RFC3339 timestamp; `--until` takes a timestamp and defaults to +now. The prose form prints one short table per section: + +| Section | What it counts | +|---|---| +| Jobs | terminal jobs by state | +| Accepts | clean accepts (green CI on a head jig pushed, or a definition that does not wait for CI), person-fixed accepts (green CI on a head jig did not push), and accepts the ledger cannot classify | +| Publish | publish outcomes of accepted work, the held rate, and failure codes | +| CI repair | attempts that waited for CI, first-pass greens, repair entry and success rates, rounds, stop codes, and the `ci_timeout` and retried-still-red rates | +| Re-runs | flaky-check re-runs, and passes that came only after one | +| Spend | total agent spend, spend per clean accept, and each outcome's share | + +A rate with nothing to divide by prints `n/a`, never `0%`. + +Two limits are printed with every report. jig does not watch CI after an +accept, so there is no post-merge CI rate. And spend is a floor: a send +killed before it returned a result has no cost to record (the report counts +these as unmetered sends), and spend recorded by jig v0.2.0 or earlier +omits phases that did not pass. + +--- + ## Operational notes These are the things that will surprise you if nobody says them out loud. diff --git a/cmd/jig/main.go b/cmd/jig/main.go index 7b2ddcf..ccb7783 100644 --- a/cmd/jig/main.go +++ b/cmd/jig/main.go @@ -1,10 +1,11 @@ // jig is a local-first software factory: one binary that runs repeatable, // phased coding-agent workflows against Git repositories. // -// Five subcommands, one binary, no Node (R18): run is the serverless direct +// Six subcommands, one binary, no Node (R18): run is the serverless direct // harness (U11), serve is the control plane with its embedded UI (U2/U6/U8), -// worker is the execution host (U3/U4/U7), and def and trigger are the -// operator's surface over definitions and admission (U5/U6). +// worker is the execution host (U3/U4/U7), def and trigger are the +// operator's surface over definitions and admission (U5/U6), and report reads +// the factory's quality over a window (plan 2026-09-28-001, U3). package main import ( @@ -29,6 +30,7 @@ Usage: jig run Run a definition directly against a local repository jig def Validate, manage, and invoke job definitions jig trigger Manage admission triggers (cron schedules, GitHub polling) + jig report Summarize the factory's quality over a time window jig version Print the release identity Run any subcommand with --help for its flags and exit codes. @@ -61,6 +63,8 @@ func run(ctx context.Context, args []string) int { return defCommand(ctx, args[1:], os.Stdout, os.Stderr) case "trigger": return triggerCommand(ctx, args[1:], os.Stdout, os.Stderr) + case "report": + return reportCommand(ctx, args[1:], os.Stdout, os.Stderr) case "version", "--version": fmt.Fprintf(os.Stdout, "jig %s\n", version) return 0 diff --git a/cmd/jig/report.go b/cmd/jig/report.go new file mode 100644 index 0000000..fef7ff8 --- /dev/null +++ b/cmd/jig/report.go @@ -0,0 +1,270 @@ +// report.go — `jig report` (plan 2026-09-28-001, U3): the operator's surface +// over GET /api/report. The control plane computes the report (KTD3); this +// command only chooses the window and renders it. +// +// It follows def.go's conventions: flags before positionals, --json prints +// the API's own object verbatim, and exit codes 0/1/2. +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "text/tabwriter" + "time" + + "github.com/StructuPath/jig/internal/controlplane" +) + +const reportUsage = `Usage: jig report [flags] + +Summarize the factory's quality over a time window: jobs by terminal state, +accepts, publish outcomes, CI repair, flaky-check re-runs, and spend. The +control plane computes it from its ledger; nothing is written. + +Flags: + --server control plane URL (default http://127.0.0.1:8383) + --since window start: a duration back from now (7d, 24h, 90m) or + an RFC3339 timestamp (default: 7 days before --until) + --until window end, RFC3339 (default: now) + --json emit the API object instead of the prose report + +A job is in the window when it is terminal and was last updated in +[since, until). A rate with nothing to divide by prints n/a. + +Exit codes: + 0 success + 1 the control plane could not be reached, or rejected the request + 2 usage error +` + +func reportCommand(ctx context.Context, args []string, stdout, stderr io.Writer) int { + if len(args) > 0 && (args[0] == "help" || args[0] == "-h" || args[0] == "--help") { + fmt.Fprint(stdout, reportUsage) + return exitAccepted + } + flags, server, asJSON := defFlags("jig report", stderr, reportUsage) + sinceFlag := flags.String("since", "", "window start: a duration back from now or RFC3339") + untilFlag := flags.String("until", "", "window end, RFC3339") + if err := flags.Parse(args); err != nil { + return exitUsage + } + if flags.NArg() != 0 { + fmt.Fprintf(stderr, "jig report: unexpected argument %q\n\n%s", flags.Arg(0), reportUsage) + return exitUsage + } + query := url.Values{} + if *sinceFlag != "" { + since, err := parseReportSince(*sinceFlag, time.Now()) + if err != nil { + fmt.Fprintf(stderr, "jig report: --since: %v\n\n%s", err, reportUsage) + return exitUsage + } + query.Set("since", since.UTC().Format(time.RFC3339)) + } + if *untilFlag != "" { + until, err := time.Parse(time.RFC3339, strings.TrimSpace(*untilFlag)) + if err != nil { + fmt.Fprintf(stderr, "jig report: --until: %q is not an RFC3339 timestamp\n\n%s", + *untilFlag, reportUsage) + return exitUsage + } + query.Set("until", until.UTC().Format(time.RFC3339)) + } + path := "/api/report" + if len(query) > 0 { + path += "?" + query.Encode() + } + + var raw json.RawMessage + if err := apiCall(ctx, *server, http.MethodGet, path, nil, &raw); err != nil { + fmt.Fprintf(stderr, "jig report: %v\n", err) + return exitInfraFailed + } + if *asJSON { + fmt.Fprintf(stdout, "%s\n", raw) + return exitAccepted + } + var report controlplane.Report + if err := json.Unmarshal(raw, &report); err != nil { + fmt.Fprintf(stderr, "jig report: decode response: %v\n", err) + return exitInfraFailed + } + printReport(stdout, report) + return exitAccepted +} + +// parseReportSince reads --since: a positive duration back from now — Go's +// units plus whole days (7d) — or an RFC3339 timestamp. +func parseReportSince(value string, now time.Time) (time.Time, error) { + value = strings.TrimSpace(value) + if at, err := time.Parse(time.RFC3339, value); err == nil { + return at, nil + } + var span time.Duration + if days, found := strings.CutSuffix(value, "d"); found { + count, err := strconv.Atoi(days) + if err != nil { + return time.Time{}, fmt.Errorf("%q is not a duration (7d, 24h, 90m) or an RFC3339 timestamp", value) + } + span = time.Duration(count) * 24 * time.Hour + } else { + parsed, err := time.ParseDuration(value) + if err != nil { + return time.Time{}, fmt.Errorf("%q is not a duration (7d, 24h, 90m) or an RFC3339 timestamp", value) + } + span = parsed + } + if span <= 0 { + return time.Time{}, fmt.Errorf("%q must be a positive duration", value) + } + return now.Add(-span), nil +} + +// printReport renders the report as one short table per section, then the +// two lines that say what the numbers cannot (R4) and what spend misses. +func printReport(w io.Writer, report controlplane.Report) { + fmt.Fprintf(w, "jig report %s → %s\n", + report.Since.UTC().Format(time.RFC3339), report.Until.UTC().Format(time.RFC3339)) + + section(w, "Jobs", [][2]string{ + {"terminal", count(report.Jobs.Total)}, + {"accepted", count(report.Jobs.Accepted)}, + {"accepted, unpublished", count(report.Jobs.AcceptedUnpublished)}, + {"failed", count(report.Jobs.Failed)}, + {"cancelled", count(report.Jobs.Cancelled)}, + }) + + accepts := report.Accepts + section(w, "Accepts", [][2]string{ + {"clean", count(accepts.Clean)}, + {" green CI on a head jig pushed", count(accepts.CIGreen)}, + {" definition does not wait for CI", count(accepts.NoCIWait)}, + {"person-fixed (green CI on a head jig did not push)", count(accepts.PersonFixed)}, + {"unverified", count(accepts.Unverified)}, + }) + + publish := report.Publish + rows := [][2]string{ + {"eligible", count(publish.Eligible)}, + {"published", count(publish.Published)}, + {"held", count(publish.Held)}, + {"held rate", rate(publish.HeldRate)}, + {"not attempted", count(publish.NotAttempted)}, + {"failed", count(publish.Failed)}, + } + rows = append(rows, codes(" ", publish.FailedCodes)...) + rows = append(rows, [2]string{"unreadable", count(publish.Unreadable)}) + section(w, "Publish", rows) + + ci := report.CI + repair := ci.Repair + rows = [][2]string{ + {"attempts that waited for CI", count(ci.Waited)}, + {"first-pass green", count(ci.FirstPassGreen)}, + {"entered repair", count(repair.Entered)}, + {"entry rate", rate(repair.EntryRate)}, + {"repaired to green", count(repair.Succeeded)}, + {"success rate", rate(repair.SuccessRate)}, + {"rounds pushed", count(repair.Rounds)}, + } + if len(repair.StopCodes) > 0 { + rows = append(rows, [2]string{"stopped on", ""}) + rows = append(rows, codes(" ", repair.StopCodes)...) + } + rows = append(rows, + [2]string{"ci_timeout", fmt.Sprintf("%d (%s)", ci.Revisit.CITimeouts, rate(ci.Revisit.CITimeoutRate))}, + [2]string{"retried, still red", fmt.Sprintf("%d (%s)", ci.Revisit.RetryStillRed, rate(ci.Revisit.RetryStillRedRate))}, + ) + section(w, "CI repair", rows) + + section(w, "Re-runs", [][2]string{ + {"attempts with a re-run", count(ci.Reruns.Attempts)}, + {"re-runs", count(ci.Reruns.Reruns)}, + {"flaky passes (green after a re-run)", count(ci.Reruns.FlakyPasses)}, + }) + + spend := report.Spend + rows = [][2]string{ + {"total", usd(spend.TotalUSD)}, + {"per clean accept", usdPointer(spend.PerCleanAcceptUSD)}, + } + buckets := make([]string, 0, len(spend.ByOutcome)) + for name := range spend.ByOutcome { + buckets = append(buckets, name) + } + sort.Strings(buckets) + for _, name := range buckets { + bucket := spend.ByOutcome[name] + if bucket.Jobs == 0 { + continue + } + rows = append(rows, [2]string{" " + strings.ReplaceAll(name, "_", " "), + fmt.Sprintf("%s over %d job(s)", usd(bucket.CostUSD), bucket.Jobs)}) + } + section(w, "Spend", rows) + + if report.UnreadableResults > 0 { + fmt.Fprintf(w, "\n%d result(s) could not be read in full and are counted, not guessed.\n", + report.UnreadableResults) + } else { + fmt.Fprintln(w) + } + fmt.Fprintln(w, "Not seen: jig does not watch CI after accept, so there is no post-merge CI rate; "+ + "person-fixed accepts and accepts that did not wait for CI are counted separately.") + if spend.UnmeteredSends > 0 { + fmt.Fprintf(w, "Spend undercounts: %d unmetered send(s) were killed before returning a cost, "+ + "and spend recorded by jig v0.2.0 or earlier omits phases that did not pass.\n", + spend.UnmeteredSends) + } else { + fmt.Fprintln(w, "Unmetered sends: 0. Spend recorded by jig v0.2.0 or earlier omits phases that did not pass.") + } +} + +func section(w io.Writer, title string, rows [][2]string) { + fmt.Fprintf(w, "\n%s\n", title) + table := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + for _, row := range rows { + fmt.Fprintf(table, " %s\t%s\n", row[0], row[1]) + } + table.Flush() +} + +func codes(indent string, byCode map[string]int) [][2]string { + names := make([]string, 0, len(byCode)) + for name := range byCode { + names = append(names, name) + } + sort.Strings(names) + rows := make([][2]string, 0, len(names)) + for _, name := range names { + rows = append(rows, [2]string{indent + name, count(byCode[name])}) + } + return rows +} + +func count(value int) string { return strconv.Itoa(value) } + +// rate prints a fraction as a percentage; a null rate had nothing to divide +// by, and a zero there would read as a measurement. +func rate(value *float64) string { + if value == nil { + return "n/a" + } + return fmt.Sprintf("%.1f%%", *value*100) +} + +func usd(value float64) string { return fmt.Sprintf("$%.2f", value) } + +func usdPointer(value *float64) string { + if value == nil { + return "n/a" + } + return usd(*value) +} diff --git a/cmd/jig/report_test.go b/cmd/jig/report_test.go new file mode 100644 index 0000000..7e359b0 --- /dev/null +++ b/cmd/jig/report_test.go @@ -0,0 +1,256 @@ +// report_test.go — `jig report` (plan 2026-09-28-001, U3) against a live +// `jig serve` whose ledger is seeded row by row: the prose names every +// section and both caveat lines, --json is the API's own object, --since +// takes a duration or a timestamp, and the failures exit 2 or 1. +package main + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "io" + "net" + "net/http" + "os" + "path/filepath" + "reflect" + "regexp" + "strings" + "testing" + "time" + + "github.com/StructuPath/jig/internal/controlplane" + "github.com/StructuPath/jig/internal/protocol" +) + +// startReportServe seeds a ledger and serves it. Three terminal jobs: a +// clean accept under a definition that does not wait for CI and a failure +// with unmetered sends, both updated an hour ago, and a failure updated 30 +// days ago that only a wide window sees. +func startReportServe(t *testing.T) (string, func()) { + t.Helper() + dataDir := t.TempDir() + dbPath := filepath.Join(dataDir, "jig.db") + store, err := controlplane.Open(context.Background(), dbPath) + if err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + source, err := os.ReadFile(stockDefinition("smoke.yaml")) + if err != nil { + t.Fatal(err) + } + + db, err := sql.Open("sqlite", "file:"+dbPath) + if err != nil { + t.Fatal(err) + } + defer db.Close() + exec := func(query string, args ...any) { + t.Helper() + if _, err := db.Exec(query, args...); err != nil { + t.Fatalf("seed %q: %v", query, err) + } + } + now := time.Now() + exec(`INSERT INTO definitions(id, name, generation, source, created_at, updated_at) + VALUES ('def-smoke', 'smoke', 1, ?, ?, ?)`, string(source), now.UnixMilli(), now.UnixMilli()) + exec(`INSERT INTO runs(id, definition_id, definition_generation, snapshot, parameters, targets, state, created_at, updated_at) + VALUES ('run-smoke', 'def-smoke', 1, ?, '{}', '[]', 'active', ?, ?)`, + string(source), now.UnixMilli(), now.UnixMilli()) + + seq := int64(0) + job := func(id, state, attemptState, result string, updated time.Time, cost float64, unmetered int) string { + t.Helper() + exec(`INSERT INTO jobs(id, run_id, repository, base_sha, state, created_at, updated_at) + VALUES (?, 'run-smoke', ?, ?, ?, ?, ?)`, id, "github.com/example/"+id, + strings.Repeat("a", 40), state, updated.UnixMilli(), updated.UnixMilli()) + attemptID := id + "-attempt-1" + var stored any + if result != "" { + stored = result + } + exec(`INSERT INTO attempts(id, job_id, attempt_number, state, result, created_at) + VALUES (?, ?, 1, ?, ?, ?)`, attemptID, id, attemptState, stored, updated.UnixMilli()) + payload, _ := json.Marshal(map[string]any{ + "cost": cost, "tokens": 100, "sends": 1, "unmetered_sends": unmetered}) + event, _ := json.Marshal(protocol.Event{ + Seq: seq, Type: protocol.EventAgentEnd, Phase: "scout", Name: "scout", Payload: payload}) + exec(`INSERT INTO events(attempt_id, seq, type, phase, payload, payload_bytes, server_time) + VALUES (?, ?, ?, 'scout', ?, ?, ?)`, attemptID, seq, protocol.EventAgentEnd, event, + len(event), updated.UnixMilli()) + seq++ + return attemptID + } + hourAgo := now.Add(-time.Hour) + clean := job("job-clean", protocol.JobAccepted, protocol.AttemptAccepted, + `{"publish":{"state":"published","branch":"jig/job-clean"}}`, hourAgo, 0.75, 0) + exec(`INSERT INTO publish_records(attempt_id, step, branch, remote_ref, pr_url, completed_at) + VALUES (?, ?, 'jig/job-clean', ?, '', ?)`, clean, protocol.PublishStepProof, + strings.Repeat("b", 40), hourAgo.UnixMilli()) + job("job-failed", protocol.JobFailed, protocol.AttemptFailed, "", hourAgo, 0.25, 2) + job("job-old", protocol.JobFailed, protocol.AttemptFailed, "", now.Add(-30*24*time.Hour), 4, 0) + + return startServe(t, dataDir, "--no-github-poll") +} + +func jigReport(t *testing.T, args ...string) (int, string, string) { + t.Helper() + var stdout, stderr bytes.Buffer + code := reportCommand(context.Background(), args, &stdout, &stderr) + return code, stdout.String(), stderr.String() +} + +// reportObject decodes a report and drops observed_at, the one field that +// differs between two reads of the same window. +func reportObject(t *testing.T, body []byte) map[string]any { + t.Helper() + var object map[string]any + if err := json.Unmarshal(body, &object); err != nil { + t.Fatalf("decode report %q: %v", body, err) + } + if _, found := object["observed_at"]; !found { + t.Fatalf("report has no observed_at: %s", body) + } + delete(object, "observed_at") + return object +} + +func TestReportProseNamesEverySectionAndBothCaveats(t *testing.T) { + base, stop := startReportServe(t) + defer stop() + + code, stdout, stderr := jigReport(t, "--server", base) + if code != exitAccepted { + t.Fatalf("exit = %d, want %d (stderr: %s)", code, exitAccepted, stderr) + } + for _, want := range []string{ + "\nJobs\n", "\nAccepts\n", "\nPublish\n", "\nCI repair\n", "\nRe-runs\n", "\nSpend\n", + // The R4 caveat, one line. + "Not seen: jig does not watch CI after accept, so there is no post-merge CI rate; " + + "person-fixed accepts and accepts that did not wait for CI are counted separately.\n", + // The seeded failure's two killed sends, one line. + "Spend undercounts: 2 unmetered send(s) were killed before returning a cost", + } { + if !strings.Contains(stdout, want) { + t.Errorf("prose is missing %q\n---\n%s", want, stdout) + } + } + for _, want := range []string{ + // Nothing waited for CI, so a CI rate is n/a, never 0%. + `(?m)^ entry rate +n/a$`, + // $1.00 over the one clean accept; the 30-day-old job is outside 7d. + `(?m)^ per clean accept +\$1\.00$`, + `(?m)^ clean +1$`, + } { + if !regexp.MustCompile(want).MatchString(stdout) { + t.Errorf("prose does not match %q\n---\n%s", want, stdout) + } + } +} + +func TestReportJSONIsTheAPIObject(t *testing.T) { + base, stop := startReportServe(t) + defer stop() + since, until := "2026-01-01T00:00:00Z", time.Now().Add(time.Hour).UTC().Format(time.RFC3339) + + code, stdout, stderr := jigReport(t, "--server", base, "--json", "--since", since, "--until", until) + if code != exitAccepted { + t.Fatalf("exit = %d, want %d (stderr: %s)", code, exitAccepted, stderr) + } + response, err := http.Get(base + "/api/report?since=" + since + "&until=" + until) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + api, err := io.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + if got, want := reportObject(t, []byte(stdout)), reportObject(t, api); !reflect.DeepEqual(got, want) { + t.Errorf("--json =\n%v\nwant GET /api/report =\n%v", got, want) + } +} + +func TestReportSinceTakesADurationOrATimestamp(t *testing.T) { + base, stop := startReportServe(t) + defer stop() + + window := func(args ...string) controlplane.Report { + t.Helper() + code, stdout, stderr := jigReport(t, append([]string{"--server", base, "--json"}, args...)...) + if code != exitAccepted { + t.Fatalf("%v: exit = %d (stderr: %s)", args, code, stderr) + } + var report controlplane.Report + if err := json.Unmarshal([]byte(stdout), &report); err != nil { + t.Fatal(err) + } + return report + } + + week := window("--since", "7d") + if drift := time.Since(week.Since) - 7*24*time.Hour; drift < 0 || drift > time.Minute { + t.Errorf("--since 7d sent since = %s, want seven days before now", week.Since) + } + if week.Jobs.Total != 2 { + t.Errorf("--since 7d: jobs = %d, want 2 (the 30-day-old job is outside)", week.Jobs.Total) + } + if recent := window("--since", "90m"); recent.Jobs.Total != 2 { + t.Errorf("--since 90m: jobs = %d, want 2", recent.Jobs.Total) + } + if narrow := window("--since", "30m"); narrow.Jobs.Total != 0 { + t.Errorf("--since 30m: jobs = %d, want 0 (both recent jobs are an hour old)", narrow.Jobs.Total) + } + + since := time.Now().Add(-40 * 24 * time.Hour).UTC().Truncate(time.Second) + wide := window("--since", since.Format(time.RFC3339)) + if !wide.Since.Equal(since) { + t.Errorf("--since %s sent since = %s", since.Format(time.RFC3339), wide.Since) + } + if wide.Jobs.Total != 3 { + t.Errorf("--since 40 days ago: jobs = %d, want 3", wide.Jobs.Total) + } +} + +func TestReportRejectsAnUnparseableWindowWithUsage(t *testing.T) { + for _, args := range [][]string{ + {"--since", "last week"}, + {"--since", "7"}, + {"--since", "-7d"}, + {"--until", "7d"}, + {"stray"}, + } { + // No server is contacted: the URL is never reached. + code, _, stderr := jigReport(t, append([]string{"--server", "http://127.0.0.1:1"}, args...)...) + if code != exitUsage { + t.Errorf("%v: exit = %d, want %d", args, code, exitUsage) + } + if !strings.Contains(stderr, "Usage: jig report") { + t.Errorf("%v: stderr carries no usage:\n%s", args, stderr) + } + } +} + +func TestReportExitsOneWhenTheServerIsUnreachable(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + address := listener.Addr().String() + listener.Close() + + code, stdout, stderr := jigReport(t, "--server", "http://"+address) + if code != exitInfraFailed { + t.Fatalf("exit = %d, want %d (stderr: %s)", code, exitInfraFailed, stderr) + } + if stdout != "" { + t.Errorf("stdout = %q, want nothing", stdout) + } + if !strings.Contains(stderr, "jig report: GET /api/report") { + t.Errorf("stderr does not name the failed request: %s", stderr) + } +}