Skip to content

Commit f58a494

Browse files
aledbfclaude
andcommitted
test(parity): retry parity sides on transient docker build flakes
The runtime parity lane's docker builds flake on contended CI runners (BuildKit "failed to solve", a registry hiccup, or a container-setup failure surfaced as `Command failed: docker build`). A flake on one side while the other succeeds shows up as a false TS-vs-Go divergence and fails the job under PARITY_STRICT=true, where an inconclusive skip is red too. This turned unrelated dependabot PRs (#9, #10) and a main daily run red on up.dotfiles-local* cases. Wrap each side's CLI invocation in a bounded retry (2 attempts) gated on isRetryableFailure, a predicate deliberately broader than isInfraError. Retrying is safe: a deterministic product failure reproduces on the next attempt and still goes red. The narrow isInfraError/classifyParitySide oracle-usability path is unchanged, so no real divergence is masked as a skip. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent a99f1d3 commit f58a494

2 files changed

Lines changed: 187 additions & 0 deletions

File tree

‎internal/cli/parity_matrix_test.go‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -518,7 +518,34 @@ func runParitySide(t *testing.T, ctx context.Context, repoRoot, cli string, tc p
518518
return result
519519
}
520520

521+
// parityInfraRetries bounds how many times a side is re-run when it fails with a
522+
// transient build/environment error. The runtime lane's docker builds flake on
523+
// contended CI runners (BuildKit "failed to solve", a registry hiccup, or a
524+
// container-setup failure surfaced as `Command failed: docker build`), which then
525+
// shows up as a false TS-vs-Go divergence (one side's build flaked while the other
526+
// succeeded). A single retry absorbs the flake. Retrying is safe: a deterministic
527+
// product failure reproduces on every attempt and still fails the case.
528+
const parityInfraRetries = 2 // total attempts, not extra retries
529+
521530
func runParityCLI(ctx context.Context, repoRoot, cli, cmdArgs string, env map[string]string) (stdout, stderr string, exitCode int) {
531+
return runWithInfraRetry(ctx, parityInfraRetries, func() (string, string, int) {
532+
return runParityCLIOnce(ctx, repoRoot, cli, cmdArgs, env)
533+
})
534+
}
535+
536+
// runWithInfraRetry re-runs run while it fails with a retryable transient error,
537+
// up to maxAttempts total. It stops early on success, on a non-retryable failure,
538+
// or once the context is done (so a retry never runs past the per-case deadline).
539+
func runWithInfraRetry(ctx context.Context, maxAttempts int, run func() (stdout, stderr string, exitCode int)) (stdout, stderr string, exitCode int) {
540+
for attempt := 1; ; attempt++ {
541+
stdout, stderr, exitCode = run()
542+
if exitCode == 0 || attempt >= maxAttempts || !isRetryableFailure(stdout, stderr) || ctx.Err() != nil {
543+
return
544+
}
545+
}
546+
}
547+
548+
func runParityCLIOnce(ctx context.Context, repoRoot, cli, cmdArgs string, env map[string]string) (stdout, stderr string, exitCode int) {
522549
cmd := exec.CommandContext(ctx, "/bin/sh", "-lc", cli+" "+cmdArgs)
523550
cmd.Dir = repoRoot
524551
cmd.Env = append(os.Environ(), envList(env)...)
@@ -1211,6 +1238,32 @@ func isInfraError(stdout, stderr string) bool {
12111238
return false
12121239
}
12131240

1241+
// isRetryableFailure reports whether a non-zero CLI result looks like a transient
1242+
// build/environment failure worth re-running, as opposed to a stable product
1243+
// divergence. It is deliberately BROADER than isInfraError: a retry is safe (a
1244+
// deterministic failure reproduces on the next attempt and still goes RED), so it
1245+
// also matches the container-setup / `docker build` wrapper a flaky BuildKit or
1246+
// registry hiccup surfaces without a recognizable low-level signal. Do NOT use it
1247+
// to mark a side an unusable oracle — classifyParitySide/isInfraError must stay
1248+
// narrow so a real divergence is never silently skipped; use this only to decide a
1249+
// retry.
1250+
func isRetryableFailure(stdout, stderr string) bool {
1251+
if isInfraError(stdout, stderr) {
1252+
return true
1253+
}
1254+
combined := strings.ToLower(stdout + stderr)
1255+
retryablePatterns := []string{
1256+
"an error occurred setting up the container",
1257+
"command failed: docker build",
1258+
}
1259+
for _, p := range retryablePatterns {
1260+
if strings.Contains(combined, p) {
1261+
return true
1262+
}
1263+
}
1264+
return false
1265+
}
1266+
12141267
type paritySideStatus struct {
12151268
Skip bool
12161269
Timeout bool

‎internal/cli/parity_retry_test.go‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
package cli
2+
3+
import (
4+
"context"
5+
"testing"
6+
)
7+
8+
// TestIsRetryableFailure guards the classifier that decides whether a non-zero
9+
// parity side is a transient build/environment flake (retryable) or a stable
10+
// product divergence (not). The two positive fixtures are the exact signatures
11+
// that turned dependabot PRs #9/#10 red: a BuildKit "failed to solve" on one
12+
// side and the reference's container-setup wrapper on the other, while the other
13+
// side succeeded. The negative fixtures must stay non-retryable so a real CLI
14+
// contract failure is never masked by a retry.
15+
func TestIsRetryableFailure(t *testing.T) {
16+
cases := []struct {
17+
name string
18+
stdout string
19+
stderr string
20+
wantRetryable bool
21+
}{
22+
{
23+
name: "buildkit failed to solve (isInfraError)",
24+
stderr: "ERROR: failed to solve: rpc error: code = Unknown desc = failed to fetch",
25+
wantRetryable: true,
26+
},
27+
{
28+
name: "docker daemon unavailable (isInfraError)",
29+
stderr: "Cannot connect to the Docker daemon at unix:///var/run/docker.sock",
30+
wantRetryable: true,
31+
},
32+
{
33+
// PR #10 / main 07-14 signature: TS reference build flaked and the
34+
// low-level BuildKit error is hidden behind the generic wrapper.
35+
name: "reference container-setup wrapper",
36+
stdout: `{"description":"An error occurred setting up the container.","message":"Command failed: docker build -f /tmp/x -t vsc-parity-dotfiles --platform linux/amd64 /tmp/x","outcome":"error"}`,
37+
wantRetryable: true,
38+
},
39+
{
40+
name: "config not found is a stable contract error",
41+
stdout: `{"outcome":"error","message":"Dev container config (path/devcontainer.json) not found."}`,
42+
wantRetryable: false,
43+
},
44+
{
45+
name: "flag validation is a stable contract error",
46+
stderr: "Unknown argument: --bogus-flag",
47+
wantRetryable: false,
48+
},
49+
{
50+
name: "empty output is not retryable",
51+
wantRetryable: false,
52+
},
53+
}
54+
for _, tc := range cases {
55+
t.Run(tc.name, func(t *testing.T) {
56+
if got := isRetryableFailure(tc.stdout, tc.stderr); got != tc.wantRetryable {
57+
t.Errorf("isRetryableFailure(%q, %q) = %v, want %v", tc.stdout, tc.stderr, got, tc.wantRetryable)
58+
}
59+
})
60+
}
61+
}
62+
63+
// TestRunWithInfraRetry asserts the retry loop absorbs a transient build flake
64+
// (success on the second attempt) while never masking a stable failure: a
65+
// deterministic contract error runs exactly once, and a persistent transient
66+
// error is retried up to the cap and then surfaced as the final non-zero result.
67+
func TestRunWithInfraRetry(t *testing.T) {
68+
const transient = "ERROR: failed to solve: connection reset by peer"
69+
const stable = `{"outcome":"error","message":"Dev container config not found."}`
70+
71+
t.Run("transient failure then success is absorbed", func(t *testing.T) {
72+
calls := 0
73+
_, _, exit := runWithInfraRetry(context.Background(), parityInfraRetries, func() (string, string, int) {
74+
calls++
75+
if calls == 1 {
76+
return "", transient, 1
77+
}
78+
return "ok", "", 0
79+
})
80+
if exit != 0 {
81+
t.Errorf("exit = %d, want 0 (flake should be absorbed)", exit)
82+
}
83+
if calls != 2 {
84+
t.Errorf("calls = %d, want 2 (one retry)", calls)
85+
}
86+
})
87+
88+
t.Run("stable contract failure is not retried", func(t *testing.T) {
89+
calls := 0
90+
_, _, exit := runWithInfraRetry(context.Background(), parityInfraRetries, func() (string, string, int) {
91+
calls++
92+
return stable, "", 1
93+
})
94+
if exit != 1 {
95+
t.Errorf("exit = %d, want 1", exit)
96+
}
97+
if calls != 1 {
98+
t.Errorf("calls = %d, want 1 (no retry for a stable contract error)", calls)
99+
}
100+
})
101+
102+
t.Run("persistent transient failure surfaces after the cap", func(t *testing.T) {
103+
calls := 0
104+
_, stderr, exit := runWithInfraRetry(context.Background(), parityInfraRetries, func() (string, string, int) {
105+
calls++
106+
return "", transient, 1
107+
})
108+
if exit != 1 {
109+
t.Errorf("exit = %d, want 1 (real failure must still go red)", exit)
110+
}
111+
if calls != parityInfraRetries {
112+
t.Errorf("calls = %d, want %d (retry cap)", calls, parityInfraRetries)
113+
}
114+
if stderr != transient {
115+
t.Errorf("stderr = %q, want the final attempt's output", stderr)
116+
}
117+
})
118+
119+
t.Run("cancelled context stops retries", func(t *testing.T) {
120+
ctx, cancel := context.WithCancel(context.Background())
121+
cancel()
122+
calls := 0
123+
_, _, exit := runWithInfraRetry(ctx, parityInfraRetries, func() (string, string, int) {
124+
calls++
125+
return "", transient, 1
126+
})
127+
if exit != 1 {
128+
t.Errorf("exit = %d, want 1", exit)
129+
}
130+
if calls != 1 {
131+
t.Errorf("calls = %d, want 1 (no retry once the deadline passed)", calls)
132+
}
133+
})
134+
}

0 commit comments

Comments
 (0)