From a2a5057791fca471fb2ed0988dc1dfbaaf835388 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Thu, 22 Oct 2026 16:05:23 -0600 Subject: [PATCH 1/2] fix(ci): retry provider update tests once on Windows Go runtime crash --- hack/provider_update_ci/capture.go | 33 ++++++++++++--- hack/provider_update_ci/main.go | 7 +-- hack/provider_update_ci/retry.go | 68 ++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 hack/provider_update_ci/retry.go diff --git a/hack/provider_update_ci/capture.go b/hack/provider_update_ci/capture.go index 8b755d496a..1d5a0745d1 100644 --- a/hack/provider_update_ci/capture.go +++ b/hack/provider_update_ci/capture.go @@ -32,6 +32,7 @@ type outputCapture struct { states map[testID]testState listed map[testID]struct{} writeErr error + runtimeFatal bool } func newOutputCapture(log, stdout, stderr io.Writer) *outputCapture { @@ -91,9 +92,7 @@ func (c *outputCapture) writeArtifact(data []byte, stdout bool) { return } line := (*pending)[:newline+1] - if stdout { - c.readEvent(line[:newline]) - } + c.readLine(line[:newline], stdout) if err := writeAll(c.log, line); err != nil { c.recordError(fmt.Errorf("write artifact: %w", err)) } @@ -101,11 +100,21 @@ func (c *outputCapture) writeArtifact(data []byte, stdout bool) { } } +func (c *outputCapture) readLine(line []byte, stdout bool) { + if stdout { + c.readEvent(line) + return + } + c.noteRuntimeFatal(string(line)) +} + func (c *outputCapture) readEvent(line []byte) { var event testEvent if json.Unmarshal(bytes.TrimSpace(line), &event) != nil || event.Package == "" { + c.noteRuntimeFatal(string(line)) return } + c.noteRuntimeFatal(event.Output) if event.Test == "" { c.readListedTests(event.Package, event.Output) return @@ -157,9 +166,7 @@ func (c *outputCapture) finish() { if len(*stream.pending) == 0 { continue } - if stream.stdout { - c.readEvent(*stream.pending) - } + c.readLine(*stream.pending, stream.stdout) if err := writeAll(c.log, *stream.pending); err != nil { c.recordError(fmt.Errorf("write artifact: %w", err)) } @@ -167,6 +174,20 @@ func (c *outputCapture) finish() { } } +func (c *outputCapture) noteRuntimeFatal(output string) { + for line := range strings.SplitSeq(output, "\n") { + if isRuntimeFatalLine(strings.TrimSuffix(line, "\r")) { + c.runtimeFatal = true + } + } +} + +func (c *outputCapture) sawRuntimeFatal() bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.runtimeFatal +} + func (c *outputCapture) events() map[testID]testState { c.mu.Lock() defer c.mu.Unlock() diff --git a/hack/provider_update_ci/main.go b/hack/provider_update_ci/main.go index f3449a11b6..2a856c14ad 100644 --- a/hack/provider_update_ci/main.go +++ b/hack/provider_update_ci/main.go @@ -58,10 +58,11 @@ func runProviderUpdateTests(artifactsDir string, stdout, stderr io.Writer) int { expected, discoveryErr := discoverProviderTests(logFile, stdout, stderr) result := testRunResult{discoveryErr: discoveryErr} if discoveryErr == nil { - result = runGoTests(capture) + result, capture = runGoTestsWithRetry(logFile, capture, stdout, stderr) + } else { + capture.finish() + result.exitCode = executionExitCode(result) } - capture.finish() - result.exitCode = executionExitCode(result) result.captureErr = capture.err() exitWriteErr, logCloseErr := finishTestArtifacts(artifactsDir, logFile, result.exitCode) result.exitWriteErr = exitWriteErr diff --git a/hack/provider_update_ci/retry.go b/hack/provider_update_ci/retry.go new file mode 100644 index 0000000000..9e73737d0b --- /dev/null +++ b/hack/provider_update_ci/retry.go @@ -0,0 +1,68 @@ +package main + +import ( + "io" + "strings" +) + +const ( + maxTestAttempts = 2 + windowsAccessViolated = 0xc0000005 +) + +// Prefixes the Go runtime prints when the test process itself dies. The raw +// pointer panic value is the heap-corruption symptom seen in +// golang/go#81238; test code never panics with one. +var runtimeFatalPrefixes = []string{ + "fatal error:", + "Exception 0xc0000005", + "panic: (interface {}) 0x", +} + +// runGoTestsWithRetry reruns the tests once when the Go runtime itself crashed. +// Windows/amd64 AMX runners corrupt the heap when an exception frame overflows +// the goroutine stack reserve (golang/go#81238, unfixed). Remove this retry once +// a released Go toolchain contains the upstream fix. +func runGoTestsWithRetry( + logFile io.Writer, + capture *outputCapture, + stdout, stderr io.Writer, +) (testRunResult, *outputCapture) { + for attempt := 1; ; attempt++ { + result := runGoTests(capture) + capture.finish() + result.exitCode = executionExitCode(result) + if attempt == maxTestAttempts || !isRuntimeCrash(result, capture) { + return result, capture + } + reportf( + stderr, + "::warning::Go runtime crashed on Windows (golang/go#81238); "+ + "retrying provider update tests (attempt %d of %d)\n", + attempt+1, + maxTestAttempts, + ) + capture = newOutputCapture(logFile, stdout, stderr) + } +} + +func isRuntimeCrash(result testRunResult, capture *outputCapture) bool { + if result.exitCode == 0 || result.startErr != nil || capture.err() != nil { + return false + } + for _, state := range capture.events() { + if state.failed { + return false + } + } + return uint32(result.exitCode) == windowsAccessViolated || capture.sawRuntimeFatal() +} + +func isRuntimeFatalLine(line string) bool { + for _, prefix := range runtimeFatalPrefixes { + if strings.HasPrefix(line, prefix) { + return true + } + } + return false +} From 097587040e829cc39ba6dc4ada707427b51c05d2 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Thu, 22 Oct 2026 17:49:30 -0600 Subject: [PATCH 2/2] test(ci): cover provider update runtime-crash retry --- hack/provider_update_ci/main_test.go | 97 ++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/hack/provider_update_ci/main_test.go b/hack/provider_update_ci/main_test.go index 624ccb93ab..a5ae94ddc2 100644 --- a/hack/provider_update_ci/main_test.go +++ b/hack/provider_update_ci/main_test.go @@ -18,12 +18,16 @@ const ( fakeModeEnv = "DEVSY_PROVIDER_UPDATE_CI_FAKE_MODE" cliModeEnv = "DEVSY_PROVIDER_UPDATE_CI_RUN_CLI" fakeExitEnv = "DEVSY_PROVIDER_UPDATE_CI_FAKE_EXIT" + attemptsEnv = "DEVSY_PROVIDER_UPDATE_CI_ATTEMPTS_FILE" largeLineEnv = "DEVSY_PROVIDER_UPDATE_CI_LARGE_LINE" fakeGoBasename = "go" missingMode = "missing" skipMode = "skip" noPassMode = "no-pass" multipleMode = "multiple" + crashOnceMode = "crash-once" + crashTwiceMode = "crash-twice" + crashFailMode = "crash-after-failure" ) var fixtureTests = []testID{ @@ -45,6 +49,8 @@ var fakeCaseExceptions = map[string]map[int][]string{ }, } +var fakeCrashCounts = map[string]int{crashOnceMode: 1, crashTwiceMode: 2, crashFailMode: 1} + func TestMain(m *testing.M) { if isFakeGoInvocation(os.Args[0]) { os.Exit(runFakeGo()) @@ -63,11 +69,42 @@ func runFakeGo() int { if os.Args[len(os.Args)-2] == "-list" { return emitFakeDiscovery(os.Getenv(fakeModeEnv)) } + attempt := recordFakeAttempt() + if attempt > 0 && attempt <= fakeCrashCounts[os.Getenv(fakeModeEnv)] { + emitFakeCrash(os.Getenv(fakeModeEnv)) + return 2 + } emitFakeTestEvents(os.Getenv(fakeModeEnv)) reportf(os.Stderr, "fake Go stderr\n") return fakeGoExitCode() } +func recordFakeAttempt() int { + path := os.Getenv(attemptsEnv) + if path == "" { + return 0 + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return 0 + } + defer func() { _ = file.Close() }() + _, _ = file.WriteString("x") + info, err := file.Stat() + if err != nil { + return 0 + } + return int(info.Size()) +} + +func emitFakeCrash(mode string) { + writeFakeEvent("run", fixtureTests[0], "") + writeFakeEvent("output", fixtureTests[0], "fatal error: found pointer to free object\n") + if mode == crashFailMode { + writeFakeEvent("fail", fixtureTests[0], "") + } +} + func isFakeGoInvocation(path string) bool { base := filepath.Base(path) return base == fakeGoBasename || base == fakeGoBasename+".exe" @@ -312,6 +349,66 @@ func TestDefaultArtifactsDirectoryUsesRunnerTemp(t *testing.T) { } } +func TestCLIRetriesRuntimeCrashOnce(t *testing.T) { + attempts := setUpFakeGoWithAttempts(t, crashOnceMode, "0") + _, stderr, code := runCLIProcess(t, "--artifacts-dir", t.TempDir()) + if code != 0 { + t.Fatalf("CLI failed after a recoverable runtime crash: %q", stderr) + } + if got := countAttempts(t, attempts); got != 2 { + t.Fatalf("attempts = %d, want 2", got) + } + if !strings.Contains(stderr, "::warning::") || !strings.Contains(stderr, "attempt 2 of 2") { + t.Fatalf("retry warning missing: %q", stderr) + } +} + +func TestCLIFailsWhenRuntimeCrashRepeats(t *testing.T) { + attempts := setUpFakeGoWithAttempts(t, crashTwiceMode, "0") + _, _, code := runCLIProcess(t, "--artifacts-dir", t.TempDir()) + if code != 2 { + t.Fatalf("CLI exit code = %d, want 2", code) + } + if got := countAttempts(t, attempts); got != 2 { + t.Fatalf("attempts = %d, want 2", got) + } +} + +func TestCLIDoesNotRetryRealFailures(t *testing.T) { + for _, mode := range []string{crashFailMode, "success"} { + t.Run(mode, func(t *testing.T) { + attempts := setUpFakeGoWithAttempts(t, mode, "2") + _, stderr, code := runCLIProcess(t, "--artifacts-dir", t.TempDir()) + if code != 2 { + t.Fatalf("CLI exit code = %d, want 2", code) + } + if got := countAttempts(t, attempts); got != 1 { + t.Fatalf("attempts = %d, want 1", got) + } + if strings.Contains(stderr, "::warning::") { + t.Fatalf("unexpected retry warning: %q", stderr) + } + }) + } +} + +func setUpFakeGoWithAttempts(t *testing.T, mode, exitCode string) string { + t.Helper() + setUpFakeGo(t, mode, exitCode) + attempts := filepath.Join(t.TempDir(), "attempts") + t.Setenv(attemptsEnv, attempts) + return attempts +} + +func countAttempts(t *testing.T, path string) int { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return len(data) +} + func setUpFakeGo(t *testing.T, mode, exitCode string) { t.Helper() dir := t.TempDir()