From e4a5a42fde5f5a36e589f0c1c8e21f3a0e323df2 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 15 Nov 2026 14:19:00 -0600 Subject: [PATCH 1/4] feat(telemetry): report the slow phase when the analytics flush times out --- pkg/telemetry/analytics/client.go | 17 ++- pkg/telemetry/analytics/client_test.go | 5 +- pkg/telemetry/analytics/trace.go | 168 +++++++++++++++++++++++++ pkg/telemetry/analytics/trace_test.go | 107 ++++++++++++++++ 4 files changed, 290 insertions(+), 7 deletions(-) create mode 100644 pkg/telemetry/analytics/trace.go create mode 100644 pkg/telemetry/analytics/trace_test.go diff --git a/pkg/telemetry/analytics/client.go b/pkg/telemetry/analytics/client.go index a2843b5a3..b11935d0a 100644 --- a/pkg/telemetry/analytics/client.go +++ b/pkg/telemetry/analytics/client.go @@ -27,23 +27,26 @@ func NewClient() Client { return NewNoopClient() } - phClient, err := posthog.NewWithConfig(posthogAPIKey, posthogConfig(posthogEndpoint)) + cfg, trace := posthogConfig(posthogEndpoint) + phClient, err := posthog.NewWithConfig(posthogAPIKey, cfg) if err != nil { log.Debugf("failed to initialize analytics client: %v", err) return NewNoopClient() } - return &client{phClient: phClient, flushTimeout: flushTimeout} + return &client{phClient: phClient, flushTimeout: flushTimeout, trace: trace} } -func posthogConfig(endpoint string) posthog.Config { +func posthogConfig(endpoint string) (posthog.Config, *phaseTrace) { + transport, trace := newTracingTransport() return posthog.Config{ Endpoint: endpoint, DisableGeoIP: new(false), + Transport: transport, // Bounds Close as well, so no shutdown path can block the CLI. ShutdownTimeout: flushTimeout, Logger: quietLogger{}, - } + }, trace } // quietLogger keeps the SDK off stderr, which machine consumers parse. @@ -57,6 +60,7 @@ func (quietLogger) Errorf(format string, args ...any) { log.Debugf("posthog: "+f type client struct { phClient posthog.Client flushTimeout time.Duration + trace *phaseTrace closeOnce sync.Once } @@ -130,7 +134,10 @@ func (c *client) Flush() { switch err := c.phClient.CloseWithContext(ctx); { case err == nil: case errors.Is(err, context.DeadlineExceeded): - log.Debugf("analytics flush timed out after %s; dropping queued events", c.flushTimeout) + log.Debugf( + "analytics flush timed out after %s; dropping queued events (%s)", + c.flushTimeout, c.trace.summary(), + ) default: log.Debugf("error flushing analytics client: %v", err) } diff --git a/pkg/telemetry/analytics/client_test.go b/pkg/telemetry/analytics/client_test.go index 4c31fd3c5..5dd62130b 100644 --- a/pkg/telemetry/analytics/client_test.go +++ b/pkg/telemetry/analytics/client_test.go @@ -76,11 +76,12 @@ func TestBuildProperties_ExcludesReservedKeys(t *testing.T) { func newTestClient(t *testing.T, endpoint string, timeout time.Duration) *client { t.Helper() - ph, err := posthog.NewWithConfig("test-key", posthogConfig(endpoint)) + cfg, trace := posthogConfig(endpoint) + ph, err := posthog.NewWithConfig("test-key", cfg) if err != nil { t.Fatal(err) } - return &client{phClient: ph, flushTimeout: timeout} + return &client{phClient: ph, flushTimeout: timeout, trace: trace} } func testEvent() Event { diff --git a/pkg/telemetry/analytics/trace.go b/pkg/telemetry/analytics/trace.go new file mode 100644 index 000000000..d05bf700e --- /dev/null +++ b/pkg/telemetry/analytics/trace.go @@ -0,0 +1,168 @@ +package analytics + +import ( + "crypto/tls" + "net/http" + "net/http/httptrace" + "strconv" + "strings" + "sync" + "time" + + "github.com/posthog/posthog-go" +) + +// Phases of one upload, in the order the HTTP transport reaches them. +const ( + phaseDNS = "dns" + phaseConnect = "connect" + phaseTLS = "tls" + phaseSend = "send" + phaseResponse = "response" +) + +var phaseOrder = []string{phaseDNS, phaseConnect, phaseTLS, phaseSend, phaseResponse} + +// phaseTrace records where upload requests spend time, so a flush that hits +// its deadline can say which step was slow. +type phaseTrace struct { + mu sync.Mutex + attempts int + open map[string]openPhase + done []donePhase +} + +type openPhase struct { + start time.Time + count int +} + +type donePhase struct { + name string + duration time.Duration + err error +} + +func (p *phaseTrace) newAttempt() { + p.mu.Lock() + defer p.mu.Unlock() + p.attempts++ + p.open = map[string]openPhase{} + p.done = nil +} + +func (p *phaseTrace) begin(name string) { + p.mu.Lock() + defer p.mu.Unlock() + entry := p.open[name] + if entry.count == 0 { + entry.start = time.Now() + } + entry.count++ + p.open[name] = entry +} + +func (p *phaseTrace) end(name string, err error) { + p.mu.Lock() + defer p.mu.Unlock() + p.endLocked(name, err) +} + +func (p *phaseTrace) endLocked(name string, err error) { + entry, ok := p.open[name] + if !ok { + return + } + entry.count-- + if entry.count > 0 && err == nil { + p.open[name] = entry + return + } + delete(p.open, name) + p.done = append(p.done, donePhase{name: name, duration: time.Since(entry.start), err: err}) +} + +// fail closes every phase still running when a request ends in error. The SDK +// cancels the request at the flush deadline, so this keeps the phase that was +// in flight visible after the cancellation. +func (p *phaseTrace) fail(err error) { + p.mu.Lock() + defer p.mu.Unlock() + for _, name := range phaseOrder { + p.endLocked(name, err) + } +} + +// summary describes the latest upload attempt. It is safe to call on a nil +// receiver and while requests are still running. +func (p *phaseTrace) summary() string { + if p == nil { + return "" + } + p.mu.Lock() + defer p.mu.Unlock() + + if p.attempts == 0 { + return "no upload started" + } + var parts []string + for _, d := range p.done { + part := d.name + " " + d.duration.Round(time.Millisecond).String() + if d.err != nil { + part += " (" + d.err.Error() + ")" + } + parts = append(parts, part) + } + now := time.Now() + for _, name := range phaseOrder { + if entry, ok := p.open[name]; ok { + parts = append(parts, "stuck in "+name+" for "+ + now.Sub(entry.start).Round(time.Millisecond).String()) + } + } + if len(parts) == 0 { + parts = append(parts, "no phase reached") + } + return "upload attempt " + strconv.Itoa(p.attempts) + ": " + strings.Join(parts, ", ") +} + +func (p *phaseTrace) clientTrace() *httptrace.ClientTrace { + return &httptrace.ClientTrace{ + DNSStart: func(httptrace.DNSStartInfo) { p.begin(phaseDNS) }, + DNSDone: func(info httptrace.DNSDoneInfo) { p.end(phaseDNS, info.Err) }, + ConnectStart: func(_, _ string) { p.begin(phaseConnect) }, + ConnectDone: func(_, _ string, err error) { p.end(phaseConnect, err) }, + TLSHandshakeStart: func() { p.begin(phaseTLS) }, + TLSHandshakeDone: func(_ tls.ConnectionState, err error) { p.end(phaseTLS, err) }, + // A reused connection skips DNS, connect and TLS and lands here. + GotConn: func(httptrace.GotConnInfo) { p.begin(phaseSend) }, + WroteRequest: func(httptrace.WroteRequestInfo) { p.end(phaseSend, nil); p.begin(phaseResponse) }, + GotFirstResponseByte: func() { p.end(phaseResponse, nil) }, + } +} + +// tracingTransport reports phase timings of each request to a phaseTrace. +type tracingTransport struct { + base http.RoundTripper + trace *phaseTrace +} + +func (t *tracingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + t.trace.newAttempt() + ctx := httptrace.WithClientTrace(req.Context(), t.trace.clientTrace()) + resp, err := t.base.RoundTrip(req.WithContext(ctx)) + if err != nil { + t.trace.fail(err) + } + return resp, err +} + +// newTracingTransport mirrors the SDK's default transport, which a custom +// Config.Transport replaces. +func newTracingTransport() (http.RoundTripper, *phaseTrace) { + base := http.DefaultTransport.(*http.Transport).Clone() + base.MaxIdleConns = posthog.DefaultIdleConns + base.MaxIdleConnsPerHost = posthog.DefaultIdleConnsPerHost + trace := &phaseTrace{} + return &tracingTransport{base: base, trace: trace}, trace +} diff --git a/pkg/telemetry/analytics/trace_test.go b/pkg/telemetry/analytics/trace_test.go new file mode 100644 index 000000000..cb18092da --- /dev/null +++ b/pkg/telemetry/analytics/trace_test.go @@ -0,0 +1,107 @@ +package analytics + +import ( + "errors" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestPhaseTraceSummary_NoUpload(t *testing.T) { + if got := (&phaseTrace{}).summary(); got != "no upload started" { + t.Fatalf("summary = %q", got) + } + var nilTrace *phaseTrace + if got := nilTrace.summary(); got != "" { + t.Fatalf("nil summary = %q", got) + } +} + +func TestPhaseTraceSummary_ReportsOpenAndFailedPhases(t *testing.T) { + trace := &phaseTrace{} + trace.newAttempt() + trace.begin(phaseDNS) + trace.end(phaseDNS, nil) + trace.begin(phaseConnect) + + got := trace.summary() + for _, want := range []string{"upload attempt 1", "dns ", "stuck in connect"} { + if !strings.Contains(got, want) { + t.Errorf("summary %q lacks %q", got, want) + } + } + + trace.fail(errors.New("context canceled")) + got = trace.summary() + if !strings.Contains(got, "connect") || !strings.Contains(got, "(context canceled)") { + t.Errorf("summary after failure = %q", got) + } + if strings.Contains(got, "stuck in") { + t.Errorf("failed phase still reported as stuck: %q", got) + } +} + +func TestPhaseTraceSummary_RestartsOnNewAttempt(t *testing.T) { + trace := &phaseTrace{} + trace.newAttempt() + trace.begin(phaseTLS) + trace.newAttempt() + + got := trace.summary() + if !strings.Contains(got, "upload attempt 2") || strings.Contains(got, "tls") { + t.Fatalf("summary = %q", got) + } +} + +func TestFlushTimeout_ReportsStuckTLSHandshake(t *testing.T) { + // Accepts the connection but never answers the TLS ClientHello. + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + t.Cleanup(func() { _ = conn.Close() }) + } + }() + + c := newTestClient(t, "https://"+listener.Addr().String(), 300*time.Millisecond) + c.RecordEvent(testEvent()) + c.Flush() + + got := c.trace.summary() + // Whether the SDK's cancellation has reached the transport by now varies, + // so the stuck phase reads either as "stuck in tls" or as an interrupted "tls". + if !strings.Contains(got, "connect ") || !strings.Contains(got, "tls ") { + t.Fatalf("summary = %q, want completed connect and a tls phase", got) + } +} + +func TestFlushTimeout_ReportsStuckResponse(t *testing.T) { + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + case <-release: + } + })) + t.Cleanup(srv.Close) + t.Cleanup(func() { close(release) }) + + c := newTestClient(t, srv.URL, 300*time.Millisecond) + c.RecordEvent(testEvent()) + c.Flush() + + got := c.trace.summary() + if !strings.Contains(got, "send ") || !strings.Contains(got, "response ") { + t.Fatalf("summary = %q, want sent request and a response phase", got) + } +} From 4787b303c7d1c5fe88ecdb9d60fbab2131bb12ab Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 15 Nov 2026 15:13:00 -0600 Subject: [PATCH 2/4] fix(telemetry): keep analytics trace state per request and per dial --- pkg/telemetry/analytics/trace.go | 165 ++++++++++++++++---------- pkg/telemetry/analytics/trace_test.go | 117 ++++++++++++++++-- 2 files changed, 212 insertions(+), 70 deletions(-) diff --git a/pkg/telemetry/analytics/trace.go b/pkg/telemetry/analytics/trace.go index d05bf700e..d61458c3a 100644 --- a/pkg/telemetry/analytics/trace.go +++ b/pkg/telemetry/analytics/trace.go @@ -24,17 +24,51 @@ const ( var phaseOrder = []string{phaseDNS, phaseConnect, phaseTLS, phaseSend, phaseResponse} // phaseTrace records where upload requests spend time, so a flush that hits -// its deadline can say which step was slow. +// its deadline can say which step was slow. Each request owns its own attempt, +// because the SDK uploads batches concurrently; the trace only publishes the +// latest one for the summary. type phaseTrace struct { mu sync.Mutex attempts int - open map[string]openPhase - done []donePhase + latest *attempt +} + +func (p *phaseTrace) newAttempt() *attempt { + p.mu.Lock() + defer p.mu.Unlock() + p.attempts++ + p.latest = &attempt{number: p.attempts, open: map[string]openPhase{}} + return p.latest +} + +// summary describes the latest upload attempt. It is safe to call on a nil +// receiver and while requests are still running. +func (p *phaseTrace) summary() string { + if p == nil { + return "" + } + p.mu.Lock() + latest := p.latest + p.mu.Unlock() + + if latest == nil { + return "no upload started" + } + return latest.summary() +} + +// attempt is the phase state of a single request. +type attempt struct { + mu sync.Mutex + number int + open map[string]openPhase + done []donePhase } type openPhase struct { start time.Time count int + err error } type donePhase struct { @@ -43,70 +77,68 @@ type donePhase struct { err error } -func (p *phaseTrace) newAttempt() { - p.mu.Lock() - defer p.mu.Unlock() - p.attempts++ - p.open = map[string]openPhase{} - p.done = nil -} - -func (p *phaseTrace) begin(name string) { - p.mu.Lock() - defer p.mu.Unlock() - entry := p.open[name] +func (a *attempt) begin(name string) { + a.mu.Lock() + defer a.mu.Unlock() + entry := a.open[name] if entry.count == 0 { entry.start = time.Now() } entry.count++ - p.open[name] = entry -} - -func (p *phaseTrace) end(name string, err error) { - p.mu.Lock() - defer p.mu.Unlock() - p.endLocked(name, err) + a.open[name] = entry } -func (p *phaseTrace) endLocked(name string, err error) { - entry, ok := p.open[name] +// end closes a phase. Dials can overlap (dual-stack addresses), so a failed +// one only closes the phase after every other dial has finished. The first +// success closes it at once and later results are ignored. +func (a *attempt) end(name string, err error) { + a.mu.Lock() + defer a.mu.Unlock() + entry, ok := a.open[name] if !ok { return } entry.count-- - if entry.count > 0 && err == nil { - p.open[name] = entry + if err != nil && entry.err == nil { + entry.err = err + } + if err != nil && entry.count > 0 { + a.open[name] = entry return } - delete(p.open, name) - p.done = append(p.done, donePhase{name: name, duration: time.Since(entry.start), err: err}) + if err == nil { + entry.err = nil + } + a.closeLocked(name, entry) } -// fail closes every phase still running when a request ends in error. The SDK -// cancels the request at the flush deadline, so this keeps the phase that was -// in flight visible after the cancellation. -func (p *phaseTrace) fail(err error) { - p.mu.Lock() - defer p.mu.Unlock() +// fail closes every phase still running when the request ends in error. The +// SDK cancels the request at the flush deadline, so this keeps the phase that +// was in flight visible after the cancellation. +func (a *attempt) fail(err error) { + a.mu.Lock() + defer a.mu.Unlock() for _, name := range phaseOrder { - p.endLocked(name, err) + if entry, ok := a.open[name]; ok { + entry.err = err + a.closeLocked(name, entry) + } } } -// summary describes the latest upload attempt. It is safe to call on a nil -// receiver and while requests are still running. -func (p *phaseTrace) summary() string { - if p == nil { - return "" - } - p.mu.Lock() - defer p.mu.Unlock() +func (a *attempt) closeLocked(name string, entry openPhase) { + delete(a.open, name) + a.done = append(a.done, donePhase{ + name: name, duration: time.Since(entry.start), err: entry.err, + }) +} + +func (a *attempt) summary() string { + a.mu.Lock() + defer a.mu.Unlock() - if p.attempts == 0 { - return "no upload started" - } var parts []string - for _, d := range p.done { + for _, d := range a.done { part := d.name + " " + d.duration.Round(time.Millisecond).String() if d.err != nil { part += " (" + d.err.Error() + ")" @@ -115,7 +147,7 @@ func (p *phaseTrace) summary() string { } now := time.Now() for _, name := range phaseOrder { - if entry, ok := p.open[name]; ok { + if entry, ok := a.open[name]; ok { parts = append(parts, "stuck in "+name+" for "+ now.Sub(entry.start).Round(time.Millisecond).String()) } @@ -123,21 +155,28 @@ func (p *phaseTrace) summary() string { if len(parts) == 0 { parts = append(parts, "no phase reached") } - return "upload attempt " + strconv.Itoa(p.attempts) + ": " + strings.Join(parts, ", ") + return "upload attempt " + strconv.Itoa(a.number) + ": " + strings.Join(parts, ", ") } -func (p *phaseTrace) clientTrace() *httptrace.ClientTrace { +func (a *attempt) clientTrace() *httptrace.ClientTrace { return &httptrace.ClientTrace{ - DNSStart: func(httptrace.DNSStartInfo) { p.begin(phaseDNS) }, - DNSDone: func(info httptrace.DNSDoneInfo) { p.end(phaseDNS, info.Err) }, - ConnectStart: func(_, _ string) { p.begin(phaseConnect) }, - ConnectDone: func(_, _ string, err error) { p.end(phaseConnect, err) }, - TLSHandshakeStart: func() { p.begin(phaseTLS) }, - TLSHandshakeDone: func(_ tls.ConnectionState, err error) { p.end(phaseTLS, err) }, + DNSStart: func(httptrace.DNSStartInfo) { a.begin(phaseDNS) }, + DNSDone: func(info httptrace.DNSDoneInfo) { a.end(phaseDNS, info.Err) }, + ConnectStart: func(_, _ string) { a.begin(phaseConnect) }, + ConnectDone: func(_, _ string, err error) { a.end(phaseConnect, err) }, + TLSHandshakeStart: func() { a.begin(phaseTLS) }, + TLSHandshakeDone: func(_ tls.ConnectionState, err error) { a.end(phaseTLS, err) }, // A reused connection skips DNS, connect and TLS and lands here. - GotConn: func(httptrace.GotConnInfo) { p.begin(phaseSend) }, - WroteRequest: func(httptrace.WroteRequestInfo) { p.end(phaseSend, nil); p.begin(phaseResponse) }, - GotFirstResponseByte: func() { p.end(phaseResponse, nil) }, + GotConn: func(httptrace.GotConnInfo) { a.begin(phaseSend) }, + WroteRequest: func(info httptrace.WroteRequestInfo) { + a.end(phaseSend, info.Err) + if info.Err == nil { + a.begin(phaseResponse) + } + }, + // The response phase stays open past the first byte: a server that + // sends part of the headers and stalls is still a stuck response. + // RoundTrip closes it. } } @@ -148,11 +187,13 @@ type tracingTransport struct { } func (t *tracingTransport) RoundTrip(req *http.Request) (*http.Response, error) { - t.trace.newAttempt() - ctx := httptrace.WithClientTrace(req.Context(), t.trace.clientTrace()) + a := t.trace.newAttempt() + ctx := httptrace.WithClientTrace(req.Context(), a.clientTrace()) resp, err := t.base.RoundTrip(req.WithContext(ctx)) if err != nil { - t.trace.fail(err) + a.fail(err) + } else { + a.end(phaseResponse, nil) } return resp, err } diff --git a/pkg/telemetry/analytics/trace_test.go b/pkg/telemetry/analytics/trace_test.go index cb18092da..ce18e0233 100644 --- a/pkg/telemetry/analytics/trace_test.go +++ b/pkg/telemetry/analytics/trace_test.go @@ -5,6 +5,8 @@ import ( "net" "net/http" "net/http/httptest" + "net/http/httptrace" + "regexp" "strings" "testing" "time" @@ -22,10 +24,10 @@ func TestPhaseTraceSummary_NoUpload(t *testing.T) { func TestPhaseTraceSummary_ReportsOpenAndFailedPhases(t *testing.T) { trace := &phaseTrace{} - trace.newAttempt() - trace.begin(phaseDNS) - trace.end(phaseDNS, nil) - trace.begin(phaseConnect) + a := trace.newAttempt() + a.begin(phaseDNS) + a.end(phaseDNS, nil) + a.begin(phaseConnect) got := trace.summary() for _, want := range []string{"upload attempt 1", "dns ", "stuck in connect"} { @@ -34,7 +36,7 @@ func TestPhaseTraceSummary_ReportsOpenAndFailedPhases(t *testing.T) { } } - trace.fail(errors.New("context canceled")) + a.fail(errors.New("context canceled")) got = trace.summary() if !strings.Contains(got, "connect") || !strings.Contains(got, "(context canceled)") { t.Errorf("summary after failure = %q", got) @@ -44,10 +46,10 @@ func TestPhaseTraceSummary_ReportsOpenAndFailedPhases(t *testing.T) { } } -func TestPhaseTraceSummary_RestartsOnNewAttempt(t *testing.T) { +func TestPhaseTraceSummary_ReportsLatestAttempt(t *testing.T) { trace := &phaseTrace{} - trace.newAttempt() - trace.begin(phaseTLS) + first := trace.newAttempt() + first.begin(phaseTLS) trace.newAttempt() got := trace.summary() @@ -56,6 +58,70 @@ func TestPhaseTraceSummary_RestartsOnNewAttempt(t *testing.T) { } } +func TestAttempts_DoNotShareState(t *testing.T) { + trace := &phaseTrace{} + first := trace.newAttempt() + first.begin(phaseResponse) + second := trace.newAttempt() + second.begin(phaseResponse) + + // Finishing the first request must not close the second one's wait. + first.end(phaseResponse, nil) + + got := trace.summary() + if !strings.Contains(got, "upload attempt 2") || !strings.Contains(got, "stuck in response") { + t.Fatalf("summary = %q, want the second attempt still waiting", got) + } +} + +func TestAttempt_FailedDialKeepsPhaseOpenWhileAnotherIsPending(t *testing.T) { + trace := &phaseTrace{} + a := trace.newAttempt() + a.begin(phaseConnect) + a.begin(phaseConnect) + + a.end(phaseConnect, errors.New("network unreachable")) + if got := trace.summary(); !strings.Contains(got, "stuck in connect") { + t.Fatalf("summary = %q, want connect still open for the pending dial", got) + } + + a.fail(errors.New("context canceled")) + if got := trace.summary(); !strings.Contains(got, "(context canceled)") { + t.Fatalf("summary = %q, want the cancellation on the pending dial", got) + } +} + +func TestAttempt_FirstSuccessfulDialClosesPhase(t *testing.T) { + trace := &phaseTrace{} + a := trace.newAttempt() + a.begin(phaseConnect) + a.begin(phaseConnect) + + a.end(phaseConnect, nil) + // The losing dial reports a cancellation after the winner connected. + a.end(phaseConnect, errors.New("operation was canceled")) + + got := trace.summary() + if strings.Contains(got, "stuck in") || strings.Contains(got, "canceled") { + t.Fatalf("summary = %q, want a clean connect", got) + } +} + +func TestAttempt_FailedWriteIsAttributedToSend(t *testing.T) { + trace := &phaseTrace{} + a := trace.newAttempt() + a.clientTrace().GotConn(httptrace.GotConnInfo{}) + a.clientTrace().WroteRequest(httptrace.WroteRequestInfo{Err: errors.New("broken pipe")}) + + got := trace.summary() + if !strings.Contains(got, "send ") || !strings.Contains(got, "(broken pipe)") { + t.Fatalf("summary = %q, want the write failure on send", got) + } + if strings.Contains(got, "response") { + t.Fatalf("summary = %q, response must not start after a failed write", got) + } +} + func TestFlushTimeout_ReportsStuckTLSHandshake(t *testing.T) { // Accepts the connection but never answers the TLS ClientHello. listener, err := net.Listen("tcp", "127.0.0.1:0") @@ -105,3 +171,38 @@ func TestFlushTimeout_ReportsStuckResponse(t *testing.T) { t.Fatalf("summary = %q, want sent request and a response phase", got) } } + +func TestFlushTimeout_ReportsResponseStalledAfterFirstByte(t *testing.T) { + // Sends part of the status line, then stalls. + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + t.Cleanup(func() { _ = conn.Close() }) + go func() { + buf := make([]byte, 4096) + _, _ = conn.Read(buf) + _, _ = conn.Write([]byte("HTTP/1.1 2")) + }() + } + }() + + c := newTestClient(t, "http://"+listener.Addr().String(), 300*time.Millisecond) + c.RecordEvent(testEvent()) + c.Flush() + + // Either still waiting or interrupted with an error, never a clean finish. + got := c.trace.summary() + if !stalledResponse.MatchString(got) { + t.Fatalf("summary = %q, want the stall reported in the response phase", got) + } +} + +var stalledResponse = regexp.MustCompile(`stuck in response|response \S+ \(`) From 9f2385f4942ae1496cd910abc608d573cf71d6a4 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 15 Nov 2026 15:45:49 -0600 Subject: [PATCH 3/4] fix(telemetry): keep the analytics response phase open until the body is read --- pkg/telemetry/analytics/trace.go | 35 ++++++++++++++++--- pkg/telemetry/analytics/trace_test.go | 50 ++++++++++++++++++++++----- 2 files changed, 72 insertions(+), 13 deletions(-) diff --git a/pkg/telemetry/analytics/trace.go b/pkg/telemetry/analytics/trace.go index d61458c3a..8d25eef02 100644 --- a/pkg/telemetry/analytics/trace.go +++ b/pkg/telemetry/analytics/trace.go @@ -2,6 +2,8 @@ package analytics import ( "crypto/tls" + "errors" + "io" "net/http" "net/http/httptrace" "strconv" @@ -176,7 +178,7 @@ func (a *attempt) clientTrace() *httptrace.ClientTrace { }, // The response phase stays open past the first byte: a server that // sends part of the headers and stalls is still a stuck response. - // RoundTrip closes it. + // The response body closes it. } } @@ -192,10 +194,35 @@ func (t *tracingTransport) RoundTrip(req *http.Request) (*http.Response, error) resp, err := t.base.RoundTrip(req.WithContext(ctx)) if err != nil { a.fail(err) - } else { - a.end(phaseResponse, nil) + return resp, err } - return resp, err + // RoundTrip returns once the headers are in; the body is read later. + resp.Body = &tracedBody{ReadCloser: resp.Body, attempt: a} + return resp, nil +} + +// tracedBody keeps the response phase open until the body is fully read, +// fails, or is closed, so a stalled body read is not reported as finished. +type tracedBody struct { + io.ReadCloser + attempt *attempt +} + +func (b *tracedBody) Read(p []byte) (int, error) { + n, err := b.ReadCloser.Read(p) + switch { + case errors.Is(err, io.EOF): + b.attempt.end(phaseResponse, nil) + case err != nil: + b.attempt.end(phaseResponse, err) + } + return n, err +} + +func (b *tracedBody) Close() error { + err := b.ReadCloser.Close() + b.attempt.end(phaseResponse, nil) + return err } // newTracingTransport mirrors the SDK's default transport, which a custom diff --git a/pkg/telemetry/analytics/trace_test.go b/pkg/telemetry/analytics/trace_test.go index ce18e0233..706442f75 100644 --- a/pkg/telemetry/analytics/trace_test.go +++ b/pkg/telemetry/analytics/trace_test.go @@ -172,8 +172,10 @@ func TestFlushTimeout_ReportsStuckResponse(t *testing.T) { } } -func TestFlushTimeout_ReportsResponseStalledAfterFirstByte(t *testing.T) { - // Sends part of the status line, then stalls. +// stallingServer answers each request with reply and then goes silent. It +// returns the address to upload to. +func stallingServer(t *testing.T, reply string) string { + t.Helper() listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) @@ -189,20 +191,50 @@ func TestFlushTimeout_ReportsResponseStalledAfterFirstByte(t *testing.T) { go func() { buf := make([]byte, 4096) _, _ = conn.Read(buf) - _, _ = conn.Write([]byte("HTTP/1.1 2")) + _, _ = conn.Write([]byte(reply)) }() } }() + return "http://" + listener.Addr().String() +} + +func TestFlushTimeout_ReportsResponseStalls(t *testing.T) { + tests := map[string]string{ + // Part of the status line, then silence. + "stalled after first byte": "HTTP/1.1 2", + // Complete headers, then a body that never finishes. The SDK reads + // the body of non-2xx replies, so this stalls after RoundTrip returns. + "stalled body": "HTTP/1.1 500 Internal Server Error\r\nContent-Length: 100\r\n\r\npartial", + } + for name, reply := range tests { + t.Run(name, func(t *testing.T) { + c := newTestClient(t, stallingServer(t, reply), 300*time.Millisecond) + c.RecordEvent(testEvent()) + c.Flush() + + // Either still waiting or interrupted with an error, never a clean finish. + got := c.trace.summary() + if !stalledResponse.MatchString(got) { + t.Fatalf("summary = %q, want the stall reported in the response phase", got) + } + }) + } +} + +var stalledResponse = regexp.MustCompile(`stuck in response|response \S+ \(`) + +func TestFlush_SuccessfulUploadLeavesNoOpenPhase(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"status":1}`)) + })) + t.Cleanup(srv.Close) - c := newTestClient(t, "http://"+listener.Addr().String(), 300*time.Millisecond) + c := newTestClient(t, srv.URL, 5*time.Second) c.RecordEvent(testEvent()) c.Flush() - // Either still waiting or interrupted with an error, never a clean finish. got := c.trace.summary() - if !stalledResponse.MatchString(got) { - t.Fatalf("summary = %q, want the stall reported in the response phase", got) + if strings.Contains(got, "stuck in") || strings.Contains(got, "(") { + t.Fatalf("summary = %q, want every phase completed cleanly", got) } } - -var stalledResponse = regexp.MustCompile(`stuck in response|response \S+ \(`) From 70390e651c075398bd3b6c8e1bcf8d19b250d7b2 Mon Sep 17 00:00:00 2001 From: Samuel K Date: Sun, 15 Nov 2026 15:48:26 -0600 Subject: [PATCH 4/4] fix(telemetry): ignore trace callbacks that arrive after a failed upload --- pkg/telemetry/analytics/trace.go | 7 +++++++ pkg/telemetry/analytics/trace_test.go | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/pkg/telemetry/analytics/trace.go b/pkg/telemetry/analytics/trace.go index 8d25eef02..a3579530b 100644 --- a/pkg/telemetry/analytics/trace.go +++ b/pkg/telemetry/analytics/trace.go @@ -63,6 +63,9 @@ func (p *phaseTrace) summary() string { type attempt struct { mu sync.Mutex number int + // failed stops late callbacks from adding phases: Go's detached dials can + // keep reporting after RoundTrip has returned an error. + failed bool open map[string]openPhase done []donePhase } @@ -82,6 +85,9 @@ type donePhase struct { func (a *attempt) begin(name string) { a.mu.Lock() defer a.mu.Unlock() + if a.failed { + return + } entry := a.open[name] if entry.count == 0 { entry.start = time.Now() @@ -120,6 +126,7 @@ func (a *attempt) end(name string, err error) { func (a *attempt) fail(err error) { a.mu.Lock() defer a.mu.Unlock() + a.failed = true for _, name := range phaseOrder { if entry, ok := a.open[name]; ok { entry.err = err diff --git a/pkg/telemetry/analytics/trace_test.go b/pkg/telemetry/analytics/trace_test.go index 706442f75..4630efbe7 100644 --- a/pkg/telemetry/analytics/trace_test.go +++ b/pkg/telemetry/analytics/trace_test.go @@ -107,6 +107,25 @@ func TestAttempt_FirstSuccessfulDialClosesPhase(t *testing.T) { } } +func TestAttempt_IgnoresPhasesStartedAfterFailure(t *testing.T) { + trace := &phaseTrace{} + a := trace.newAttempt() + a.begin(phaseDNS) + a.fail(errors.New("context canceled")) + + // A detached dial keeps going after RoundTrip returned its error. + a.begin(phaseConnect) + a.begin(phaseTLS) + + got := trace.summary() + if strings.Contains(got, "connect") || strings.Contains(got, "tls") { + t.Fatalf("summary = %q, want no phases started after the failure", got) + } + if !strings.Contains(got, "dns") || !strings.Contains(got, "(context canceled)") { + t.Fatalf("summary = %q, want the interrupted dns phase", got) + } +} + func TestAttempt_FailedWriteIsAttributedToSend(t *testing.T) { trace := &phaseTrace{} a := trace.newAttempt()