diff --git a/pkg/telemetry/analytics/client.go b/pkg/telemetry/analytics/client.go index a2843b5a3..b11935d0a 100644 --- a/pkg/telemetry/analytics/client.go +++ b/pkg/telemetry/analytics/client.go @@ -27,23 +27,26 @@ func NewClient() Client { return NewNoopClient() } - phClient, err := posthog.NewWithConfig(posthogAPIKey, posthogConfig(posthogEndpoint)) + cfg, trace := posthogConfig(posthogEndpoint) + phClient, err := posthog.NewWithConfig(posthogAPIKey, cfg) if err != nil { log.Debugf("failed to initialize analytics client: %v", err) return NewNoopClient() } - return &client{phClient: phClient, flushTimeout: flushTimeout} + return &client{phClient: phClient, flushTimeout: flushTimeout, trace: trace} } -func posthogConfig(endpoint string) posthog.Config { +func posthogConfig(endpoint string) (posthog.Config, *phaseTrace) { + transport, trace := newTracingTransport() return posthog.Config{ Endpoint: endpoint, DisableGeoIP: new(false), + Transport: transport, // Bounds Close as well, so no shutdown path can block the CLI. ShutdownTimeout: flushTimeout, Logger: quietLogger{}, - } + }, trace } // quietLogger keeps the SDK off stderr, which machine consumers parse. @@ -57,6 +60,7 @@ func (quietLogger) Errorf(format string, args ...any) { log.Debugf("posthog: "+f type client struct { phClient posthog.Client flushTimeout time.Duration + trace *phaseTrace closeOnce sync.Once } @@ -130,7 +134,10 @@ func (c *client) Flush() { switch err := c.phClient.CloseWithContext(ctx); { case err == nil: case errors.Is(err, context.DeadlineExceeded): - log.Debugf("analytics flush timed out after %s; dropping queued events", c.flushTimeout) + log.Debugf( + "analytics flush timed out after %s; dropping queued events (%s)", + c.flushTimeout, c.trace.summary(), + ) default: log.Debugf("error flushing analytics client: %v", err) } diff --git a/pkg/telemetry/analytics/client_test.go b/pkg/telemetry/analytics/client_test.go index 4c31fd3c5..5dd62130b 100644 --- a/pkg/telemetry/analytics/client_test.go +++ b/pkg/telemetry/analytics/client_test.go @@ -76,11 +76,12 @@ func TestBuildProperties_ExcludesReservedKeys(t *testing.T) { func newTestClient(t *testing.T, endpoint string, timeout time.Duration) *client { t.Helper() - ph, err := posthog.NewWithConfig("test-key", posthogConfig(endpoint)) + cfg, trace := posthogConfig(endpoint) + ph, err := posthog.NewWithConfig("test-key", cfg) if err != nil { t.Fatal(err) } - return &client{phClient: ph, flushTimeout: timeout} + return &client{phClient: ph, flushTimeout: timeout, trace: trace} } func testEvent() Event { diff --git a/pkg/telemetry/analytics/trace.go b/pkg/telemetry/analytics/trace.go new file mode 100644 index 000000000..a3579530b --- /dev/null +++ b/pkg/telemetry/analytics/trace.go @@ -0,0 +1,243 @@ +package analytics + +import ( + "crypto/tls" + "errors" + "io" + "net/http" + "net/http/httptrace" + "strconv" + "strings" + "sync" + "time" + + "github.com/posthog/posthog-go" +) + +// Phases of one upload, in the order the HTTP transport reaches them. +const ( + phaseDNS = "dns" + phaseConnect = "connect" + phaseTLS = "tls" + phaseSend = "send" + phaseResponse = "response" +) + +var phaseOrder = []string{phaseDNS, phaseConnect, phaseTLS, phaseSend, phaseResponse} + +// phaseTrace records where upload requests spend time, so a flush that hits +// its deadline can say which step was slow. Each request owns its own attempt, +// because the SDK uploads batches concurrently; the trace only publishes the +// latest one for the summary. +type phaseTrace struct { + mu sync.Mutex + attempts int + latest *attempt +} + +func (p *phaseTrace) newAttempt() *attempt { + p.mu.Lock() + defer p.mu.Unlock() + p.attempts++ + p.latest = &attempt{number: p.attempts, open: map[string]openPhase{}} + return p.latest +} + +// summary describes the latest upload attempt. It is safe to call on a nil +// receiver and while requests are still running. +func (p *phaseTrace) summary() string { + if p == nil { + return "" + } + p.mu.Lock() + latest := p.latest + p.mu.Unlock() + + if latest == nil { + return "no upload started" + } + return latest.summary() +} + +// attempt is the phase state of a single request. +type attempt struct { + mu sync.Mutex + number int + // failed stops late callbacks from adding phases: Go's detached dials can + // keep reporting after RoundTrip has returned an error. + failed bool + open map[string]openPhase + done []donePhase +} + +type openPhase struct { + start time.Time + count int + err error +} + +type donePhase struct { + name string + duration time.Duration + err error +} + +func (a *attempt) begin(name string) { + a.mu.Lock() + defer a.mu.Unlock() + if a.failed { + return + } + entry := a.open[name] + if entry.count == 0 { + entry.start = time.Now() + } + entry.count++ + a.open[name] = entry +} + +// end closes a phase. Dials can overlap (dual-stack addresses), so a failed +// one only closes the phase after every other dial has finished. The first +// success closes it at once and later results are ignored. +func (a *attempt) end(name string, err error) { + a.mu.Lock() + defer a.mu.Unlock() + entry, ok := a.open[name] + if !ok { + return + } + entry.count-- + if err != nil && entry.err == nil { + entry.err = err + } + if err != nil && entry.count > 0 { + a.open[name] = entry + return + } + if err == nil { + entry.err = nil + } + a.closeLocked(name, entry) +} + +// fail closes every phase still running when the request ends in error. The +// SDK cancels the request at the flush deadline, so this keeps the phase that +// was in flight visible after the cancellation. +func (a *attempt) fail(err error) { + a.mu.Lock() + defer a.mu.Unlock() + a.failed = true + for _, name := range phaseOrder { + if entry, ok := a.open[name]; ok { + entry.err = err + a.closeLocked(name, entry) + } + } +} + +func (a *attempt) closeLocked(name string, entry openPhase) { + delete(a.open, name) + a.done = append(a.done, donePhase{ + name: name, duration: time.Since(entry.start), err: entry.err, + }) +} + +func (a *attempt) summary() string { + a.mu.Lock() + defer a.mu.Unlock() + + var parts []string + for _, d := range a.done { + part := d.name + " " + d.duration.Round(time.Millisecond).String() + if d.err != nil { + part += " (" + d.err.Error() + ")" + } + parts = append(parts, part) + } + now := time.Now() + for _, name := range phaseOrder { + if entry, ok := a.open[name]; ok { + parts = append(parts, "stuck in "+name+" for "+ + now.Sub(entry.start).Round(time.Millisecond).String()) + } + } + if len(parts) == 0 { + parts = append(parts, "no phase reached") + } + return "upload attempt " + strconv.Itoa(a.number) + ": " + strings.Join(parts, ", ") +} + +func (a *attempt) clientTrace() *httptrace.ClientTrace { + return &httptrace.ClientTrace{ + DNSStart: func(httptrace.DNSStartInfo) { a.begin(phaseDNS) }, + DNSDone: func(info httptrace.DNSDoneInfo) { a.end(phaseDNS, info.Err) }, + ConnectStart: func(_, _ string) { a.begin(phaseConnect) }, + ConnectDone: func(_, _ string, err error) { a.end(phaseConnect, err) }, + TLSHandshakeStart: func() { a.begin(phaseTLS) }, + TLSHandshakeDone: func(_ tls.ConnectionState, err error) { a.end(phaseTLS, err) }, + // A reused connection skips DNS, connect and TLS and lands here. + GotConn: func(httptrace.GotConnInfo) { a.begin(phaseSend) }, + WroteRequest: func(info httptrace.WroteRequestInfo) { + a.end(phaseSend, info.Err) + if info.Err == nil { + a.begin(phaseResponse) + } + }, + // The response phase stays open past the first byte: a server that + // sends part of the headers and stalls is still a stuck response. + // The response body closes it. + } +} + +// tracingTransport reports phase timings of each request to a phaseTrace. +type tracingTransport struct { + base http.RoundTripper + trace *phaseTrace +} + +func (t *tracingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + a := t.trace.newAttempt() + ctx := httptrace.WithClientTrace(req.Context(), a.clientTrace()) + resp, err := t.base.RoundTrip(req.WithContext(ctx)) + if err != nil { + a.fail(err) + return resp, err + } + // RoundTrip returns once the headers are in; the body is read later. + resp.Body = &tracedBody{ReadCloser: resp.Body, attempt: a} + return resp, nil +} + +// tracedBody keeps the response phase open until the body is fully read, +// fails, or is closed, so a stalled body read is not reported as finished. +type tracedBody struct { + io.ReadCloser + attempt *attempt +} + +func (b *tracedBody) Read(p []byte) (int, error) { + n, err := b.ReadCloser.Read(p) + switch { + case errors.Is(err, io.EOF): + b.attempt.end(phaseResponse, nil) + case err != nil: + b.attempt.end(phaseResponse, err) + } + return n, err +} + +func (b *tracedBody) Close() error { + err := b.ReadCloser.Close() + b.attempt.end(phaseResponse, nil) + return err +} + +// newTracingTransport mirrors the SDK's default transport, which a custom +// Config.Transport replaces. +func newTracingTransport() (http.RoundTripper, *phaseTrace) { + base := http.DefaultTransport.(*http.Transport).Clone() + base.MaxIdleConns = posthog.DefaultIdleConns + base.MaxIdleConnsPerHost = posthog.DefaultIdleConnsPerHost + trace := &phaseTrace{} + return &tracingTransport{base: base, trace: trace}, trace +} diff --git a/pkg/telemetry/analytics/trace_test.go b/pkg/telemetry/analytics/trace_test.go new file mode 100644 index 000000000..4630efbe7 --- /dev/null +++ b/pkg/telemetry/analytics/trace_test.go @@ -0,0 +1,259 @@ +package analytics + +import ( + "errors" + "net" + "net/http" + "net/http/httptest" + "net/http/httptrace" + "regexp" + "strings" + "testing" + "time" +) + +func TestPhaseTraceSummary_NoUpload(t *testing.T) { + if got := (&phaseTrace{}).summary(); got != "no upload started" { + t.Fatalf("summary = %q", got) + } + var nilTrace *phaseTrace + if got := nilTrace.summary(); got != "" { + t.Fatalf("nil summary = %q", got) + } +} + +func TestPhaseTraceSummary_ReportsOpenAndFailedPhases(t *testing.T) { + trace := &phaseTrace{} + a := trace.newAttempt() + a.begin(phaseDNS) + a.end(phaseDNS, nil) + a.begin(phaseConnect) + + got := trace.summary() + for _, want := range []string{"upload attempt 1", "dns ", "stuck in connect"} { + if !strings.Contains(got, want) { + t.Errorf("summary %q lacks %q", got, want) + } + } + + a.fail(errors.New("context canceled")) + got = trace.summary() + if !strings.Contains(got, "connect") || !strings.Contains(got, "(context canceled)") { + t.Errorf("summary after failure = %q", got) + } + if strings.Contains(got, "stuck in") { + t.Errorf("failed phase still reported as stuck: %q", got) + } +} + +func TestPhaseTraceSummary_ReportsLatestAttempt(t *testing.T) { + trace := &phaseTrace{} + first := trace.newAttempt() + first.begin(phaseTLS) + trace.newAttempt() + + got := trace.summary() + if !strings.Contains(got, "upload attempt 2") || strings.Contains(got, "tls") { + t.Fatalf("summary = %q", got) + } +} + +func TestAttempts_DoNotShareState(t *testing.T) { + trace := &phaseTrace{} + first := trace.newAttempt() + first.begin(phaseResponse) + second := trace.newAttempt() + second.begin(phaseResponse) + + // Finishing the first request must not close the second one's wait. + first.end(phaseResponse, nil) + + got := trace.summary() + if !strings.Contains(got, "upload attempt 2") || !strings.Contains(got, "stuck in response") { + t.Fatalf("summary = %q, want the second attempt still waiting", got) + } +} + +func TestAttempt_FailedDialKeepsPhaseOpenWhileAnotherIsPending(t *testing.T) { + trace := &phaseTrace{} + a := trace.newAttempt() + a.begin(phaseConnect) + a.begin(phaseConnect) + + a.end(phaseConnect, errors.New("network unreachable")) + if got := trace.summary(); !strings.Contains(got, "stuck in connect") { + t.Fatalf("summary = %q, want connect still open for the pending dial", got) + } + + a.fail(errors.New("context canceled")) + if got := trace.summary(); !strings.Contains(got, "(context canceled)") { + t.Fatalf("summary = %q, want the cancellation on the pending dial", got) + } +} + +func TestAttempt_FirstSuccessfulDialClosesPhase(t *testing.T) { + trace := &phaseTrace{} + a := trace.newAttempt() + a.begin(phaseConnect) + a.begin(phaseConnect) + + a.end(phaseConnect, nil) + // The losing dial reports a cancellation after the winner connected. + a.end(phaseConnect, errors.New("operation was canceled")) + + got := trace.summary() + if strings.Contains(got, "stuck in") || strings.Contains(got, "canceled") { + t.Fatalf("summary = %q, want a clean connect", got) + } +} + +func TestAttempt_IgnoresPhasesStartedAfterFailure(t *testing.T) { + trace := &phaseTrace{} + a := trace.newAttempt() + a.begin(phaseDNS) + a.fail(errors.New("context canceled")) + + // A detached dial keeps going after RoundTrip returned its error. + a.begin(phaseConnect) + a.begin(phaseTLS) + + got := trace.summary() + if strings.Contains(got, "connect") || strings.Contains(got, "tls") { + t.Fatalf("summary = %q, want no phases started after the failure", got) + } + if !strings.Contains(got, "dns") || !strings.Contains(got, "(context canceled)") { + t.Fatalf("summary = %q, want the interrupted dns phase", got) + } +} + +func TestAttempt_FailedWriteIsAttributedToSend(t *testing.T) { + trace := &phaseTrace{} + a := trace.newAttempt() + a.clientTrace().GotConn(httptrace.GotConnInfo{}) + a.clientTrace().WroteRequest(httptrace.WroteRequestInfo{Err: errors.New("broken pipe")}) + + got := trace.summary() + if !strings.Contains(got, "send ") || !strings.Contains(got, "(broken pipe)") { + t.Fatalf("summary = %q, want the write failure on send", got) + } + if strings.Contains(got, "response") { + t.Fatalf("summary = %q, response must not start after a failed write", got) + } +} + +func TestFlushTimeout_ReportsStuckTLSHandshake(t *testing.T) { + // Accepts the connection but never answers the TLS ClientHello. + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + t.Cleanup(func() { _ = conn.Close() }) + } + }() + + c := newTestClient(t, "https://"+listener.Addr().String(), 300*time.Millisecond) + c.RecordEvent(testEvent()) + c.Flush() + + got := c.trace.summary() + // Whether the SDK's cancellation has reached the transport by now varies, + // so the stuck phase reads either as "stuck in tls" or as an interrupted "tls". + if !strings.Contains(got, "connect ") || !strings.Contains(got, "tls ") { + t.Fatalf("summary = %q, want completed connect and a tls phase", got) + } +} + +func TestFlushTimeout_ReportsStuckResponse(t *testing.T) { + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + case <-release: + } + })) + t.Cleanup(srv.Close) + t.Cleanup(func() { close(release) }) + + c := newTestClient(t, srv.URL, 300*time.Millisecond) + c.RecordEvent(testEvent()) + c.Flush() + + got := c.trace.summary() + if !strings.Contains(got, "send ") || !strings.Contains(got, "response ") { + t.Fatalf("summary = %q, want sent request and a response phase", got) + } +} + +// stallingServer answers each request with reply and then goes silent. It +// returns the address to upload to. +func stallingServer(t *testing.T, reply string) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = listener.Close() }) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + t.Cleanup(func() { _ = conn.Close() }) + go func() { + buf := make([]byte, 4096) + _, _ = conn.Read(buf) + _, _ = conn.Write([]byte(reply)) + }() + } + }() + return "http://" + listener.Addr().String() +} + +func TestFlushTimeout_ReportsResponseStalls(t *testing.T) { + tests := map[string]string{ + // Part of the status line, then silence. + "stalled after first byte": "HTTP/1.1 2", + // Complete headers, then a body that never finishes. The SDK reads + // the body of non-2xx replies, so this stalls after RoundTrip returns. + "stalled body": "HTTP/1.1 500 Internal Server Error\r\nContent-Length: 100\r\n\r\npartial", + } + for name, reply := range tests { + t.Run(name, func(t *testing.T) { + c := newTestClient(t, stallingServer(t, reply), 300*time.Millisecond) + c.RecordEvent(testEvent()) + c.Flush() + + // Either still waiting or interrupted with an error, never a clean finish. + got := c.trace.summary() + if !stalledResponse.MatchString(got) { + t.Fatalf("summary = %q, want the stall reported in the response phase", got) + } + }) + } +} + +var stalledResponse = regexp.MustCompile(`stuck in response|response \S+ \(`) + +func TestFlush_SuccessfulUploadLeavesNoOpenPhase(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"status":1}`)) + })) + t.Cleanup(srv.Close) + + c := newTestClient(t, srv.URL, 5*time.Second) + c.RecordEvent(testEvent()) + c.Flush() + + got := c.trace.summary() + if strings.Contains(got, "stuck in") || strings.Contains(got, "(") { + t.Fatalf("summary = %q, want every phase completed cleanly", got) + } +}