Repository navigation
feat(telemetry): report the slow phase when the analytics flush times out #1840
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
4 commits
Select commit
Hold shift + click to select a range
e4a5a42
feat(telemetry): report the slow phase when the analytics flush times…
skevetter 4787b30
fix(telemetry): keep analytics trace state per request and per dial
skevetter 9f2385f
fix(telemetry): keep the analytics response phase open until the body…
skevetter 70390e6
fix(telemetry): ignore trace callbacks that arrive after a failed upload
skevetter File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,243 @@ | ||
| package analytics | ||
|
|
||
| import ( | ||
| "crypto/tls" | ||
| "errors" | ||
| "io" | ||
| "net/http" | ||
| "net/http/httptrace" | ||
| "strconv" | ||
| "strings" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "github.com/posthog/posthog-go" | ||
| ) | ||
|
|
||
| // Phases of one upload, in the order the HTTP transport reaches them. | ||
| const ( | ||
| phaseDNS = "dns" | ||
| phaseConnect = "connect" | ||
| phaseTLS = "tls" | ||
| phaseSend = "send" | ||
| phaseResponse = "response" | ||
| ) | ||
|
|
||
| var phaseOrder = []string{phaseDNS, phaseConnect, phaseTLS, phaseSend, phaseResponse} | ||
|
|
||
| // phaseTrace records where upload requests spend time, so a flush that hits | ||
| // its deadline can say which step was slow. Each request owns its own attempt, | ||
| // because the SDK uploads batches concurrently; the trace only publishes the | ||
| // latest one for the summary. | ||
| type phaseTrace struct { | ||
| mu sync.Mutex | ||
| attempts int | ||
| latest *attempt | ||
| } | ||
|
|
||
| func (p *phaseTrace) newAttempt() *attempt { | ||
| p.mu.Lock() | ||
| defer p.mu.Unlock() | ||
| p.attempts++ | ||
| p.latest = &attempt{number: p.attempts, open: map[string]openPhase{}} | ||
| return p.latest | ||
| } | ||
|
|
||
| // summary describes the latest upload attempt. It is safe to call on a nil | ||
| // receiver and while requests are still running. | ||
| func (p *phaseTrace) summary() string { | ||
| if p == nil { | ||
| return "" | ||
| } | ||
| p.mu.Lock() | ||
| latest := p.latest | ||
| p.mu.Unlock() | ||
|
|
||
| if latest == nil { | ||
| return "no upload started" | ||
| } | ||
| return latest.summary() | ||
| } | ||
|
|
||
| // attempt is the phase state of a single request. | ||
| type attempt struct { | ||
| mu sync.Mutex | ||
| number int | ||
| // failed stops late callbacks from adding phases: Go's detached dials can | ||
| // keep reporting after RoundTrip has returned an error. | ||
| failed bool | ||
| open map[string]openPhase | ||
| done []donePhase | ||
| } | ||
|
|
||
| type openPhase struct { | ||
| start time.Time | ||
| count int | ||
| err error | ||
| } | ||
|
|
||
| type donePhase struct { | ||
| name string | ||
| duration time.Duration | ||
| err error | ||
| } | ||
|
|
||
| func (a *attempt) begin(name string) { | ||
| a.mu.Lock() | ||
| defer a.mu.Unlock() | ||
| if a.failed { | ||
| return | ||
| } | ||
| entry := a.open[name] | ||
| if entry.count == 0 { | ||
| entry.start = time.Now() | ||
| } | ||
| entry.count++ | ||
| a.open[name] = entry | ||
| } | ||
|
|
||
| // end closes a phase. Dials can overlap (dual-stack addresses), so a failed | ||
| // one only closes the phase after every other dial has finished. The first | ||
| // success closes it at once and later results are ignored. | ||
| func (a *attempt) end(name string, err error) { | ||
| a.mu.Lock() | ||
| defer a.mu.Unlock() | ||
| entry, ok := a.open[name] | ||
| if !ok { | ||
| return | ||
| } | ||
| entry.count-- | ||
| if err != nil && entry.err == nil { | ||
| entry.err = err | ||
| } | ||
| if err != nil && entry.count > 0 { | ||
| a.open[name] = entry | ||
| return | ||
| } | ||
| if err == nil { | ||
| entry.err = nil | ||
| } | ||
| a.closeLocked(name, entry) | ||
| } | ||
|
|
||
| // fail closes every phase still running when the request ends in error. The | ||
| // SDK cancels the request at the flush deadline, so this keeps the phase that | ||
| // was in flight visible after the cancellation. | ||
| func (a *attempt) fail(err error) { | ||
| a.mu.Lock() | ||
| defer a.mu.Unlock() | ||
| a.failed = true | ||
| for _, name := range phaseOrder { | ||
| if entry, ok := a.open[name]; ok { | ||
| entry.err = err | ||
| a.closeLocked(name, entry) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func (a *attempt) closeLocked(name string, entry openPhase) { | ||
| delete(a.open, name) | ||
| a.done = append(a.done, donePhase{ | ||
| name: name, duration: time.Since(entry.start), err: entry.err, | ||
| }) | ||
| } | ||
|
|
||
| func (a *attempt) summary() string { | ||
| a.mu.Lock() | ||
| defer a.mu.Unlock() | ||
|
|
||
| var parts []string | ||
| for _, d := range a.done { | ||
| part := d.name + " " + d.duration.Round(time.Millisecond).String() | ||
| if d.err != nil { | ||
| part += " (" + d.err.Error() + ")" | ||
| } | ||
| parts = append(parts, part) | ||
| } | ||
| now := time.Now() | ||
| for _, name := range phaseOrder { | ||
| if entry, ok := a.open[name]; ok { | ||
| parts = append(parts, "stuck in "+name+" for "+ | ||
| now.Sub(entry.start).Round(time.Millisecond).String()) | ||
| } | ||
| } | ||
| if len(parts) == 0 { | ||
| parts = append(parts, "no phase reached") | ||
| } | ||
| return "upload attempt " + strconv.Itoa(a.number) + ": " + strings.Join(parts, ", ") | ||
| } | ||
|
|
||
| func (a *attempt) clientTrace() *httptrace.ClientTrace { | ||
| return &httptrace.ClientTrace{ | ||
| DNSStart: func(httptrace.DNSStartInfo) { a.begin(phaseDNS) }, | ||
| DNSDone: func(info httptrace.DNSDoneInfo) { a.end(phaseDNS, info.Err) }, | ||
| ConnectStart: func(_, _ string) { a.begin(phaseConnect) }, | ||
| ConnectDone: func(_, _ string, err error) { a.end(phaseConnect, err) }, | ||
| TLSHandshakeStart: func() { a.begin(phaseTLS) }, | ||
| TLSHandshakeDone: func(_ tls.ConnectionState, err error) { a.end(phaseTLS, err) }, | ||
| // A reused connection skips DNS, connect and TLS and lands here. | ||
| GotConn: func(httptrace.GotConnInfo) { a.begin(phaseSend) }, | ||
| WroteRequest: func(info httptrace.WroteRequestInfo) { | ||
| a.end(phaseSend, info.Err) | ||
| if info.Err == nil { | ||
| a.begin(phaseResponse) | ||
| } | ||
| }, | ||
| // The response phase stays open past the first byte: a server that | ||
| // sends part of the headers and stalls is still a stuck response. | ||
| // The response body closes it. | ||
| } | ||
| } | ||
|
|
||
| // tracingTransport reports phase timings of each request to a phaseTrace. | ||
| type tracingTransport struct { | ||
| base http.RoundTripper | ||
| trace *phaseTrace | ||
| } | ||
|
|
||
| func (t *tracingTransport) RoundTrip(req *http.Request) (*http.Response, error) { | ||
| a := t.trace.newAttempt() | ||
| ctx := httptrace.WithClientTrace(req.Context(), a.clientTrace()) | ||
| resp, err := t.base.RoundTrip(req.WithContext(ctx)) | ||
| if err != nil { | ||
| a.fail(err) | ||
| return resp, err | ||
| } | ||
| // RoundTrip returns once the headers are in; the body is read later. | ||
| resp.Body = &tracedBody{ReadCloser: resp.Body, attempt: a} | ||
| return resp, nil | ||
| } | ||
|
|
||
| // tracedBody keeps the response phase open until the body is fully read, | ||
| // fails, or is closed, so a stalled body read is not reported as finished. | ||
| type tracedBody struct { | ||
| io.ReadCloser | ||
| attempt *attempt | ||
| } | ||
|
|
||
| func (b *tracedBody) Read(p []byte) (int, error) { | ||
| n, err := b.ReadCloser.Read(p) | ||
| switch { | ||
| case errors.Is(err, io.EOF): | ||
| b.attempt.end(phaseResponse, nil) | ||
| case err != nil: | ||
| b.attempt.end(phaseResponse, err) | ||
| } | ||
| return n, err | ||
| } | ||
|
|
||
| func (b *tracedBody) Close() error { | ||
| err := b.ReadCloser.Close() | ||
| b.attempt.end(phaseResponse, nil) | ||
| return err | ||
| } | ||
|
|
||
| // newTracingTransport mirrors the SDK's default transport, which a custom | ||
| // Config.Transport replaces. | ||
| func newTracingTransport() (http.RoundTripper, *phaseTrace) { | ||
| base := http.DefaultTransport.(*http.Transport).Clone() | ||
| base.MaxIdleConns = posthog.DefaultIdleConns | ||
| base.MaxIdleConnsPerHost = posthog.DefaultIdleConnsPerHost | ||
| trace := &phaseTrace{} | ||
| return &tracingTransport{base: base, trace: trace}, trace | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.