Repository navigation
feat: replace permanent event-delivery disable with a bounded cooldown #256
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
828021b
0cd76e2
f40324a
8358410
47fec12
4d3457f
c0066f0
d3bd6a8
6a9a1b0
5075d56
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -319,6 +319,7 @@ static final class EventDispatcher { | |
| private final AtomicInteger busyFlushWorkersCount; | ||
| private final AtomicLong lastKnownPastTime = new AtomicLong(0); | ||
| private final AtomicBoolean disabled = new AtomicBoolean(false); | ||
| private final DeliveryCooldownTracker cooldown; | ||
| private final AtomicBoolean didSendInitEvent = new AtomicBoolean(false); | ||
| final DiagnosticStore diagnosticStore; // visible for testing | ||
| private final EventContextDeduplicator contextDeduplicator; | ||
|
|
@@ -346,6 +347,7 @@ private EventDispatcher( | |
| this.diagnosticStore = eventsConfig.diagnosticStore; | ||
| this.busyFlushWorkersCount = new AtomicInteger(0); | ||
| this.logger = logger; | ||
| this.cooldown = new DeliveryCooldownTracker(eventsConfig.cooldownNanoTimeFn); | ||
|
|
||
| ThreadFactory threadFactory = new ThreadFactory() { | ||
| @Override | ||
|
|
@@ -603,10 +605,21 @@ private boolean shouldDebugEvent(Event.FeatureRequest fe) { | |
| } | ||
|
|
||
| private void triggerFlush(EventBuffer outbox, BlockingQueue<FlushPayload> payloadQueue) { | ||
| if (disabled.get() || outbox.isEmpty()) { | ||
| if (disabled.get()) { | ||
| return; | ||
| } | ||
| // The cooldown check and the token issue are one atomic step, so a window opening | ||
| // concurrently on a worker thread either diverts this flush to the discard path or makes | ||
| // the dispatched payload's token stale. | ||
| DeliveryCooldownTracker.WindowToken token = cooldown.beginDeliveryUnlessInCooldown(); | ||
| if (token == null) { | ||
| discardPayloadInCooldown(outbox); | ||
| return; | ||
| } | ||
| if (outbox.isEmpty()) { | ||
| return; | ||
| } | ||
| FlushPayload payload = outbox.getPayload(); | ||
| FlushPayload payload = outbox.getPayload(token); | ||
| if (diagnosticStore != null) { | ||
| int summaryCount = 0; | ||
| for (EventSummary summary : payload.summaries) { | ||
|
|
@@ -632,12 +645,79 @@ private void triggerFlush(EventBuffer outbox, BlockingQueue<FlushPayload> payloa | |
| } | ||
| } | ||
|
|
||
| private void handleResponse(EventSender.Result result) { | ||
| // While a window is open, a flush of any kind is a no-op that behaves exactly as if the batch | ||
| // had been sent and the service had refused it: the buffered events are disposed of, and any | ||
| // context introductions they carried are forgotten so a later payload can introduce those | ||
| // contexts again. Only the network attempt and the serialization work are skipped. | ||
| private void discardPayloadInCooldown(EventBuffer outbox) { | ||
| if (outbox.isEmpty()) { | ||
| return; | ||
| } | ||
| // No batch reached the service, so the diagnostic batch-size stat must not report one. | ||
| if (diagnosticStore != null) { | ||
| diagnosticStore.recordEventsInBatch(0); | ||
| } | ||
| FlushPayload payload = outbox.getPayload(null); | ||
| outbox.clear(); | ||
| int eventCount = payload.events.length; | ||
| for (EventSummary summary : payload.summaries) { | ||
| if (!summary.isEmpty()) { | ||
| eventCount++; | ||
| } | ||
| } | ||
| boolean hadContextIntroductions = false; | ||
| for (Event e : payload.events) { | ||
| if (e instanceof Event.Index || e instanceof Event.Identify) { | ||
| hadContextIntroductions = true; | ||
| break; | ||
| } | ||
| } | ||
| if (hadContextIntroductions && contextDeduplicator != null) { | ||
| contextDeduplicator.flush(); | ||
| } | ||
| long remainingSeconds = Math.max(1, (cooldown.remainingMillis() + 999) / 1000); | ||
| if (cooldown.takeDiscardWarning()) { | ||
| logger.warn("Discarded {} events: event delivery is cooling down for another {} seconds", | ||
| eventCount, remainingSeconds); | ||
| } else { | ||
| logger.debug("Discarded {} events: event delivery is cooling down for another {} seconds", | ||
| eventCount, remainingSeconds); | ||
| } | ||
| } | ||
|
|
||
| private void handleResponse(EventSender.Result result, DeliveryCooldownTracker.WindowToken token) { | ||
| if (result.getTimeFromServer() != null) { | ||
| lastKnownPastTime.set(result.getTimeFromServer().getTime()); | ||
| } | ||
| if (result.isUnexpectedFailure()) { | ||
| noteUnexpectedFailure(token); | ||
| } else if (result.isSuccess()) { | ||
| if (cooldown.recordSuccess(token)) { | ||
| logger.info("Event delivery resumed after cooldown"); | ||
| } | ||
| } else { | ||
| cooldown.recordFailure(token); | ||
| } | ||
| } | ||
|
|
||
| // The cooldown governs analytics (flush) delivery only. A diagnostic delivery never touches | ||
| // cooldown state. | ||
| private void handleDiagnosticResponse(EventSender.Result result) { | ||
| if (result.getTimeFromServer() != null) { | ||
| lastKnownPastTime.set(result.getTimeFromServer().getTime()); | ||
| } | ||
| if (result.isMustShutDown()) { | ||
| disabled.set(true); | ||
| } | ||
|
|
||
| private void noteUnexpectedFailure(DeliveryCooldownTracker.WindowToken token) { | ||
| long windowMillis = cooldown.recordUnexpectedFailure(token); | ||
| if (windowMillis > 0) { | ||
| // The refused batch's index and identify events were lost with it, so forget every | ||
| // context key | ||
| if (contextDeduplicator != null) { | ||
| inbox.offer(new EventProcessorMessage(MessageType.FLUSH_USERS, null, false)); | ||
| } | ||
| logger.error("Event delivery failed with an unexpected error;" | ||
| + " event delivery will not be attempted for the next {} seconds", windowMillis / 1000); | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -652,7 +732,7 @@ public void run() { | |
| writer.flush(); | ||
| EventSender.Result result = eventsConfig.eventSender.sendDiagnosticEvent( | ||
| buffer.toByteArray(), eventsConfig.eventsUri); | ||
| handleResponse(result); | ||
| handleDiagnosticResponse(result); | ||
| if (diagnosticEvent.initEvent) { | ||
| didSendInitEvent.set(true); | ||
| } | ||
|
|
@@ -716,10 +796,10 @@ long getAndClearDroppedCount() { | |
| return res; | ||
| } | ||
|
|
||
| FlushPayload getPayload() { | ||
| FlushPayload getPayload(DeliveryCooldownTracker.WindowToken token) { | ||
| Event[] eventsOut = events.toArray(new Event[events.size()]); | ||
| List<EventSummarizer.EventSummary> summaries = summarizer.getSummariesAndReset(); | ||
| return new FlushPayload(eventsOut, summaries); | ||
| return new FlushPayload(eventsOut, summaries, token); | ||
| } | ||
|
|
||
| void clear() { | ||
|
|
@@ -731,15 +811,17 @@ void clear() { | |
| private static final class FlushPayload { | ||
| final Event[] events; | ||
| final List<EventSummary> summaries; | ||
| final DeliveryCooldownTracker.WindowToken token; | ||
|
|
||
| FlushPayload(Event[] events, List<EventSummary> summaries) { | ||
| FlushPayload(Event[] events, List<EventSummary> summaries, DeliveryCooldownTracker.WindowToken token) { | ||
| this.events = events; | ||
| this.summaries = summaries; | ||
| this.token = token; | ||
| } | ||
| } | ||
|
|
||
| private static interface EventResponseListener { | ||
| void handleResponse(EventSender.Result result); | ||
| void handleResponse(EventSender.Result result, DeliveryCooldownTracker.WindowToken token); | ||
| } | ||
|
|
||
| private static final class SendEventsTask implements Runnable { | ||
|
|
@@ -781,23 +863,48 @@ public void run() { | |
| continue; | ||
| } | ||
| try { | ||
| ByteArrayOutputStream buffer = new ByteArrayOutputStream(INITIAL_OUTPUT_BUFFER_SIZE); | ||
| Writer writer = new BufferedWriter(new OutputStreamWriter(buffer, Charset.forName("UTF-8")), INITIAL_OUTPUT_BUFFER_SIZE); | ||
| int outputEventCount = formatter.writeOutputEvents(payload.events, payload.summaries, writer); | ||
| writer.flush(); | ||
| EventSender.Result result = eventsConfig.eventSender.sendAnalyticsEvents( | ||
| buffer.toByteArray(), | ||
| outputEventCount, | ||
| eventsConfig.eventsUri | ||
| ); | ||
| responseListener.handleResponse(result); | ||
| } catch (Exception e) { | ||
| logger.error("Unexpected error in event processor: {}", LogValues.exceptionSummary(e)); | ||
| logger.debug(LogValues.exceptionTrace(e)); | ||
| } | ||
| synchronized (activeFlushWorkersCount) { | ||
| activeFlushWorkersCount.decrementAndGet(); | ||
| activeFlushWorkersCount.notifyAll(); | ||
| EventSender.Result result; | ||
| try { | ||
| ByteArrayOutputStream buffer = new ByteArrayOutputStream(INITIAL_OUTPUT_BUFFER_SIZE); | ||
| Writer writer = new BufferedWriter(new OutputStreamWriter(buffer, Charset.forName("UTF-8")), INITIAL_OUTPUT_BUFFER_SIZE); | ||
| int outputEventCount = formatter.writeOutputEvents(payload.events, payload.summaries, writer); | ||
| writer.flush(); | ||
| result = eventsConfig.eventSender.sendAnalyticsEvents( | ||
| buffer.toByteArray(), | ||
| outputEventCount, | ||
| eventsConfig.eventsUri | ||
| ); | ||
| } catch (Throwable e) { | ||
| logger.error("Unexpected error in event processor: {}", LogValues.exceptionSummary(e)); | ||
| logger.debug(LogValues.exceptionTrace(e)); | ||
| // The batch was not delivered, and the delivery schedule must know that: a failure | ||
| // here counts as an ordinary failure, which breaks the consecutive-success streak | ||
| // without opening a cooldown window. Catching Throwable rather than Exception keeps an | ||
| // Error (such as an OutOfMemoryError while serializing) from killing this worker. | ||
| result = EventSender.Result.failure(); | ||
| } | ||
| if (result == null) { | ||
| // A custom EventSender that returns no result reported no delivery, so treat it as an | ||
| // ordinary failure. | ||
| result = EventSender.Result.failure(); | ||
| } | ||
| try { | ||
| responseListener.handleResponse(result, payload.token); | ||
| } catch (Throwable e) { | ||
| // Any remaining throw from response handling must not kill this worker thread, so this | ||
| // catches Throwable rather than Exception. | ||
| logger.error("Unexpected error in event processor: {}", LogValues.exceptionSummary(e)); | ||
| logger.debug(LogValues.exceptionTrace(e)); | ||
| } | ||
| } finally { | ||
| // Restore the busy-worker count for this payload however the body exits, or a later | ||
| // close() flush would wait forever in waitUntilAllFlushWorkersInactive. The body catches | ||
| // Throwable, so no worker is lost to an Error; on Android, which runs a single worker, a | ||
| // lost worker would leave that close() flush with nobody to drain it. | ||
| synchronized (activeFlushWorkersCount) { | ||
| activeFlushWorkersCount.decrementAndGet(); | ||
| activeFlushWorkersCount.notifyAll(); | ||
| } | ||
|
Comment on lines
+899
to
+907
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. AI review: The This test times out on 6a9a1b0, stuck in @Test(timeout = 15000)
public void errorOnTheOnlyFlushWorkerDoesNotWedgeClose() throws Exception {
EventSender throwingSender = new EventSender() {
@Override
public Result sendAnalyticsEvents(byte[] data, int eventCount, URI eventsBaseUri) {
throw new AssertionError("boom");
}
@Override
public Result sendDiagnosticEvent(byte[] data, URI eventsBaseUri) {
return Result.success(null);
}
@Override
public void close() throws IOException {}
};
// An Error from the sender ends the only flush worker.
DefaultEventProcessor ep = makeEventProcessor(baseConfig(throwingSender).eventSendingThreadPoolSize(1));
ep.sendEvent(identifyEvent(user));
ep.flushAsync();
ep.waitUntilInactive();
// close() must still return when it flushes a pending event that no worker is left to deliver.
ep.sendEvent(identifyEvent(user));
ep.close();
}
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. AI review comment: Fixed in 5075d56. Both flush-worker work sites now catch Throwable rather than Exception, so an Error (for example an OutOfMemoryError while serializing) is handled as an ordinary failure and the worker survives instead of dying; the finally comment is corrected to match. I also strengthened errorFromEventSenderDoesNotWedgeClose to reproduce exactly this case: a single flush worker (as on Android), a real Error on the first delivery, then an assertion that a second delivery still lands (proving the worker lived) in addition to close() returning. I confirmed it fails when the fix is reverted and passes with it. Leaving this open for you to confirm the approach (catch Throwable to keep the worker alive, rather than replacing a dead worker). |
||
| } | ||
| } | ||
| } | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.