From 0b9d867235a996c0ce64cc99345801984d6c2b07 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 13:06:20 +0000 Subject: [PATCH] fix(sync): drop durable calendar-list discovery failures Classify Google calendarList 4xx refusals as discoveryFailed and settle them as drops instead of retryableTransient, matching the readFailed path. Keeps 429/5xx/network and rate-limit 403s as transient retries. Co-authored-by: Tyler Dane --- .../calendar-list-sync.service.db.test.ts | 30 ++++--- .../src/domain/calendar-list-sync.service.ts | 4 +- .../sync-job-dispatch.service.db.test.ts | 81 ++++++++++++++++- .../src/domain/sync-job-dispatch.service.ts | 40 ++++++++- .../google/google-calendar.adapter.test.ts | 66 ++++++++++++-- .../google/google-calendar.adapter.ts | 88 ++++++++++++++++--- .../src/providers/provider-calendar.port.ts | 10 ++- 7 files changed, 286 insertions(+), 33 deletions(-) diff --git a/packages/sync/src/domain/calendar-list-sync.service.db.test.ts b/packages/sync/src/domain/calendar-list-sync.service.db.test.ts index 9c01da3cd7..b705903b1c 100644 --- a/packages/sync/src/domain/calendar-list-sync.service.db.test.ts +++ b/packages/sync/src/domain/calendar-list-sync.service.db.test.ts @@ -432,18 +432,26 @@ describe("syncCalendarList", () => { expect(stale?.active).toBe(false); }); - it("throws on a non-cursor discovery failure so the worker retries", async () => { + it("rethrows non-cursor discovery failures for dispatch/worker settlement", async () => { + // syncCalendarList does not settle these itself: durable discoveryFailed + // is dropped by dispatch, and transient is retried by the worker. const conn = connection(); - const discovery = { - provider: "google" as const, - discoverCalendars: async () => { - throw new ProviderCalendarError("discoveryFailed", "boom"); - }, - }; - - await expect( - syncCalendarList(deps(discovery as unknown as FakeDiscovery), conn, now), - ).rejects.toThrow("boom"); + for (const reason of ["discoveryFailed", "transient"] as const) { + const discovery = { + provider: "google" as const, + discoverCalendars: async () => { + throw new ProviderCalendarError(reason, `${reason}-boom`); + }, + }; + + await expect( + syncCalendarList( + deps(discovery as unknown as FakeDiscovery), + conn, + now, + ), + ).rejects.toThrow(`${reason}-boom`); + } }); it("is idempotent: a repeated pass coalesces into one import per calendar", async () => { diff --git a/packages/sync/src/domain/calendar-list-sync.service.ts b/packages/sync/src/domain/calendar-list-sync.service.ts index 5b712061fc..8f03d3d232 100644 --- a/packages/sync/src/domain/calendar-list-sync.service.ts +++ b/packages/sync/src/domain/calendar-list-sync.service.ts @@ -90,7 +90,9 @@ export async function syncCalendarList( fullList = true; discovery = await deps.discovery.discoverCalendars({ accessToken }); } else { - // discoveryFailed or unexpected: transient, let the worker retry. + // transient: worker retries with backoff. discoveryFailed: dispatch + // catches and drops (durable 4xx). Anything unexpected also throws so + // the worker's generic catch can bound it. throw error; } } diff --git a/packages/sync/src/domain/sync-job-dispatch.service.db.test.ts b/packages/sync/src/domain/sync-job-dispatch.service.db.test.ts index b371fbc0ab..37561243da 100644 --- a/packages/sync/src/domain/sync-job-dispatch.service.db.test.ts +++ b/packages/sync/src/domain/sync-job-dispatch.service.db.test.ts @@ -9,6 +9,7 @@ import { type SyncJobDispatchDeps, } from "@sync/domain/sync-job-dispatch.service"; import { ProviderAuthError } from "@sync/providers/provider-auth.port"; +import { ProviderCalendarError } from "@sync/providers/provider-calendar.port"; import { type ProviderEvent, type ProviderEventRead, @@ -172,13 +173,14 @@ describe("dispatchSyncJob", () => { reader: FakeReader, custody: SyncJobDispatchDeps["custody"] = tokenSource, notificationsOverride: SyncJobDispatchDeps["notifications"] = notifications, + discoveryOverride: SyncJobDispatchDeps["discovery"] = discovery, ): SyncJobDispatchDeps => ({ events, occurrences, resources, calendars, connections, - discovery, + discovery: discoveryOverride, jobs, commands, reader, @@ -806,4 +808,81 @@ describe("dispatchSyncJob", () => { } expect(discarded).toEqual([connectionId]); }); + + it("drops a durable calendarListSync discovery failure instead of burning retries", async () => { + // 2026-08-08 prod: "The user must be signed up for Google Calendar." was + // classified as discoveryFailed and retried as retryableTransient for all + // 20 attempts, then self-heal-requeued until the requeue budget exhausted. + const tenantId = objectId(); + const principalId = objectId(); + const connectionId = objectId(); + stubbedConnection = { + _id: connectionId, + tenantId, + principalId, + } as ProviderConnectionRecord; + const refusing: SyncJobDispatchDeps["discovery"] = { + provider: "google", + discoverCalendars: async () => { + throw new ProviderCalendarError( + "discoveryFailed", + "Google rejected the calendar list read", + { + cause: new Error( + "The user must be signed up for Google Calendar. (HTTP 403, reason notACalendarUser)", + ), + }, + ); + }, + }; + + const outcome = await dispatchSyncJob( + deps(new FakeReader([]), tokenSource, notifications, refusing), + calendarListJob(connectionId, tenantId, principalId), + now, + ); + + expect(outcome.result).toBe("drop"); + if (outcome.result === "drop") { + expect(outcome.reason).toContain("notACalendarUser"); + expect(outcome.reason).toContain(connectionId); + } + const calendarList = ( + await resources.listByConnection( + tenantId as SyncResourceRecord["tenantId"], + principalId as SyncResourceRecord["principalId"], + connectionId as SyncResourceRecord["connectionId"], + ) + ).find((resource) => resource.resourceKind === "calendarList"); + expect(calendarList?.lastReadFailureAt).toEqual(now()); + expect(calendarList?.lastReadFailureDetail).toContain("notACalendarUser"); + }); + + it("rethrows a transient calendarListSync discovery failure for the worker to retry", async () => { + const tenantId = objectId(); + const principalId = objectId(); + const connectionId = objectId(); + stubbedConnection = { + _id: connectionId, + tenantId, + principalId, + } as ProviderConnectionRecord; + const flaky: SyncJobDispatchDeps["discovery"] = { + provider: "google", + discoverCalendars: async () => { + throw new ProviderCalendarError( + "transient", + "Google calendar list temporarily unavailable", + ); + }, + }; + + await expect( + dispatchSyncJob( + deps(new FakeReader([]), tokenSource, notifications, flaky), + calendarListJob(connectionId, tenantId, principalId), + now, + ), + ).rejects.toMatchObject({ reason: "transient" }); + }); }); diff --git a/packages/sync/src/domain/sync-job-dispatch.service.ts b/packages/sync/src/domain/sync-job-dispatch.service.ts index c718fa1621..e36979e273 100644 --- a/packages/sync/src/domain/sync-job-dispatch.service.ts +++ b/packages/sync/src/domain/sync-job-dispatch.service.ts @@ -5,7 +5,10 @@ import { repairCalendar } from "@sync/domain/calendar-repair.service"; import { type AccessTokenSource } from "@sync/domain/provider-command.service"; import { maintainSubscription } from "@sync/domain/subscription-maintenance.service"; import { ProviderAuthError } from "@sync/providers/provider-auth.port"; -import { type ProviderCalendarAdapter } from "@sync/providers/provider-calendar.port"; +import { + type ProviderCalendarAdapter, + ProviderCalendarError, +} from "@sync/providers/provider-calendar.port"; import { ProviderEventReadError, type ProviderEventReader, @@ -142,6 +145,41 @@ export async function dispatchSyncJob( reason: `provider durably rejected reads for resource ${job.resourceId} (${detail}); sync resumes once the calendar is readable again`, }; } + + // Same shape for calendar-list discovery: a durable 4xx (account not a + // Calendar user, forbidden list, etc.) used to fall through as + // retryableTransient and burn all 20 attempts, then trip the self-heal + // requeue budget (2026-08-08: job 6a76454e… / connection 6a653974…). + // Dropping frees the coalescing key so a later rediscovery or reconnect + // can try again; stamp the calendarList resource so triage still has a + // durable trace after the job row is gone. + if ( + error instanceof ProviderCalendarError && + error.reason === "discoveryFailed" + ) { + const detail = + error.cause instanceof Error ? error.cause.message : error.message; + const calendarList = ( + await deps.resources.listByConnection( + job.tenantId, + job.principalId, + job.connectionId, + ) + ).find((resource) => resource.resourceKind === "calendarList"); + if (calendarList) { + await deps.resources.markReadFailure( + job.tenantId, + job.principalId, + calendarList._id, + now(), + detail, + ); + } + return { + result: "drop", + reason: `provider durably rejected calendar discovery for connection ${job.connectionId} (${detail}); sync resumes once the account can list calendars again`, + }; + } throw error; } } diff --git a/packages/sync/src/providers/google/google-calendar.adapter.test.ts b/packages/sync/src/providers/google/google-calendar.adapter.test.ts index 1dd3ca9529..2368130eef 100644 --- a/packages/sync/src/providers/google/google-calendar.adapter.test.ts +++ b/packages/sync/src/providers/google/google-calendar.adapter.test.ts @@ -381,15 +381,26 @@ describe("GoogleCalendarAdapter", () => { expect(error.reason).toBe("cursorExpired"); }); - it("maps any other provider error to discoveryFailed and redacts the cause", async () => { + it("maps a durable 403 to discoveryFailed and keeps status/reason for triage", async () => { // A gaxios-shaped error whose config carries the bearer token must never - // survive onto the ProviderCalendarError cause chain. - const leaky = Object.assign(new Error("Request failed with status 403"), { - config: { - headers: { Authorization: "Bearer super-secret-access-token" }, + // survive onto the ProviderCalendarError cause chain — but HTTP status and + // Google's machine-readable reason must, or PostHog triage is blind. + const leaky = Object.assign( + new Error("The user must be signed up for Google Calendar."), + { + config: { + headers: { Authorization: "Bearer super-secret-access-token" }, + }, + response: { + status: 403, + data: { + error: { + errors: [{ reason: "notACalendarUser" }], + }, + }, + }, }, - response: { status: 403 }, - }); + ); const api = new FakeCalendarListApi([], leaky); const { adapter } = adapterWith(api); @@ -399,9 +410,50 @@ describe("GoogleCalendarAdapter", () => { expect(error.reason).toBe("discoveryFailed"); expect(error.cause).toBeInstanceOf(Error); + expect((error.cause as Error).message).toContain("HTTP 403"); + expect((error.cause as Error).message).toContain("notACalendarUser"); expect((error.cause as { config?: unknown }).config).toBeUndefined(); expect(JSON.stringify(error.cause)).not.toContain( "super-secret-access-token", ); }); + + it("maps 429 / 5xx / network failures to transient discovery errors", async () => { + for (const status of [429, 500, 503]) { + const api = new FakeCalendarListApi([], { + response: { status }, + }); + const { adapter } = adapterWith(api); + const error = (await adapter + .discoverCalendars({ accessToken: "at" }) + .catch((e) => e)) as ProviderCalendarError; + expect(error.reason).toBe("transient"); + } + + const network = new FakeCalendarListApi([], new Error("socket hang up")); + const { adapter } = adapterWith(network); + const error = (await adapter + .discoverCalendars({ accessToken: "at" }) + .catch((e) => e)) as ProviderCalendarError; + expect(error.reason).toBe("transient"); + }); + + it("keeps a 403 rate-limit reason transient, not durable", async () => { + const limited = Object.assign(new Error("Rate Limit Exceeded"), { + response: { + status: 403, + data: { + error: { errors: [{ reason: "rateLimitExceeded" }] }, + }, + }, + }); + const api = new FakeCalendarListApi([], limited); + const { adapter } = adapterWith(api); + + const error = (await adapter + .discoverCalendars({ accessToken: "at" }) + .catch((e) => e)) as ProviderCalendarError; + + expect(error.reason).toBe("transient"); + }); }); diff --git a/packages/sync/src/providers/google/google-calendar.adapter.ts b/packages/sync/src/providers/google/google-calendar.adapter.ts index 123d056e23..3bcc17cd36 100644 --- a/packages/sync/src/providers/google/google-calendar.adapter.ts +++ b/packages/sync/src/providers/google/google-calendar.adapter.ts @@ -150,13 +150,17 @@ export class GoogleCalendarAdapter implements ProviderCalendarAdapter { try { return await api.listPage(params); } catch (error) { - // An expired syncToken (410 Gone) is not retryable with the same token: - // the caller must drop it and re-list in full. Everything else is a - // generic discovery failure. + // Classify before wrapping so durable account/calendar refusals do not + // burn retries the way a 5xx or rate-limit blip should. Keep HTTP status + // + Google's machine-readable reason on the cause for triage — nested + // Error.cause is only serialized one level deep in PostHog/logs. + const reason = classifyDiscoveryError(error); throw new ProviderCalendarError( - isCursorExpired(error) ? "cursorExpired" : "discoveryFailed", - "Google rejected the calendar list read", - { cause: redactedCause(error) }, + reason, + reason === "transient" + ? "Google calendar list temporarily unavailable" + : "Google rejected the calendar list read", + { cause: discoveryFailureCause(error) }, ); } } @@ -234,11 +238,75 @@ function mapAccessRole( return (googleRole && ACCESS_ROLE_BY_GOOGLE[googleRole]) || "busyOnly"; } -// Google signals an expired calendar-list syncToken with HTTP 410 Gone. The -// status lives on the response, so reading it does not touch the request. -function isCursorExpired(error: unknown): boolean { +// Map a Google calendarList.list failure to a discovery-error reason. An +// expired syncToken (410 Gone) forces a full re-list; a rate limit, server, or +// network error is retryable; anything else is an unrecoverable discovery +// failure (account not a Calendar user, authz, etc.). +function classifyDiscoveryError( + error: unknown, +): "cursorExpired" | "transient" | "discoveryFailed" { + const status = httpStatus(error); + if (status === 410) return "cursorExpired"; + if (status === 429 || status === undefined || status >= 500) { + return "transient"; + } + // Google's rate-limit / quota rejections often arrive as 403, not 429. + if ( + googleErrorReasons(error).some((reason) => + TRANSIENT_DISCOVERY_REASONS.includes(reason), + ) + ) { + return "transient"; + } + return "discoveryFailed"; +} + +const TRANSIENT_DISCOVERY_REASONS = [ + "rateLimitExceeded", + "userRateLimitExceeded", + "quotaExceeded", + "backendError", + "internalError", +]; + +// Like redactedCause: drop request-derived fields (bearer token on config), +// but keep the two response facts triage needs — numeric HTTP status and +// Google's machine-readable reason. +function discoveryFailureCause(error: unknown): Error | undefined { + const status = httpStatus(error); + const reason = googleErrorReasons(error)[0]; + const facts = [ + ...(status === undefined ? [] : [`HTTP ${status}`]), + ...(reason === undefined ? [] : [`reason ${reason}`]), + ]; + if (facts.length === 0) return redactedCause(error); + const message = error instanceof Error ? error.message : null; + return new Error( + message ? `${message} (${facts.join(", ")})` : facts.join(", "), + ); +} + +function googleErrorReasons(error: unknown): string[] { + const fromBody = ( + error as { + response?: { + data?: { error?: { errors?: { reason?: unknown }[] } }; + }; + } + )?.response?.data?.error?.errors; + const fromError = (error as { errors?: { reason?: unknown }[] })?.errors; + const reasons = [...(fromBody ?? []), ...(fromError ?? [])] + .map((entry) => entry?.reason) + .filter((reason): reason is string => typeof reason === "string"); + return reasons; +} + +// The HTTP status of a googleapis/gaxios error, from the response or the error +// code. Reading it does not touch the request. Undefined means no HTTP response +// reached us (a network failure), which classifies as transient. +function httpStatus(error: unknown): number | undefined { const status = (error as { response?: { status?: number } })?.response?.status ?? (error as { code?: number })?.code; - return status === 410; + return typeof status === "number" ? status : undefined; } diff --git a/packages/sync/src/providers/provider-calendar.port.ts b/packages/sync/src/providers/provider-calendar.port.ts index 0e7d271b6a..c1107e1498 100644 --- a/packages/sync/src/providers/provider-calendar.port.ts +++ b/packages/sync/src/providers/provider-calendar.port.ts @@ -56,9 +56,15 @@ export interface ProviderCalendarAdapter { // Why a discovery attempt could not complete. `cursorExpired` is distinct so the // caller can drop the stale cursor and re-list in full instead of retrying with -// a token the provider will keep rejecting. +// a token the provider will keep rejecting. `transient` is a retryable network/ +// rate-limit/5xx blip; `discoveryFailed` is a durable 4xx refusal (e.g. the +// Google account is not signed up for Calendar) that retrying cannot fix — +// dispatch settles those as a drop rather than burning the retry ladder +// (2026-08-08: one calendarListSync job logged ~80 exceptions + self-heal +// alarms on "The user must be signed up for Google Calendar."). export type ProviderCalendarErrorReason = - | "discoveryFailed" // the provider rejected or failed the calendar-list read + | "discoveryFailed" // durable provider rejection; settle and wait for rediscovery + | "transient" // retryable network / rate-limit / 5xx failure | "cursorExpired"; // the incremental cursor is too old; a full re-list is required export class ProviderCalendarError extends ProviderError {}