From f8bfd315adea49cd12841120b2684809002fb147 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Fri, 17 Jul 2026 15:47:29 +0530 Subject: [PATCH] fix(audit): report the onboarding auto-audit to PostHog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runPostSetupAudit() — the audit that runs automatically at the end of first-run setup, and therefore the first audit any new user ever runs — emitted no telemetry at all, while an explicit `failproofai audit` reported cli_audit_started / cli_audit_completed / cli_audit_failed. First-run audits were invisible, so the audit funnel silently undercounted exactly the activation moment it exists to measure. Confirmed live: a fresh install's auto-audit wrote its dashboard cache and PostHog saw nothing. It now emits the same three events tagged `source: "onboarding"`, against the existing `source: "cli"`, so the two paths stay distinguishable. - cli_audit_completed fires before the empty-history return, matching runAuditCli, so a fresh user with no agent history is still counted. - cli_audit_failed is awaited: this function returns straight into the dashboard boot, which would race a fire-and-forget send. - The completed-event properties now come from one shared helper so the two entry points cannot drift apart. Onboarding stays best-effort — never throws, never exits. Verified by driving the real (unmocked) runPostSetupAudit against a local capture server: cli_audit_started and cli_audit_completed both land with source=onboarding and real counts (7,468 tool calls / 210 sessions), and the function returns without throwing or exiting. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 1 + __tests__/audit/audit-cli-telemetry.test.ts | 92 ++++++++++++++++++++- src/audit/cli.ts | 50 +++++++++-- 3 files changed, 133 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 168aef80..3fd21aa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Redirect the 13 AgentEye pages the upstream syncs deleted (`/agenteye/deployment`, `/agenteye/kubernetes-deployment`, `/agenteye/troubleshooting`, …) to a live page instead of hard-404ing. (#556) ### Fixes +- Report the onboarding auto-audit to PostHog. `runPostSetupAudit()` — the audit that runs automatically at the end of first-run setup, and therefore the **first audit every new user ever runs** — emitted no telemetry at all, while an explicit `failproofai audit` reported `cli_audit_started` / `cli_audit_completed` / `cli_audit_failed`. First-run audits were invisible, so the audit funnel silently undercounted exactly the activation moment it exists to measure (confirmed live: a fresh install's auto-audit wrote its dashboard cache and reached PostHog with nothing). It now emits the same three events, tagged `source: "onboarding"` against the existing `source: "cli"`, so the two paths stay distinguishable. `cli_audit_completed` fires before the empty-history return, matching `runAuditCli`, so a fresh user with no agent history is still counted; `cli_audit_failed` is awaited because the function returns straight into the dashboard boot, which would otherwise race a fire-and-forget send. The completed-event properties are now built by one shared helper so the two entry points cannot drift. Onboarding remains best-effort: it never throws and never exits. (#562) - Delete translated pages whose English source no longer exists, and stop them coming back: `translate-docs` gained a `--prune` mode that runs by default on every translation pass (`--no-prune` opts out) and as an explicit step in the `consolidate` job — that job re-checks-out `main` and *overlays* the artifacts, so a prune done only in the per-language jobs would be undone. Translation only ever moved forward, so the 11 pages the AgentEye syncs removed upstream left 154 orphans (11 × 14 locales) that `--update-nav` dropped from the sidebar but Mintlify still served and indexed — non-English readers could land on docs for a deleted feature with no way out. A repo invariant test now fails if any translation outlives its English source. (#556) - Move the docs auto-translation daily cron from 06:00 UTC to 11:05 AM IST (05:35 UTC, encoded as `35 5 * * *` since GitHub Actions cron is always UTC). (#553) diff --git a/__tests__/audit/audit-cli-telemetry.test.ts b/__tests__/audit/audit-cli-telemetry.test.ts index f000423c..06a15be3 100644 --- a/__tests__/audit/audit-cli-telemetry.test.ts +++ b/__tests__/audit/audit-cli-telemetry.test.ts @@ -32,7 +32,7 @@ vi.mock("../../src/audit/open-browser", () => ({ openWhenReady: h.openWhenReady vi.mock("../../scripts/launch", () => ({ launch: h.launch })); vi.mock("../../lib/telemetry-id", () => ({ getInstanceId: () => "test-instance" })); -import { runAuditCli } from "../../src/audit/cli"; +import { runAuditCli, runPostSetupAudit } from "../../src/audit/cli"; function result(over: Partial): AuditResult { return { @@ -129,3 +129,93 @@ describe("failproofai audit telemetry", () => { expect(exitInfo?.resolvedAtExit.has("cli_audit_failed")).toBe(true); }); }); + +/** + * The onboarding auto-audit — the FIRST audit any new user runs — previously + * emitted nothing at all, so first-run audits were invisible in PostHog while + * explicit `failproofai audit` runs reported normally. + */ +describe("post-setup (onboarding) audit telemetry", () => { + afterEach(() => { + delete process.env.FAILPROOFAI_NO_AUTO_AUDIT; + }); + + it("emits cli_audit_started then cli_audit_completed, tagged source=onboarding", async () => { + h.runAudit.mockResolvedValue(result({ eventsScanned: 100, totals: { hits: 2, projectsWithHits: 1 } })); + + await runPostSetupAudit(); + + expect(names()).toEqual(["cli_audit_started", "cli_audit_completed"]); + expect(h.trackHookEvent).toHaveBeenCalledWith("test-instance", "cli_audit_started", { + source: "onboarding", + }); + expect(h.trackHookEvent).toHaveBeenCalledWith("test-instance", "cli_audit_completed", { + source: "onboarding", + events_scanned: 100, + sessions_scanned: 3, + total_hits: 2, + findings: 0, + }); + }); + + // source is what separates the onboarding run from a deliberate one in PostHog. + it("distinguishes itself from an explicit `failproofai audit` run", async () => { + h.runAudit.mockResolvedValue(result({})); + + await runPostSetupAudit(); + const onboarding = h.trackHookEvent.mock.calls.map((c) => (c[2] as { source: string }).source); + + expect(new Set(onboarding)).toEqual(new Set(["onboarding"])); + }); + + it("emits cli_audit_failed when the scan throws, and still does not throw", async () => { + h.runAudit.mockRejectedValue(new TypeError("disk exploded")); + + await expect(runPostSetupAudit()).resolves.toBeUndefined(); + + expect(names()).toEqual(["cli_audit_started", "cli_audit_failed"]); + expect(h.trackHookEvent).toHaveBeenCalledWith("test-instance", "cli_audit_failed", { + source: "onboarding", + error_type: "TypeError", + error_message: "disk exploded", + }); + }); + + // Mirrors runAuditCli, which reports completed regardless of what it found — + // a fresh user with no agent history is exactly who we want counted. + it("still emits cli_audit_completed when there is no history to scan", async () => { + h.runAudit.mockResolvedValue( + result({ eventsScanned: 0, transcripts: { scanned: 0, skipped: 0, errors: 0, durationMs: 0 } }), + ); + + await runPostSetupAudit(); + + expect(names()).toEqual(["cli_audit_started", "cli_audit_completed"]); + expect(h.trackHookEvent).toHaveBeenCalledWith("test-instance", "cli_audit_completed", { + source: "onboarding", + events_scanned: 0, + sessions_scanned: 0, + total_hits: 0, + findings: 0, + }); + // Nothing found — no point warming a cache or opening a dashboard. + expect(h.writeDashboardCache).not.toHaveBeenCalled(); + }); + + it("emits nothing when the auto-audit is opted out", async () => { + process.env.FAILPROOFAI_NO_AUTO_AUDIT = "1"; + + await runPostSetupAudit(); + + expect(names()).toEqual([]); + expect(h.runAudit).not.toHaveBeenCalled(); + }); + + it("never exits the process — onboarding must continue to the dashboard", async () => { + h.runAudit.mockResolvedValue(result({})); + + await runPostSetupAudit(); + + expect(exitInfo).toBeNull(); + }); +}); diff --git a/src/audit/cli.ts b/src/audit/cli.ts index a4ed2855..c1831974 100644 --- a/src/audit/cli.ts +++ b/src/audit/cli.ts @@ -237,6 +237,26 @@ function printSummary(result: AuditResult): void { for (const line of buildSummary(result)) process.stdout.write(` ${line}\n`); } +// ── Audit telemetry ────────────────────────────────────────────────────────── + +/** + * Which entry point ran the audit. `onboarding` is the automatic post-setup run; + * `cli` is an explicit `failproofai audit`. Carried on every cli_audit_* event so + * the first audit a user ever runs is distinguishable from later deliberate ones. + */ +type AuditSource = "cli" | "onboarding"; + +/** Shared so both entry points report cli_audit_completed identically. */ +function auditCompletedProps(source: AuditSource, result: AuditResult) { + return { + source, + events_scanned: result.eventsScanned, + sessions_scanned: result.transcripts.scanned, + total_hits: result.totals.hits, + findings: result.results.length, + }; +} + // ── Post-setup background audit ──────────────────────────────────────────────── /** @@ -252,7 +272,13 @@ function printSummary(result: AuditResult): void { */ export async function runPostSetupAudit(): Promise { if (process.env.FAILPROOFAI_NO_AUTO_AUDIT === "1") return; - + + const instanceId = getInstanceId(); + // Fire-and-forget, as in runAuditCli: the multi-second scan below keeps the + // process alive long enough for this to land, and the completed/failed event + // that follows is awaited. + void trackHookEvent(instanceId, "cli_audit_started", { source: "onboarding" }); + process.stdout.write( `\n ${c(PINK, "✦")} ${c(BOLD, "failproofai audit now running")} ${c(DIM, "· ctrl+c to stop")}\n\n`, ); @@ -260,13 +286,25 @@ export async function runPostSetupAudit(): Promise { let result: AuditResult; try { result = await runWithProgress({}); - } catch { + } catch (err) { + // Awaited: this function returns straight into the dashboard boot, and a + // fire-and-forget fetch would race it. + await trackHookEvent(instanceId, "cli_audit_failed", { + source: "onboarding", + error_type: err instanceof Error ? err.name : "unknown", + error_message: sanitizeErrorMessage(err), + }); process.stdout.write( ` ${c(PINK, "!")} ${c(DIM, "audit couldn't finish — run")} ${c(CYAN, "failproofai audit")} ${c(DIM, "later.")}\n\n`, ); return; } + // Reported before the empty-history return below, so an onboarding audit that + // finds nothing is still counted — matching runAuditCli, which reports + // completed regardless of what the scan turned up. + await trackHookEvent(instanceId, "cli_audit_completed", auditCompletedProps("onboarding", result)); + if (result.eventsScanned === 0) { process.stdout.write( `\n ${c(DIM, "no agent sessions to audit yet — come back after using your agent.")}\n\n`, @@ -328,13 +366,7 @@ export async function runAuditCli(args: string[]): Promise { // would otherwise drop this event. On the dashboard path launch() keeps the // process alive, but awaiting makes delivery reliable on every exit path. // Bounded (5s) and never throws. - await trackHookEvent(instanceId, "cli_audit_completed", { - source: "cli", - events_scanned: result.eventsScanned, - sessions_scanned: result.transcripts.scanned, - total_hits: result.totals.hits, - findings: result.results.length, - }); + await trackHookEvent(instanceId, "cli_audit_completed", auditCompletedProps("cli", result)); // No sessions on disk — guide the user instead of opening an empty dashboard. if (result.eventsScanned === 0) {