fix(audit): report the onboarding auto-audit to PostHog

CHANGELOG.md

@@ -9,6 +9,7 @@
 - Redirect the 13 AgentEye pages the upstream syncs deleted (`/agenteye/deployment`, `/agenteye/kubernetes-deployment`, `/agenteye/troubleshooting`, …) to a live page instead of hard-404ing. (#556)
 
 ### Fixes
+- Report the onboarding auto-audit to PostHog. `runPostSetupAudit()` — the audit that runs automatically at the end of first-run setup, and therefore the **first audit every new user ever runs** — emitted no telemetry at all, while an explicit `failproofai audit` reported `cli_audit_started` / `cli_audit_completed` / `cli_audit_failed`. First-run audits were invisible, so the audit funnel silently undercounted exactly the activation moment it exists to measure (confirmed live: a fresh install's auto-audit wrote its dashboard cache and reached PostHog with nothing). It now emits the same three events, tagged `source: "onboarding"` against the existing `source: "cli"`, so the two paths stay distinguishable. `cli_audit_completed` fires before the empty-history return, matching `runAuditCli`, so a fresh user with no agent history is still counted; `cli_audit_failed` is awaited because the function returns straight into the dashboard boot, which would otherwise race a fire-and-forget send. The completed-event properties are now built by one shared helper so the two entry points cannot drift. Onboarding remains best-effort: it never throws and never exits. (#562)
 - Delete translated pages whose English source no longer exists, and stop them coming back: `translate-docs` gained a `--prune` mode that runs by default on every translation pass (`--no-prune` opts out) and as an explicit step in the `consolidate` job — that job re-checks-out `main` and *overlays* the artifacts, so a prune done only in the per-language jobs would be undone. Translation only ever moved forward, so the 11 pages the AgentEye syncs removed upstream left 154 orphans (11 × 14 locales) that `--update-nav` dropped from the sidebar but Mintlify still served and indexed — non-English readers could land on docs for a deleted feature with no way out. A repo invariant test now fails if any translation outlives its English source. (#556)
 - Move the docs auto-translation daily cron from 06:00 UTC to 11:05 AM IST (05:35 UTC, encoded as `35 5 * * *` since GitHub Actions cron is always UTC). (#553)
 

__tests__/audit/audit-cli-telemetry.test.ts

@@ -32,7 +32,7 @@ vi.mock("../../src/audit/open-browser", () => ({ openWhenReady: h.openWhenReady
 vi.mock("../../scripts/launch", () => ({ launch: h.launch }));
 vi.mock("../../lib/telemetry-id", () => ({ getInstanceId: () => "test-instance" }));
 
-import { runAuditCli } from "../../src/audit/cli";
+import { runAuditCli, runPostSetupAudit } from "../../src/audit/cli";
 
 function result(over: Partial<AuditResult>): AuditResult {
   return {
@@ -129,3 +129,93 @@ describe("failproofai audit telemetry", () => {
     expect(exitInfo?.resolvedAtExit.has("cli_audit_failed")).toBe(true);
   });
 });
+
+/**
+ * The onboarding auto-audit — the FIRST audit any new user runs — previously
+ * emitted nothing at all, so first-run audits were invisible in PostHog while
+ * explicit `failproofai audit` runs reported normally.
+ */
+describe("post-setup (onboarding) audit telemetry", () => {
+  afterEach(() => {
+    delete process.env.FAILPROOFAI_NO_AUTO_AUDIT;
+  });
+
+  it("emits cli_audit_started then cli_audit_completed, tagged source=onboarding", async () => {
+    h.runAudit.mockResolvedValue(result({ eventsScanned: 100, totals: { hits: 2, projectsWithHits: 1 } }));
+
+    await runPostSetupAudit();
+
+    expect(names()).toEqual(["cli_audit_started", "cli_audit_completed"]);
+    expect(h.trackHookEvent).toHaveBeenCalledWith("test-instance", "cli_audit_started", {
+      source: "onboarding",
+    });
+    expect(h.trackHookEvent).toHaveBeenCalledWith("test-instance", "cli_audit_completed", {
+      source: "onboarding",
+      events_scanned: 100,
+      sessions_scanned: 3,
+      total_hits: 2,
+      findings: 0,
+    });
+  });
+
+  // source is what separates the onboarding run from a deliberate one in PostHog.
+  it("distinguishes itself from an explicit `failproofai audit` run", async () => {
+    h.runAudit.mockResolvedValue(result({}));
+
+    await runPostSetupAudit();
+    const onboarding = h.trackHookEvent.mock.calls.map((c) => (c[2] as { source: string }).source);
+
+    expect(new Set(onboarding)).toEqual(new Set(["onboarding"]));
+  });
+
+  it("emits cli_audit_failed when the scan throws, and still does not throw", async () => {
+    h.runAudit.mockRejectedValue(new TypeError("disk exploded"));
+
+    await expect(runPostSetupAudit()).resolves.toBeUndefined();
+
+    expect(names()).toEqual(["cli_audit_started", "cli_audit_failed"]);
+    expect(h.trackHookEvent).toHaveBeenCalledWith("test-instance", "cli_audit_failed", {
+      source: "onboarding",
+      error_type: "TypeError",
+      error_message: "disk exploded",
+    });
+  });
+
+  // Mirrors runAuditCli, which reports completed regardless of what it found —
+  // a fresh user with no agent history is exactly who we want counted.
+  it("still emits cli_audit_completed when there is no history to scan", async () => {
+    h.runAudit.mockResolvedValue(
+      result({ eventsScanned: 0, transcripts: { scanned: 0, skipped: 0, errors: 0, durationMs: 0 } }),
+    );
+
+    await runPostSetupAudit();
+
+    expect(names()).toEqual(["cli_audit_started", "cli_audit_completed"]);
+    expect(h.trackHookEvent).toHaveBeenCalledWith("test-instance", "cli_audit_completed", {
+      source: "onboarding",
+      events_scanned: 0,
+      sessions_scanned: 0,
+      total_hits: 0,
+      findings: 0,
+    });
+    // Nothing found — no point warming a cache or opening a dashboard.
+    expect(h.writeDashboardCache).not.toHaveBeenCalled();
+  });
+
+  it("emits nothing when the auto-audit is opted out", async () => {
+    process.env.FAILPROOFAI_NO_AUTO_AUDIT = "1";
+
+    await runPostSetupAudit();
+
+    expect(names()).toEqual([]);
+    expect(h.runAudit).not.toHaveBeenCalled();
+  });
+
+  it("never exits the process — onboarding must continue to the dashboard", async () => {
+    h.runAudit.mockResolvedValue(result({}));
+
+    await runPostSetupAudit();
+
+    expect(exitInfo).toBeNull();
+  });
+});

src/audit/cli.ts

@@ -237,6 +237,26 @@ function printSummary(result: AuditResult): void {
   for (const line of buildSummary(result)) process.stdout.write(`  ${line}\n`);
 }
 
+// ── Audit telemetry ──────────────────────────────────────────────────────────
+
+/**
+ * Which entry point ran the audit. `onboarding` is the automatic post-setup run;
+ * `cli` is an explicit `failproofai audit`. Carried on every cli_audit_* event so
+ * the first audit a user ever runs is distinguishable from later deliberate ones.
+ */
+type AuditSource = "cli" | "onboarding";
+
+/** Shared so both entry points report cli_audit_completed identically. */
+function auditCompletedProps(source: AuditSource, result: AuditResult) {
+  return {
+    source,
+    events_scanned: result.eventsScanned,
+    sessions_scanned: result.transcripts.scanned,
+    total_hits: result.totals.hits,
+    findings: result.results.length,
+  };
+}
+
 // ── Post-setup background audit ────────────────────────────────────────────────
 
 /**
@@ -252,21 +272,39 @@ function printSummary(result: AuditResult): void {
  */
 export async function runPostSetupAudit(): Promise<void> {
   if (process.env.FAILPROOFAI_NO_AUTO_AUDIT === "1") return;
-
+
+  const instanceId = getInstanceId();
+  // Fire-and-forget, as in runAuditCli: the multi-second scan below keeps the
+  // process alive long enough for this to land, and the completed/failed event
+  // that follows is awaited.
+  void trackHookEvent(instanceId, "cli_audit_started", { source: "onboarding" });
+
   process.stdout.write(
     `\n  ${c(PINK, "✦")} ${c(BOLD, "failproofai audit now running")}  ${c(DIM, "· ctrl+c to stop")}\n\n`,
   );
 
   let result: AuditResult;
   try {
     result = await runWithProgress({});
-  } catch {
+  } catch (err) {
+    // Awaited: this function returns straight into the dashboard boot, and a
+    // fire-and-forget fetch would race it.
+    await trackHookEvent(instanceId, "cli_audit_failed", {
+      source: "onboarding",
+      error_type: err instanceof Error ? err.name : "unknown",
+      error_message: sanitizeErrorMessage(err),
+    });
     process.stdout.write(
       `  ${c(PINK, "!")} ${c(DIM, "audit couldn't finish — run")} ${c(CYAN, "failproofai audit")} ${c(DIM, "later.")}\n\n`,
     );
     return;
   }
 
+  // Reported before the empty-history return below, so an onboarding audit that
+  // finds nothing is still counted — matching runAuditCli, which reports
+  // completed regardless of what the scan turned up.
+  await trackHookEvent(instanceId, "cli_audit_completed", auditCompletedProps("onboarding", result));
+
   if (result.eventsScanned === 0) {
     process.stdout.write(
       `\n  ${c(DIM, "no agent sessions to audit yet — come back after using your agent.")}\n\n`,
@@ -328,13 +366,7 @@ export async function runAuditCli(args: string[]): Promise<void> {
   // would otherwise drop this event. On the dashboard path launch() keeps the
   // process alive, but awaiting makes delivery reliable on every exit path.
   // Bounded (5s) and never throws.
-  await trackHookEvent(instanceId, "cli_audit_completed", {
-    source: "cli",
-    events_scanned: result.eventsScanned,
-    sessions_scanned: result.transcripts.scanned,
-    total_hits: result.totals.hits,
-    findings: result.results.length,
-  });
+  await trackHookEvent(instanceId, "cli_audit_completed", auditCompletedProps("cli", result));
 
   // No sessions on disk — guide the user instead of opening an empty dashboard.
   if (result.eventsScanned === 0) {