From 5731174293981a858f2d8197babfb54c5a897983 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Fri, 17 Jul 2026 14:17:33 +0530 Subject: [PATCH 1/5] fix: stop shipping npm lifecycle scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit npm 12 (released 2026-07-08, now the `latest` dist-tag) blocks dependency install scripts by default, so `npm install failproofai` printed: npm warn install-scripts 1 package had install scripts blocked ... npm warn install-scripts failproofai@... (postinstall: node scripts/postinstall.mjs) and skipped the script. npm 11.16+ prints the advisory `allow-scripts` variant and still runs it. `allowScripts` is purely consumer-side — a package declaring `allowScripts` or `trustedDependencies` for itself is ignored — so the only fix is to ship no install scripts. postinstall's install telemetry (first_install / version_changed / package_installed, same names and properties) now fires from the CLI's first non-hook invocation via lib/install-check.ts: at most once per version, and a no-op once the version is recorded. It is deliberately outside the --hook fast path in bin/failproofai.mjs, which runs on every tool call. trackInstallEvent now takes an explicit version. npm only sets npm_package_version inside a lifecycle script, so outside one it reported "unknown". The script's server.js check and shadowed-PATH diagnosis were redundant: scripts/launch.ts already does both at launch, with a better message. preuninstall is deleted as dead code — npm honours no uninstall lifecycle scripts (verified with a probe package: postinstall fired, preuninstall did not), so its hook cleanup has never run. Uninstalling therefore leaves __failproofai_hook__ entries behind; that gap now needs an explicit `failproofai policies --uninstall`. Behaviour changes: - package_installed measures install->activation, not raw installs. - Same-version reinstalls (direction: "reinstall") are no longer reported; the CLI cannot distinguish one from an ordinary invocation. - The install-time welcome message is dropped; the config wizard's first-run redirect already handles onboarding. This also recovers telemetry already dropped for bun, pnpm and Yarn >=4.14 users, which have blocked install scripts for some time. Verified against a local registry on npm 12: registry install is warning-free (exit 0), the CLI runs, first_install/package_installed fire on first run, nothing fires on the second, and version_changed fires on upgrade — all carrying the real version and the existing $lib/product stamps. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 + __tests__/lib/install-check.test.ts | 313 +++++++++++++++++++ __tests__/scripts/install-telemetry.test.ts | 26 +- __tests__/scripts/postinstall.test.ts | 208 ------------- __tests__/scripts/preuninstall.test.ts | 319 -------------------- bin/failproofai.mjs | 10 + docs/package-aliases.mdx | 2 +- lib/install-check.ts | 215 +++++++++++++ package.json | 2 - scripts/install-diagnosis.mjs | 13 +- scripts/install-telemetry.mjs | 16 +- scripts/postinstall.mjs | 242 --------------- scripts/preuninstall.mjs | 131 -------- 13 files changed, 580 insertions(+), 919 deletions(-) create mode 100644 __tests__/lib/install-check.test.ts delete mode 100644 __tests__/scripts/postinstall.test.ts delete mode 100644 __tests__/scripts/preuninstall.test.ts create mode 100644 lib/install-check.ts delete mode 100644 scripts/postinstall.mjs delete mode 100644 scripts/preuninstall.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index c64e43e3..0ef0231a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,8 @@ - Surface OpenClaw sessions in the dashboard's audit + history browser: reads the real JSONL transcripts at `~/.openclaw/agents//sessions/.jsonl` (skipping the heavy `.trajectory.jsonl` OTel traces) via `lib/openclaw-sessions.ts` (a pure, unit-tested type-discriminated parser that pairs assistant `toolCall` blocks with their `toolResult` by `toolCallId`) and `lib/openclaw-projects.ts` (reads the `sessions.json` index, groups by agentId into `openclaw-` projects, parses channel metadata from the sessionKey). Wired through `src/audit/cli-adapters/openclaw.ts`, `lib/cli-registry.ts` (teal badge), `lib/projects.ts`, `lib/download-session.ts` (real-file download — no synthesis needed), and the `app/project/[name]` routes. (#489) ### Fixes +- Stop shipping npm lifecycle scripts, removing the `npm warn` every install printed. npm 12 (`allowScripts`, released 2026-07-08 and now the `latest` dist-tag) blocks dependency install scripts by default, so `npm install failproofai` warned `1 package had install scripts blocked … failproofai@… (postinstall: node scripts/postinstall.mjs)` and silently skipped the script; npm 11.16+ prints the advisory `allow-scripts` variant and still runs it. `allowScripts` is purely consumer-side — a package cannot opt itself in (verified: a package declaring `allowScripts`/`trustedDependencies` for *itself* is ignored) — so the only fix is to ship no install scripts. The `postinstall` script's install telemetry (`first_install` / `version_changed` / `package_installed`, with identical event names and properties) now fires from the CLI's first non-hook invocation via `lib/install-check.ts`, reporting at most once per version and no-op'ing on the steady-state path; it is deliberately kept out of `bin/failproofai.mjs`'s `--hook` fast path, which runs on every tool call. This also *recovers* telemetry already being dropped for bun, pnpm, and Yarn ≥4.14 users, which have blocked install scripts for some time. The script's `server.js` check and shadowed-PATH diagnosis were redundant — `scripts/launch.ts` already performs both at launch, with a better error message. `package_installed` now measures install→activation rather than raw installs, and same-version reinstalls (`direction: "reinstall"`) are no longer reported, as the CLI has no signal to detect them. The install-time welcome message is dropped; the `config` wizard's first-run redirect already handles onboarding. (#560) +- Remove `scripts/preuninstall.mjs`, which never ran. npm honours no uninstall lifecycle scripts (verified with a probe package: `postinstall` fired, `preuninstall` did not), so the hook cleanup it appeared to perform has never happened — uninstalling failproofai leaves `__failproofai_hook__` entries behind in settings files. Removing the dead code makes the gap visible; cleanup needs an explicit `failproofai policies --uninstall` before removing the package. (#560) - Move the docs auto-translation off the per-push trigger onto a daily scheduled run (`cron: 0 6 * * *`, 06:00 UTC): running the full 14-language matrix on every doc commit to `main` was expensive, so a day's English-source edits are now coalesced into one run. The content-hash cache (`scripts/translate-docs/.translation-cache.json`) still limits token spend to the pages whose source actually changed since the last successful run — so most days translate only a handful of pages (or none) — and manual `workflow_dispatch` (with `force`/`languages`) is unchanged. (#547) - Fix the auto-translate job failing repeatedly on the large AgentEye docs. `translateContent` capped `max_tokens` at 16384, but the biggest English pages (e.g. `agenteye/kubernetes-deployment.mdx` at ~1400 lines) translate to well beyond that for verbose target languages — so the response was silently truncated mid-MDX, leaving an unbalanced `{` or an unterminated JSX expression that failed `mintlify validate` in the `consolidate` job (and, for the very largest pages, tripped the proxy with a "stream ended" error in the per-language `translate` job). Worse, the truncated output was written to disk and cached as if complete, so a re-run served the same broken page. Raises the ceiling to 64000 (Claude Haiku 4.5's max output for Tier 2/3 languages; well within Claude Sonnet 4.6's 128000 for Tier 1), overridable via the new `TRANSLATE_MAX_TOKENS` env var, and — as a backstop — throws when `stop_reason === "max_tokens"` so a truncated page is never written or cached and its language is excluded from the publish step instead of shipping malformed MDX. (#546) - Fix the Mintlify docs deployment failing on every `docs/i18n/README.*.md` translation. Mintlify parses each page as MDX, where the README's top-level HTML comment (``) is a hard syntax error — it rejects the leading `!` of `` inside code fences literal (e.g. the AgentEye collector plist sample) — and the 14 committed translations are regenerated to match. Also aligns `readme-translator.ts` with `translateMdxPage` by running `sanitizeJsxAttributes` over its JSX-bearing output. (#535) diff --git a/__tests__/lib/install-check.test.ts b/__tests__/lib/install-check.test.ts new file mode 100644 index 00000000..cbb8cc43 --- /dev/null +++ b/__tests__/lib/install-check.test.ts @@ -0,0 +1,313 @@ +// @vitest-environment node +/** + * Install reporting from the CLI, replacing the postinstall lifecycle script + * that package managers now block by default. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { resolve } from "node:path"; + +const FAKE_HOME = "/fake/home"; +const LAST_VERSION = resolve(FAKE_HOME, ".failproofai", "last-version"); +const HOOKS_CONFIG = resolve(FAKE_HOME, ".failproofai", "policies-config.json"); +const USER_SETTINGS = resolve(FAKE_HOME, ".claude", "settings.json"); + +vi.mock("node:fs", () => ({ + existsSync: vi.fn(), + readFileSync: vi.fn(), + writeFileSync: vi.fn(), + mkdirSync: vi.fn(), +})); + +vi.mock("node:os", () => ({ + homedir: vi.fn(() => FAKE_HOME), + platform: vi.fn(() => "linux"), + arch: vi.fn(() => "x64"), + release: vi.fn(() => "5.15.0"), + hostname: vi.fn(() => "test-host"), +})); + +vi.mock("../../scripts/install-telemetry.mjs", () => ({ + trackInstallEvent: vi.fn(() => Promise.resolve()), +})); + +const CONFIG_WITH_TWO_POLICIES = JSON.stringify({ + enabledPolicies: ["block-sudo", "block-rm-rf"], +}); + +const SETTINGS_WITH_MARKED_HOOK = JSON.stringify({ + hooks: { + PreToolUse: [ + { + hooks: [ + { type: "command", command: "failproofai --hook PreToolUse", __failproofai_hook__: true }, + ], + }, + ], + }, +}); + +const SETTINGS_WITHOUT_MARKED_HOOK = JSON.stringify({ hooks: {} }); + +/** Point the mocked fs at a given on-disk state. */ +async function setupFs(state: { + lastVersion?: string | null; + hooksConfigContent?: string; + settingsContent?: string; +}) { + const { existsSync, readFileSync } = await import("node:fs"); + vi.mocked(existsSync).mockImplementation((p) => { + if (p === LAST_VERSION) return state.lastVersion != null; + if (p === HOOKS_CONFIG) return state.hooksConfigContent != null; + if (p === USER_SETTINGS) return state.settingsContent != null; + return false; + }); + vi.mocked(readFileSync).mockImplementation(((p: string) => { + if (p === LAST_VERSION) return state.lastVersion ?? ""; + if (p === HOOKS_CONFIG) return state.hooksConfigContent ?? ""; + if (p === USER_SETTINGS) return state.settingsContent ?? ""; + throw new Error(`unexpected read: ${p}`); + }) as never); +} + +async function eventNames() { + const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); + return vi.mocked(trackInstallEvent).mock.calls.map((c) => c[0]); +} + +describe("maybeReportInstall", () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe("brand-new install (nothing recorded)", () => { + it("fires first_install and package_installed, but not version_changed", async () => { + await setupFs({ lastVersion: null }); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await maybeReportInstall("1.0.0"); + + expect(await eventNames()).toEqual(["first_install", "package_installed"]); + }); + + it("records the version so the next run stays silent", async () => { + await setupFs({ lastVersion: null }); + const { writeFileSync } = await import("node:fs"); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await maybeReportInstall("1.0.0"); + + expect(writeFileSync).toHaveBeenCalledWith(LAST_VERSION, "1.0.0", "utf8"); + }); + + it("stamps the caller's version on the event, never 'unknown'", async () => { + await setupFs({ lastVersion: null }); + const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await maybeReportInstall("0.0.14-beta.1"); + + for (const call of vi.mocked(trackInstallEvent).mock.calls) { + expect(call[2]).toMatchObject({ version: "0.0.14-beta.1" }); + } + }); + }); + + // The whole point of moving off postinstall: this runs on every CLI + // invocation, so the steady-state path must cost nothing and stay silent. + describe("steady state (version already recorded)", () => { + it("fires no events at all", async () => { + await setupFs({ lastVersion: "1.0.0" }); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await maybeReportInstall("1.0.0"); + + expect(await eventNames()).toEqual([]); + }); + + it("does not rewrite the version file", async () => { + await setupFs({ lastVersion: "1.0.0" }); + const { writeFileSync } = await import("node:fs"); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await maybeReportInstall("1.0.0"); + + expect(writeFileSync).not.toHaveBeenCalled(); + }); + }); + + describe("version change", () => { + it("reports an upgrade with from/to versions", async () => { + await setupFs({ lastVersion: "1.0.0" }); + const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await maybeReportInstall("1.1.0"); + + expect(trackInstallEvent).toHaveBeenCalledWith( + "version_changed", + expect.objectContaining({ + from_version: "1.0.0", + to_version: "1.1.0", + direction: "upgrade", + }), + expect.anything(), + ); + }); + + it("reports a downgrade", async () => { + await setupFs({ lastVersion: "2.0.0" }); + const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await maybeReportInstall("1.0.0"); + + expect(trackInstallEvent).toHaveBeenCalledWith( + "version_changed", + expect.objectContaining({ direction: "downgrade" }), + expect.anything(), + ); + }); + + it("treats a prerelease → release move as an upgrade", async () => { + await setupFs({ lastVersion: "0.0.14-beta.1" }); + const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await maybeReportInstall("0.0.14"); + + expect(trackInstallEvent).toHaveBeenCalledWith( + "version_changed", + expect.objectContaining({ direction: "upgrade" }), + expect.anything(), + ); + }); + }); + + describe("hook state carried on package_installed", () => { + it("reports configured + registered with the policy count", async () => { + await setupFs({ + lastVersion: null, + hooksConfigContent: CONFIG_WITH_TWO_POLICIES, + settingsContent: SETTINGS_WITH_MARKED_HOOK, + }); + const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await maybeReportInstall("1.0.0"); + + expect(trackInstallEvent).toHaveBeenCalledWith( + "package_installed", + expect.objectContaining({ + hooks_configured: true, + hooks_registered: true, + enabled_policy_count: 2, + }), + expect.anything(), + ); + }); + + it("reports configured but NOT registered when the marker is absent", async () => { + await setupFs({ + lastVersion: null, + hooksConfigContent: CONFIG_WITH_TWO_POLICIES, + settingsContent: SETTINGS_WITHOUT_MARKED_HOOK, + }); + const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await maybeReportInstall("1.0.0"); + + expect(trackInstallEvent).toHaveBeenCalledWith( + "package_installed", + expect.objectContaining({ hooks_configured: true, hooks_registered: false }), + expect.anything(), + ); + }); + + it("reports neither for a user with no config", async () => { + await setupFs({ lastVersion: null }); + const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await maybeReportInstall("1.0.0"); + + expect(trackInstallEvent).toHaveBeenCalledWith( + "package_installed", + expect.objectContaining({ + hooks_configured: false, + hooks_registered: false, + enabled_policy_count: 0, + }), + expect.anything(), + ); + }); + }); + + // Reporting sits in front of every CLI command — it must never take one down. + describe("failure containment", () => { + it("does not throw when the version file is unwritable", async () => { + await setupFs({ lastVersion: null }); + const { writeFileSync } = await import("node:fs"); + vi.mocked(writeFileSync).mockImplementation(() => { + throw new Error("EACCES"); + }); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await expect(maybeReportInstall("1.0.0")).resolves.toBeUndefined(); + }); + + it("does not throw when telemetry delivery rejects", async () => { + await setupFs({ lastVersion: null }); + const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); + vi.mocked(trackInstallEvent).mockRejectedValue(new Error("network down")); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await expect(maybeReportInstall("1.0.0")).resolves.toBeUndefined(); + }); + + it("still records the version when delivery fails, so it is not retried forever", async () => { + await setupFs({ lastVersion: null }); + const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); + const { writeFileSync } = await import("node:fs"); + vi.mocked(trackInstallEvent).mockRejectedValue(new Error("network down")); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await maybeReportInstall("1.0.0"); + + expect(writeFileSync).toHaveBeenCalledWith(LAST_VERSION, "1.0.0", "utf8"); + }); + + it("treats a corrupt policies-config as unconfigured rather than failing", async () => { + await setupFs({ lastVersion: null, hooksConfigContent: "{ not json" }); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await expect(maybeReportInstall("1.0.0")).resolves.toBeUndefined(); + expect(await eventNames()).toContain("package_installed"); + }); + }); +}); + +describe("compareSemver", () => { + it("orders release above the same version's prerelease (semver §11)", async () => { + const { compareSemver } = await import("../../lib/install-check"); + expect(compareSemver("0.0.14-beta.1", "0.0.14")).toBeLessThan(0); + expect(compareSemver("0.0.14", "0.0.14-beta.1")).toBeGreaterThan(0); + }); + + it("orders prerelease numbers numerically, not lexically", async () => { + const { compareSemver } = await import("../../lib/install-check"); + expect(compareSemver("1.0.0-beta.2", "1.0.0-beta.10")).toBeLessThan(0); + }); + + it("compares major/minor/patch numerically", async () => { + const { compareSemver } = await import("../../lib/install-check"); + expect(compareSemver("1.9.0", "1.10.0")).toBeLessThan(0); + expect(compareSemver("2.0.0", "1.99.99")).toBeGreaterThan(0); + expect(compareSemver("1.2.3", "1.2.3")).toBe(0); + }); +}); diff --git a/__tests__/scripts/install-telemetry.test.ts b/__tests__/scripts/install-telemetry.test.ts index 51eda844..431b75df 100644 --- a/__tests__/scripts/install-telemetry.test.ts +++ b/__tests__/scripts/install-telemetry.test.ts @@ -1,9 +1,9 @@ // @vitest-environment node /** - * Real-payload coverage for the npm-lifecycle telemetry choke point. - * package_installed / package_uninstalled both flow through trackInstallEvent, - * so asserting the fetch body here guarantees `product: failproofai-oss` is - * stamped on install/uninstall events too. + * Real-payload coverage for the install telemetry choke point. + * first_install / version_changed / package_installed all flow through + * trackInstallEvent, so asserting the fetch body here guarantees + * `product: failproofai-oss` is stamped on every install event. */ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { trackInstallEvent } from "../../scripts/install-telemetry.mjs"; @@ -40,4 +40,22 @@ describe("install-telemetry trackInstallEvent", () => { await trackInstallEvent("package_installed"); expect(fetchSpy).not.toHaveBeenCalled(); }); + + // npm only sets npm_package_version inside a lifecycle script. These events now + // fire from the CLI, so the caller supplies the version explicitly. + it("prefers an explicitly-passed version over npm_package_version", async () => { + await trackInstallEvent("first_install", {}, { version: "1.2.3" }); + + const body = JSON.parse(fetchSpy.mock.calls[0][1].body); + expect(body.properties.failproofai_version).toBe("1.2.3"); + }); + + it("reports the real version when npm_package_version is absent (the CLI case)", async () => { + delete process.env.npm_package_version; + await trackInstallEvent("first_install", {}, { version: "4.5.6" }); + + const body = JSON.parse(fetchSpy.mock.calls[0][1].body); + expect(body.properties.failproofai_version).toBe("4.5.6"); + expect(body.properties.failproofai_version).not.toBe("unknown"); + }); }); diff --git a/__tests__/scripts/postinstall.test.ts b/__tests__/scripts/postinstall.test.ts deleted file mode 100644 index b956f553..00000000 --- a/__tests__/scripts/postinstall.test.ts +++ /dev/null @@ -1,208 +0,0 @@ -// @vitest-environment node -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { resolve } from "node:path"; - -const FAKE_HOME = "/fake/home"; -const FAKE_CWD = "/fake/project"; -const HOOKS_CONFIG = resolve(FAKE_HOME, ".failproofai", "policies-config.json"); -const USER_SETTINGS = resolve(FAKE_HOME, ".claude", "settings.json"); -const SERVER_JS = resolve(FAKE_CWD, ".next", "standalone", "server.js"); - -vi.mock("node:fs", () => ({ - existsSync: vi.fn(), - readFileSync: vi.fn(), -})); - -vi.mock("node:os", () => ({ - homedir: vi.fn(() => FAKE_HOME), - platform: vi.fn(() => "linux"), - arch: vi.fn(() => "x64"), - release: vi.fn(() => "5.15.0"), - hostname: vi.fn(() => "test-host"), -})); - -vi.mock("../../scripts/install-telemetry.mjs", () => ({ - trackInstallEvent: vi.fn(() => Promise.resolve()), -})); - -vi.mock("../../scripts/install-diagnosis.mjs", () => ({ - diagnoseShadow: vi.fn(() => ({ shadowed: false })), -})); - -const CONFIG_WITH_TWO_POLICIES = JSON.stringify({ - enabledPolicies: ["block-sudo", "block-rm-rf"], -}); - -const SETTINGS_WITH_MARKED_HOOK = JSON.stringify({ - hooks: { - PreToolUse: [ - { - hooks: [ - { type: "command", command: "failproofai --hook PreToolUse", __failproofai_hook__: true }, - ], - }, - ], - }, -}); - -const SETTINGS_WITHOUT_MARKED_HOOK = JSON.stringify({ hooks: {} }); - -describe("postinstall script", () => { - let cwdSpy: ReturnType; - let exitSpy: ReturnType; - let consoleLogSpy: ReturnType; - let consoleWarnSpy: ReturnType; - - beforeEach(() => { - vi.resetModules(); - vi.clearAllMocks(); - cwdSpy = vi.spyOn(process, "cwd").mockReturnValue(FAKE_CWD); - exitSpy = vi.spyOn(process, "exit").mockImplementation((_code?: string | number | null) => { - throw new Error("process.exit called"); - }); - consoleLogSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - consoleWarnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); - // Must differ from cwd so the dev-context guard doesn't early-exit - process.env.INIT_CWD = "/some/other/dir"; - }); - - afterEach(() => { - delete process.env.INIT_CWD; - cwdSpy.mockRestore(); - exitSpy.mockRestore(); - consoleLogSpy.mockRestore(); - consoleWarnSpy.mockRestore(); - }); - - async function runPostinstall(fs: { - hooksConfigExists?: boolean; - settingsExists?: boolean; - hooksConfigContent?: string; - settingsContent?: string; - }) { - const { existsSync, readFileSync } = await import("node:fs"); - vi.mocked(existsSync).mockImplementation((p) => { - if (p === SERVER_JS) return true; - if (p === HOOKS_CONFIG) return fs.hooksConfigExists ?? false; - if (p === USER_SETTINGS) return fs.settingsExists ?? false; - return false; - }); - vi.mocked(readFileSync).mockImplementation(((p: string) => { - if (p === HOOKS_CONFIG) return fs.hooksConfigContent ?? ""; - if (p === USER_SETTINGS) return fs.settingsContent ?? ""; - // package.json read for shadow diagnosis — return a minimal stub - return JSON.stringify({ version: "0.0.0-test" }); - }) as never); - - await import("../../scripts/postinstall.mjs"); - } - - const allLogs = () => - consoleLogSpy.mock.calls.map((c: unknown[]) => String(c[0] ?? "")).join("\n"); - - describe("missing dashboard build (server.js absent)", () => { - it("tracks package_install_failed before exiting 1 (was silently invisible)", async () => { - const { existsSync } = await import("node:fs"); - vi.mocked(existsSync).mockReturnValue(false); // SERVER_JS missing - const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); - - await expect(import("../../scripts/postinstall.mjs")).rejects.toThrow("process.exit called"); - - expect(trackInstallEvent).toHaveBeenCalledWith( - "package_install_failed", - expect.objectContaining({ reason: "server_js_missing" }), - ); - expect(trackInstallEvent).not.toHaveBeenCalledWith("package_installed", expect.anything()); - expect(exitSpy).toHaveBeenCalledWith(1); - }); - }); - - describe("brand-new user (no config, no settings)", () => { - it("prints the configure setup prompt", async () => { - await runPostinstall({ hooksConfigExists: false }); - expect(allLogs()).toContain("Installed"); - expect(allLogs()).toContain("failproofai config"); - expect(allLogs()).toContain("FAILPROOFAI_NO_FIRST_RUN=1"); - }); - - it("does NOT print the existing hooks-not-registered warning", async () => { - await runPostinstall({ hooksConfigExists: false }); - expect(allLogs()).not.toContain("hooks config exists with enabled policies"); - }); - - it("fires package_installed with hooks_configured=false, hooks_registered=false", async () => { - const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); - await runPostinstall({ hooksConfigExists: false }); - expect(trackInstallEvent).toHaveBeenCalledWith( - "package_installed", - expect.objectContaining({ - hooks_configured: false, - hooks_registered: false, - enabled_policy_count: 0, - }), - ); - }); - }); - - describe("fully installed user (config + settings + registered)", () => { - it("prints nothing — no warning, no Next steps", async () => { - await runPostinstall({ - hooksConfigExists: true, - hooksConfigContent: CONFIG_WITH_TWO_POLICIES, - settingsExists: true, - settingsContent: SETTINGS_WITH_MARKED_HOOK, - }); - expect(allLogs()).not.toContain("Next steps"); - expect(allLogs()).not.toContain("hooks config exists with enabled policies"); - }); - - it("fires package_installed with hooks_configured=true, hooks_registered=true", async () => { - const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); - await runPostinstall({ - hooksConfigExists: true, - hooksConfigContent: CONFIG_WITH_TWO_POLICIES, - settingsExists: true, - settingsContent: SETTINGS_WITH_MARKED_HOOK, - }); - expect(trackInstallEvent).toHaveBeenCalledWith( - "package_installed", - expect.objectContaining({ - hooks_configured: true, - hooks_registered: true, - enabled_policy_count: 2, - }), - ); - }); - }); - - describe("config exists but hooks not registered", () => { - it("prints printHooksWarning, NOT the Next steps block", async () => { - await runPostinstall({ - hooksConfigExists: true, - hooksConfigContent: CONFIG_WITH_TWO_POLICIES, - settingsExists: true, - settingsContent: SETTINGS_WITHOUT_MARKED_HOOK, - }); - expect(allLogs()).toContain("hooks config exists with enabled policies"); - expect(allLogs()).not.toContain("Next steps"); - }); - - it("fires package_installed with hooks_configured=true, hooks_registered=false", async () => { - const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); - await runPostinstall({ - hooksConfigExists: true, - hooksConfigContent: CONFIG_WITH_TWO_POLICIES, - settingsExists: true, - settingsContent: SETTINGS_WITHOUT_MARKED_HOOK, - }); - expect(trackInstallEvent).toHaveBeenCalledWith( - "package_installed", - expect.objectContaining({ - hooks_configured: true, - hooks_registered: false, - enabled_policy_count: 2, - }), - ); - }); - }); -}); diff --git a/__tests__/scripts/preuninstall.test.ts b/__tests__/scripts/preuninstall.test.ts deleted file mode 100644 index 04a60411..00000000 --- a/__tests__/scripts/preuninstall.test.ts +++ /dev/null @@ -1,319 +0,0 @@ -// @vitest-environment node -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { resolve } from "node:path"; - -const FAKE_HOME = "/fake/home"; -const FAKE_CWD = "/fake/project"; - -vi.mock("node:fs", () => ({ - existsSync: vi.fn(), - readFileSync: vi.fn(), - writeFileSync: vi.fn(), -})); - -vi.mock("node:os", () => ({ - homedir: vi.fn(() => FAKE_HOME), - platform: vi.fn(() => "linux"), - arch: vi.fn(() => "x64"), -})); - -vi.mock("../../scripts/install-telemetry.mjs", () => ({ - trackInstallEvent: vi.fn(() => Promise.resolve()), -})); - -const USER_SETTINGS = resolve(FAKE_HOME, ".claude", "settings.json"); -const PROJECT_SETTINGS = resolve(FAKE_CWD, ".claude", "settings.json"); -const LOCAL_SETTINGS = resolve(FAKE_CWD, ".claude", "settings.local.json"); - -/** A settings object with one failproofai-marked hook under PreToolUse. */ -function settingsWithMarkedHook(): string { - return JSON.stringify({ - hooks: { - PreToolUse: [ - { - hooks: [ - { - type: "command", - command: '"/usr/local/bin/failproofai" --hook PreToolUse', - timeout: 60000, - __failproofai_hook__: true, - }, - ], - }, - ], - }, - }); -} - -describe("preuninstall script", () => { - let cwdSpy: ReturnType; - let exitSpy: ReturnType; - let consoleSpy: ReturnType; - - beforeEach(() => { - vi.resetModules(); - vi.clearAllMocks(); - cwdSpy = vi.spyOn(process, "cwd").mockReturnValue(FAKE_CWD); - exitSpy = vi.spyOn(process, "exit").mockImplementation((_code?: string | number | null) => { - throw new Error("process.exit called"); - }); - consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - process.env.INIT_CWD = "/some/other/dir"; - }); - - afterEach(() => { - delete process.env.INIT_CWD; - vi.restoreAllMocks(); - }); - - describe("dev context guard", () => { - it("removes hooks when INIT_CWD is not set (npm uninstall scenario)", async () => { - delete process.env.INIT_CWD; - const { existsSync, readFileSync, writeFileSync } = await import("node:fs"); - vi.mocked(existsSync).mockImplementation((p) => p === USER_SETTINGS); - vi.mocked(readFileSync).mockReturnValue(settingsWithMarkedHook()); - - await import("../../scripts/preuninstall.mjs"); - - expect(exitSpy).not.toHaveBeenCalled(); - expect(writeFileSync).toHaveBeenCalledOnce(); - const written = JSON.parse(vi.mocked(writeFileSync).mock.calls[0][1] as string); - expect(written.hooks).toBeUndefined(); - }); - - it("exits 0 when INIT_CWD equals cwd", async () => { - process.env.INIT_CWD = FAKE_CWD; - await expect(import("../../scripts/preuninstall.mjs")).rejects.toThrow("process.exit called"); - expect(exitSpy).toHaveBeenCalledWith(0); - const { writeFileSync } = await import("node:fs"); - expect(writeFileSync).not.toHaveBeenCalled(); - }); - }); - - describe("removeHooksFromFile", () => { - it("removes __failproofai_hook__ marked entries from user settings", async () => { - const { existsSync, readFileSync, writeFileSync } = await import("node:fs"); - vi.mocked(existsSync).mockImplementation((p) => p === USER_SETTINGS); - vi.mocked(readFileSync).mockReturnValue(settingsWithMarkedHook()); - - await import("../../scripts/preuninstall.mjs"); - - expect(writeFileSync).toHaveBeenCalledOnce(); - const [path, content] = vi.mocked(writeFileSync).mock.calls[0]; - expect(path).toBe(USER_SETTINGS); - const written = JSON.parse(content as string); - expect(written.hooks).toBeUndefined(); - }); - - it("removes legacy hooks via command string detection", async () => { - const legacySettings = JSON.stringify({ - hooks: { - PreToolUse: [ - { - hooks: [ - { - type: "command", - command: "failproofai --hook PreToolUse", - timeout: 60000, - // no __failproofai_hook__ marker - }, - ], - }, - ], - }, - }); - - const { existsSync, readFileSync, writeFileSync } = await import("node:fs"); - vi.mocked(existsSync).mockImplementation((p) => p === USER_SETTINGS); - vi.mocked(readFileSync).mockReturnValue(legacySettings); - - await import("../../scripts/preuninstall.mjs"); - - expect(writeFileSync).toHaveBeenCalledOnce(); - const written = JSON.parse(vi.mocked(writeFileSync).mock.calls[0][1] as string); - expect(written.hooks).toBeUndefined(); - }); - - it("preserves non-failproofai hooks in the same matcher", async () => { - const mixedSettings = JSON.stringify({ - hooks: { - PreToolUse: [ - { - hooks: [ - { - type: "command", - command: "some-other-tool --pre", - timeout: 30000, - }, - { - type: "command", - command: '"/usr/local/bin/failproofai" --hook PreToolUse', - timeout: 60000, - __failproofai_hook__: true, - }, - ], - }, - ], - }, - }); - - const { existsSync, readFileSync, writeFileSync } = await import("node:fs"); - vi.mocked(existsSync).mockImplementation((p) => p === USER_SETTINGS); - vi.mocked(readFileSync).mockReturnValue(mixedSettings); - - await import("../../scripts/preuninstall.mjs"); - - expect(writeFileSync).toHaveBeenCalledOnce(); - const written = JSON.parse(vi.mocked(writeFileSync).mock.calls[0][1] as string); - expect(written.hooks.PreToolUse[0].hooks).toHaveLength(1); - expect(written.hooks.PreToolUse[0].hooks[0].command).toBe("some-other-tool --pre"); - }); - - it("cleans up empty matchers, event types, and hooks object after removal", async () => { - const settings = JSON.stringify({ - hooks: { - PreToolUse: [ - { - hooks: [ - { - type: "command", - command: '"/bin/failproofai" --hook PreToolUse', - timeout: 60000, - __failproofai_hook__: true, - }, - ], - }, - ], - PostToolUse: [ - { - hooks: [ - { - type: "command", - command: '"/bin/failproofai" --hook PostToolUse', - timeout: 60000, - __failproofai_hook__: true, - }, - ], - }, - ], - }, - permissions: { allow: ["Bash"] }, - }); - - const { existsSync, readFileSync, writeFileSync } = await import("node:fs"); - vi.mocked(existsSync).mockImplementation((p) => p === USER_SETTINGS); - vi.mocked(readFileSync).mockReturnValue(settings); - - await import("../../scripts/preuninstall.mjs"); - - const written = JSON.parse(vi.mocked(writeFileSync).mock.calls[0][1] as string); - expect(written.hooks).toBeUndefined(); - expect(written.permissions).toEqual({ allow: ["Bash"] }); // other keys preserved - }); - - it("does not write the file when no failproofai hooks are present", async () => { - const cleanSettings = JSON.stringify({ - hooks: { - PreToolUse: [ - { hooks: [{ type: "command", command: "some-other-tool", timeout: 30000 }] }, - ], - }, - }); - - const { existsSync, readFileSync, writeFileSync } = await import("node:fs"); - vi.mocked(existsSync).mockReturnValue(true); - vi.mocked(readFileSync).mockReturnValue(cleanSettings); - - await import("../../scripts/preuninstall.mjs"); - - expect(writeFileSync).not.toHaveBeenCalled(); - }); - - it("skips files that do not exist", async () => { - const { existsSync, writeFileSync } = await import("node:fs"); - vi.mocked(existsSync).mockReturnValue(false); - - await import("../../scripts/preuninstall.mjs"); - - expect(writeFileSync).not.toHaveBeenCalled(); - }); - - it("handles corrupt JSON gracefully without throwing", async () => { - const { existsSync, readFileSync, writeFileSync } = await import("node:fs"); - vi.mocked(existsSync).mockReturnValue(true); - vi.mocked(readFileSync).mockReturnValue("not valid json {{{"); - - await expect(import("../../scripts/preuninstall.mjs")).resolves.toBeDefined(); - expect(writeFileSync).not.toHaveBeenCalled(); - }); - - it("removes hooks from all three scope files (user, project, local)", async () => { - const { existsSync, readFileSync, writeFileSync } = await import("node:fs"); - vi.mocked(existsSync).mockReturnValue(true); - vi.mocked(readFileSync).mockReturnValue(settingsWithMarkedHook()); - - await import("../../scripts/preuninstall.mjs"); - - const writtenPaths = vi.mocked(writeFileSync).mock.calls.map((c) => c[0] as string); - expect(writtenPaths).toContain(USER_SETTINGS); - expect(writtenPaths).toContain(PROJECT_SETTINGS); - expect(writtenPaths).toContain(LOCAL_SETTINGS); - }); - - it("deduplicates settings paths when cwd equals home directory", async () => { - cwdSpy.mockReturnValue(FAKE_HOME); - - const { existsSync, readFileSync, writeFileSync } = await import("node:fs"); - vi.mocked(existsSync).mockReturnValue(true); - vi.mocked(readFileSync).mockReturnValue(settingsWithMarkedHook()); - - await import("../../scripts/preuninstall.mjs"); - - const writtenPaths = vi.mocked(writeFileSync).mock.calls.map((c) => c[0] as string); - const uniquePaths = new Set(writtenPaths); - expect(uniquePaths.size).toBe(writtenPaths.length); // no path written twice - }); - }); - - describe("telemetry", () => { - it("sends package_uninstalled with hooks_removed count", async () => { - const { existsSync, readFileSync } = await import("node:fs"); - vi.mocked(existsSync).mockImplementation((p) => p === USER_SETTINGS); - vi.mocked(readFileSync).mockReturnValue(settingsWithMarkedHook()); - - await import("../../scripts/preuninstall.mjs"); - - const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); - expect(vi.mocked(trackInstallEvent)).toHaveBeenCalledWith( - "package_uninstalled", - expect.objectContaining({ hooks_removed: 1 }), - ); - }); - - it("sends hooks_removed: 0 when no hooks found", async () => { - const { existsSync } = await import("node:fs"); - vi.mocked(existsSync).mockReturnValue(false); - - await import("../../scripts/preuninstall.mjs"); - - const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); - expect(vi.mocked(trackInstallEvent)).toHaveBeenCalledWith( - "package_uninstalled", - expect.objectContaining({ hooks_removed: 0 }), - ); - }); - - it("includes platform and arch in telemetry payload", async () => { - const { existsSync } = await import("node:fs"); - vi.mocked(existsSync).mockReturnValue(false); - - await import("../../scripts/preuninstall.mjs"); - - const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); - expect(vi.mocked(trackInstallEvent)).toHaveBeenCalledWith( - "package_uninstalled", - expect.objectContaining({ platform: "linux", arch: "x64" }), - ); - }); - }); -}); diff --git a/bin/failproofai.mjs b/bin/failproofai.mjs index ff9ed1db..35f85279 100755 --- a/bin/failproofai.mjs +++ b/bin/failproofai.mjs @@ -124,6 +124,16 @@ if (hookIdx >= 0) { * Error → unexpected; shows message only, exits 2 */ async function runCli() { + // Report a fresh install / upgrade. Deliberately here rather than at module + // scope: everything above this point is the --hook fast path, which runs on + // every tool call. No-ops after the first run on a given version. + try { + const { maybeReportInstall } = await import("../lib/install-check"); + await maybeReportInstall(version); + } catch { + // never block a command on reporting + } + // --help / -h (only when not inside a subcommand that handles its own --help) const SUBCOMMANDS = ["policies", "policy", "auth", "audit", "config"]; if ((args.includes("--help") || args.includes("-h")) && !SUBCOMMANDS.includes(args[0])) { diff --git a/docs/package-aliases.mdx b/docs/package-aliases.mdx index 66f40160..a9be5626 100644 --- a/docs/package-aliases.mdx +++ b/docs/package-aliases.mdx @@ -70,7 +70,7 @@ npm info failproof Each alias package: -1. Lists `failproofai` as a dependency - so the real package (including its `postinstall` hook setup) runs on install +1. Lists `failproofai` as a dependency - so the real package is installed and its binary becomes available 2. Exposes a binary matching its own name (e.g. `failprof-ai`) that proxies all arguments to the `failproofai` binary The proxy is a two-line Node script; there is no logic, no network calls, and no data collection beyond what `failproofai` itself does. diff --git a/lib/install-check.ts b/lib/install-check.ts new file mode 100644 index 00000000..a5a089fb --- /dev/null +++ b/lib/install-check.ts @@ -0,0 +1,215 @@ +/** + * Install reporting, driven from the CLI instead of an npm lifecycle script. + * + * This used to live in `scripts/postinstall.mjs`. Every package manager now + * blocks dependency install scripts by default — npm 12 (`allowScripts`), pnpm, + * bun, and Yarn ≥4.14 — and a publisher cannot opt its own package in, so a + * postinstall hook is no longer a channel that reaches users. The first non-hook + * CLI invocation is. + * + * Two constraints shape this module: + * + * 1. It must never run on the `--hook` fast path in bin/failproofai.mjs. That + * path fires on every tool call of every agent session; the check belongs + * in runCli() only. + * 2. It must be free on the steady-state path. When the recorded version + * matches, this returns before touching anything else. + * + * Everything here is best-effort: telemetry must never break, delay, or fail a + * real command. + */ +import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs"; +import { resolve } from "node:path"; +import { platform, arch, release, homedir, hostname } from "node:os"; +import { createHmac } from "node:crypto"; +import { trackInstallEvent } from "../scripts/install-telemetry.mjs"; + +const FAILPROOFAI_HOOK_MARKER = "__failproofai_hook__"; +const NAMESPACE = "failproofai-telemetry-v1"; + +/** Cap on the once-per-version report so a hanging network never stalls a command. */ +const REPORT_TIMEOUT_MS = 2000; + +function hashToId(raw: string): string { + return createHmac("sha256", NAMESPACE).update(raw).digest("hex"); +} + +function lastVersionPath(): string { + return resolve(homedir(), ".failproofai", "last-version"); +} + +/** + * Semver ordering. A release outranks the same version carrying a prerelease + * tag (semver §11); within a prerelease, numeric identifiers rank below + * non-numeric ones. + */ +export function compareSemver(a: string, b: string): number { + const parse = (v: string) => { + const m = /^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$/.exec(v); + if (!m) return null; + return { nums: [Number(m[1]), Number(m[2]), Number(m[3])], pre: m[4] ?? null }; + }; + const pa = parse(a); + const pb = parse(b); + if (!pa || !pb) return a < b ? -1 : a > b ? 1 : 0; + for (let i = 0; i < 3; i++) { + if (pa.nums[i] !== pb.nums[i]) return pa.nums[i] < pb.nums[i] ? -1 : 1; + } + if (pa.pre === null && pb.pre === null) return 0; + if (pa.pre === null) return 1; + if (pb.pre === null) return -1; + const ax = pa.pre.split(/[.-]/); + const bx = pb.pre.split(/[.-]/); + for (let i = 0; i < Math.max(ax.length, bx.length); i++) { + const ai = ax[i], bi = bx[i]; + if (ai === undefined) return -1; + if (bi === undefined) return 1; + const aNum = /^\d+$/.test(ai), bNum = /^\d+$/.test(bi); + if (aNum && bNum) { + const d = Number(ai) - Number(bi); + if (d !== 0) return d < 0 ? -1 : 1; + } else if (aNum) { + return -1; + } else if (bNum) { + return 1; + } else if (ai !== bi) { + return ai < bi ? -1 : 1; + } + } + return 0; +} + +export interface HooksState { + configured: boolean; + registered: boolean; + policyCount: number; +} + +/** + * Snapshot of the user's hook configuration, carried as properties on + * package_installed. + * + * Only inspects Claude's settings, which understates `registered` for someone + * running failproofai against another CLI. Preserved as-is from the postinstall + * script so the event keeps its historical meaning. + */ +export function checkHooks(): HooksState { + const hooksConfigPath = resolve(homedir(), ".failproofai", "policies-config.json"); + if (!existsSync(hooksConfigPath)) { + return { configured: false, registered: false, policyCount: 0 }; + } + + let config; + try { + config = JSON.parse(readFileSync(hooksConfigPath, "utf8")); + } catch { + return { configured: false, registered: false, policyCount: 0 }; + } + + if (!Array.isArray(config.enabledPolicies) || config.enabledPolicies.length === 0) { + return { configured: false, registered: false, policyCount: 0 }; + } + + const policyCount = config.enabledPolicies.length; + const settingsPath = resolve(homedir(), ".claude", "settings.json"); + if (!existsSync(settingsPath)) return { configured: true, registered: false, policyCount }; + + let settings; + try { + settings = JSON.parse(readFileSync(settingsPath, "utf8")); + } catch { + return { configured: true, registered: false, policyCount }; + } + + if (!settings?.hooks) return { configured: true, registered: false, policyCount }; + + for (const matchers of Object.values(settings.hooks)) { + if (!Array.isArray(matchers)) continue; + for (const matcher of matchers) { + if (!matcher?.hooks) continue; + if (matcher.hooks.some((h: Record) => h[FAILPROOFAI_HOOK_MARKER] === true)) { + return { configured: true, registered: true, policyCount }; + } + } + } + + return { configured: true, registered: false, policyCount }; +} + +/** + * Report an install or upgrade, at most once per version. + * + * Fires first_install (nothing recorded yet) or version_changed (recorded + * version differs), then package_installed, then records `version`. A same-version + * run is a no-op — unlike the postinstall script this replaces, the CLI has no + * signal that would distinguish a reinstall from an ordinary invocation, so + * `direction: "reinstall"` is no longer reported. + */ +export async function maybeReportInstall(version: string): Promise { + try { + const versionFile = lastVersionPath(); + + let previousVersion: string | null = null; + try { + if (existsSync(versionFile)) { + previousVersion = readFileSync(versionFile, "utf8").trim() || null; + } + } catch { + // Unreadable marker — treat as first run rather than give up. + } + + // Steady state: same version already reported. Costs one stat() per command. + if (previousVersion === version) return; + + const hooks = checkHooks(); + + const events: Promise[] = []; + if (previousVersion === null) { + events.push( + trackInstallEvent("first_install", { + platform: platform(), + arch: arch(), + os_release: release(), + node_version: process.versions.node, + }, { version, timeoutMs: REPORT_TIMEOUT_MS }), + ); + } else { + const cmp = compareSemver(previousVersion, version); + events.push( + trackInstallEvent("version_changed", { + from_version: previousVersion, + to_version: version, + direction: cmp < 0 ? "upgrade" : "downgrade", + platform: platform(), + arch: arch(), + }, { version, timeoutMs: REPORT_TIMEOUT_MS }), + ); + } + + events.push( + trackInstallEvent("package_installed", { + platform: platform(), + arch: arch(), + os_release: release(), + node_version: process.versions.node, + hostname_hash: hashToId(hostname()), + hooks_configured: hooks.configured, + hooks_registered: hooks.registered, + enabled_policy_count: hooks.policyCount, + }, { version, timeoutMs: REPORT_TIMEOUT_MS }), + ); + + // Record before awaiting: a dropped event is cheaper than re-reporting the + // same version on every command because delivery kept failing. + try { + mkdirSync(resolve(homedir(), ".failproofai"), { recursive: true }); + writeFileSync(versionFile, version, "utf8"); + } catch { + // best-effort + } + + await Promise.all(events.map((p) => p.catch(() => {}))); + } catch { + // Reporting must never break a command. + } +} diff --git a/package.json b/package.json index 26e4cd49..15ffb382 100644 --- a/package.json +++ b/package.json @@ -30,8 +30,6 @@ "test": "vitest", "test:run": "vitest run", "lint": "eslint . --config eslint.config.mjs", - "postinstall": "node scripts/postinstall.mjs", - "preuninstall": "node scripts/preuninstall.mjs", "prepare": "bun run build", "test:e2e": "vitest run --config vitest.config.e2e.mts", "test:e2e:watch": "vitest --config vitest.config.e2e.mts", diff --git a/scripts/install-diagnosis.mjs b/scripts/install-diagnosis.mjs index 4eca8609..86c8cea1 100644 --- a/scripts/install-diagnosis.mjs +++ b/scripts/install-diagnosis.mjs @@ -4,11 +4,9 @@ * a `bun install -g failproofai` whose prefix sorts ahead of npm's on PATH. * * Used by: - * - scripts/postinstall.mjs — warn at install time so the customer never sees - * the misleading "missing build output" runtime error. - * - scripts/launch.ts — when .next/standalone/server.js is missing, - * produce a shadow-shaped error if the cause is a shadow rather than a - * genuinely broken build. + * - scripts/launch.ts — when .next/standalone/server.js is missing, produce a + * shadow-shaped error if the cause is a shadow rather than a genuinely + * broken build. * * Pure Node.js built-ins, no external dependencies. Every probe is wrapped in * try/catch — diagnoseShadow() is guaranteed not to throw. @@ -125,9 +123,8 @@ function buildRecommendation(pathFirstBin) { * * @param {{ selfPackageRoot: string, selfVersion: string | null }} self * The package root and version of the binary calling diagnoseShadow(). - * Callers (bin/failproofai.mjs, scripts/postinstall.mjs) already have these - * values; passing them in keeps the helper deterministic and free of - * import.meta.url assumptions. + * Callers (scripts/launch.ts) already have these values; passing them in keeps + * the helper deterministic and free of import.meta.url assumptions. */ export function diagnoseShadow(self) { const selfPackageRoot = (() => { diff --git a/scripts/install-telemetry.mjs b/scripts/install-telemetry.mjs index 4285f7c0..5e73386e 100644 --- a/scripts/install-telemetry.mjs +++ b/scripts/install-telemetry.mjs @@ -80,12 +80,20 @@ function getInstanceId() { /** * Track a named event to PostHog. No-op when telemetry is disabled. - * Uses process.env.npm_package_version (set automatically by npm in lifecycle scripts). + * + * `opts.version` must be supplied by callers outside an npm lifecycle script: + * npm only sets npm_package_version while running one, and these events now + * fire from the CLI (see lib/install-check.ts) because package managers block + * install scripts by default. + * + * @param {string} event + * @param {Record} [properties] + * @param {{ version?: string, timeoutMs?: number }} [opts] */ -export async function trackInstallEvent(event, properties = {}) { +export async function trackInstallEvent(event, properties = {}, opts = {}) { if (process.env.FAILPROOFAI_TELEMETRY_DISABLED === "1") return; - const version = process.env.npm_package_version ?? "unknown"; + const version = opts.version ?? process.env.npm_package_version ?? "unknown"; const body = JSON.stringify({ api_key: process.env.FAILPROOFAI_POSTHOG_KEY ?? API_KEY, event, @@ -106,7 +114,7 @@ export async function trackInstallEvent(event, properties = {}) { method: "POST", headers: { "Content-Type": "application/json" }, body, - signal: AbortSignal.timeout(5000), + signal: AbortSignal.timeout(opts.timeoutMs ?? 5000), } ); } diff --git a/scripts/postinstall.mjs b/scripts/postinstall.mjs deleted file mode 100644 index 60280eda..00000000 --- a/scripts/postinstall.mjs +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env node -/** - * postinstall script for the failproofai package. - * - * 1. Warns if hooks config exists but hooks are missing from Claude Code settings. - * 2. Tracks a package_installed telemetry event. - * - * No external dependencies — Node.js built-ins only. - */ -import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs"; -import { resolve } from "node:path"; -import { platform, arch, release, homedir, hostname } from "node:os"; -import { createHmac } from "node:crypto"; -import { trackInstallEvent } from "./install-telemetry.mjs"; -import { diagnoseShadow } from "./install-diagnosis.mjs"; - -// Skip when running in development context (e.g. `bun install` in the source repo). -// INIT_CWD is set by npm/bun to the directory where install was invoked; it differs -// from process.cwd() only when we are being installed as a dependency by someone else. -if (!process.env.INIT_CWD || process.env.INIT_CWD === process.cwd()) process.exit(0); - -// Verify server.js exists — fail the install early if the dashboard build is missing. -const serverJsPath = resolve(process.cwd(), ".next", "standalone", "server.js"); -if (!existsSync(serverJsPath)) { - console.error( - `\n[failproofai] Error: server.js not found at:\n ${serverJsPath}\n\n` + - ` The package may not have been built correctly.\n` + - ` Try reinstalling: npm install -g failproofai@latest\n` - ); - // Await so the event lands before process.exit(1) kills the in-flight fetch — - // a failed install is exactly the signal we most want and would otherwise lose. - await trackInstallEvent("package_install_failed", { - reason: "server_js_missing", - platform: platform(), - arch: arch(), - node_version: process.versions.node, - }).catch(() => {}); - process.exit(1); -} - -// Detect when an older `failproofai` is shadowing this fresh install on PATH — -// classic case is a leftover `bun link` from a prior dev session, or a -// `bun install -g` whose ~/.bun/bin sorts ahead of npm's prefix. Without this -// warning the user only finds out later via a confusing runtime error from -// scripts/launch.ts pointing at the *old* install's missing build output. -try { - let selfVersion = null; - try { - selfVersion = JSON.parse(readFileSync(resolve(process.cwd(), "package.json"), "utf8")).version ?? null; - } catch {} - const diag = diagnoseShadow({ selfPackageRoot: process.cwd(), selfVersion }); - if (diag.shadowed) { - console.warn( - `\n[failproofai] Warning: another failproofai install is earlier on your PATH.\n` + - ` Just installed: ${diag.selfPackageRoot}` + (diag.selfVersion ? ` (v${diag.selfVersion})` : "") + `\n` + - ` PATH resolves : ${diag.pathFirstPath}` + (diag.pathFirstVersion ? ` (v${diag.pathFirstVersion})` : "") + `\n\n` + - ` Your shell will run the older copy. Remove the shadow with:\n` + - ` ${diag.recommendation}\n` - ); - } -} catch { - // Diagnosis is best-effort — never fail the install over a warning. -} - -const FAILPROOFAI_HOOK_MARKER = "__failproofai_hook__"; -const NAMESPACE = "failproofai-telemetry-v1"; - -function hashToId(raw) { - return createHmac("sha256", NAMESPACE).update(raw).digest("hex"); -} - -/** - * Returns the current hooks configuration state. - * @returns {{ configured: boolean, registered: boolean, policyCount: number }} - */ -function checkHooks() { - const hooksConfigPath = resolve(homedir(), ".failproofai", "policies-config.json"); - if (!existsSync(hooksConfigPath)) { - return { configured: false, registered: false, policyCount: 0 }; - } - - let config; - try { - config = JSON.parse(readFileSync(hooksConfigPath, "utf8")); - } catch { - return { configured: false, registered: false, policyCount: 0 }; - } - - if (!Array.isArray(config.enabledPolicies) || config.enabledPolicies.length === 0) { - return { configured: false, registered: false, policyCount: 0 }; - } - - const policyCount = config.enabledPolicies.length; - - // Check if Claude Code settings have failproofai hooks - const settingsPath = resolve(homedir(), ".claude", "settings.json"); - if (!existsSync(settingsPath)) { - printHooksWarning(); - return { configured: true, registered: false, policyCount }; - } - - let settings; - try { - settings = JSON.parse(readFileSync(settingsPath, "utf8")); - } catch { - printHooksWarning(); - return { configured: true, registered: false, policyCount }; - } - - if (!settings.hooks) { - printHooksWarning(); - return { configured: true, registered: false, policyCount }; - } - - // Walk settings.hooks looking for failproofai entries - for (const matchers of Object.values(settings.hooks)) { - if (!Array.isArray(matchers)) continue; - for (const matcher of matchers) { - if (!matcher.hooks) continue; - if (matcher.hooks.some((h) => h[FAILPROOFAI_HOOK_MARKER] === true)) { - return { configured: true, registered: true, policyCount }; - } - } - } - - printHooksWarning(); - return { configured: true, registered: false, policyCount }; -} - -function printHooksWarning() { - console.log( - `\n[failproofai] Warning: hooks config exists with enabled policies, but hooks are not registered in Claude Code settings.\n` + - ` To re-register hooks, run:\n` + - ` failproofai policies --uninstall && failproofai policies --install\n` - ); -} - -let hooksResult = { configured: false, registered: false, policyCount: 0 }; -try { - hooksResult = checkHooks(); -} catch { - // Non-critical — don't fail the install -} - -if (!hooksResult.configured && !hooksResult.registered) { - console.log( - `\n[failproofai] Installed ✓\n` + - ` 👋 Hi! Run \`failproofai config\` to set it up.\n` + - ` (Disable this prompt: FAILPROOFAI_NO_FIRST_RUN=1)\n` - ); -} - -// First-run + version_changed detection. The presence of ~/.failproofai/last-version -// is a stable signal: written on every postinstall, absent before the first one. -// Cannot piggy-back on instance-id because most users hit Tier 2 (OS machine ID) -// and never create that file. -// -// Semver comparison: a release (no prerelease tag) is greater than the same -// version with a prerelease tag (semver §11). Inside the prerelease, numeric -// identifiers are lower than non-numeric ones of the same length. -function compareSemver(a, b) { - const parse = (v) => { - const m = /^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$/.exec(v); - if (!m) return null; - return { nums: [Number(m[1]), Number(m[2]), Number(m[3])], pre: m[4] ?? null }; - }; - const pa = parse(a); - const pb = parse(b); - if (!pa || !pb) return a < b ? -1 : a > b ? 1 : 0; - for (let i = 0; i < 3; i++) { - if (pa.nums[i] !== pb.nums[i]) return pa.nums[i] < pb.nums[i] ? -1 : 1; - } - if (pa.pre === null && pb.pre === null) return 0; - if (pa.pre === null) return 1; - if (pb.pre === null) return -1; - const ax = pa.pre.split(/[.-]/); - const bx = pb.pre.split(/[.-]/); - for (let i = 0; i < Math.max(ax.length, bx.length); i++) { - const ai = ax[i], bi = bx[i]; - if (ai === undefined) return -1; - if (bi === undefined) return 1; - const aNum = /^\d+$/.test(ai), bNum = /^\d+$/.test(bi); - if (aNum && bNum) { - const d = Number(ai) - Number(bi); - if (d !== 0) return d < 0 ? -1 : 1; - } else if (aNum) { - return -1; - } else if (bNum) { - return 1; - } else if (ai !== bi) { - return ai < bi ? -1 : 1; - } - } - return 0; -} - -const currentVersion = process.env.npm_package_version ?? "unknown"; -const lastVersionFile = resolve(homedir(), ".failproofai", "last-version"); -let previousVersion = null; -try { - if (existsSync(lastVersionFile)) { - previousVersion = readFileSync(lastVersionFile, "utf8").trim() || null; - } -} catch {} - -if (previousVersion === null) { - trackInstallEvent("first_install", { - platform: platform(), - arch: arch(), - os_release: release(), - node_version: process.versions.node, - // `version` is carried automatically as `failproofai_version` — no explicit dup. - }).catch(() => {}); -} else { - // Same version is a reinstall — still worth tracking; users hitting `npm install -g` - // repeatedly is itself signal. Drop the `!==` guard so cmp===0 reaches the event. - const cmp = compareSemver(previousVersion, currentVersion); - trackInstallEvent("version_changed", { - from_version: previousVersion, - to_version: currentVersion, - direction: cmp < 0 ? "upgrade" : cmp > 0 ? "downgrade" : "reinstall", - platform: platform(), - arch: arch(), - }).catch(() => {}); -} - -try { - mkdirSync(resolve(homedir(), ".failproofai"), { recursive: true }); - writeFileSync(lastVersionFile, currentVersion, "utf8"); -} catch {} - -// Telemetry (best-effort, fire-and-forget) -trackInstallEvent("package_installed", { - platform: platform(), - arch: arch(), - os_release: release(), - node_version: process.versions.node, - hostname_hash: hashToId(hostname()), - hooks_configured: hooksResult.configured, - hooks_registered: hooksResult.registered, - enabled_policy_count: hooksResult.policyCount, -}).catch(() => {}); diff --git a/scripts/preuninstall.mjs b/scripts/preuninstall.mjs deleted file mode 100644 index cf73a6c6..00000000 --- a/scripts/preuninstall.mjs +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env node -/** - * preuninstall script for the failproofai package. - * - * Removes failproofai hook entries from all reachable Claude Code settings files: - * - ~/.claude/settings.json (user scope — always attempted) - * - {cwd}/.claude/settings.json (project scope — if it exists) - * - {cwd}/.claude/settings.local.json (local scope — if it exists) - * - * Does NOT delete ~/.failproofai/ (preserves cache, hooks-config, instance-id). - * - * Never exits non-zero — uninstall must not be blocked by cleanup failures. - * No external dependencies — Node.js built-ins only. - */ -import { readFileSync, writeFileSync, existsSync } from "node:fs"; -import { resolve } from "node:path"; -import { homedir, platform, arch } from "node:os"; -import { trackInstallEvent } from "./install-telemetry.mjs"; - -// Skip when running in development context (same guard as postinstall.mjs). -if (process.env.INIT_CWD && process.env.INIT_CWD === process.cwd()) process.exit(0); - -const FAILPROOFAI_HOOK_MARKER = "__failproofai_hook__"; - -/** - * Remove all failproofai-marked hook entries from a single settings file. - * Returns the number of hook entries removed. - * Writes the file only when at least one hook was removed. - */ -function removeHooksFromFile(settingsPath) { - if (!existsSync(settingsPath)) return 0; - - let settings; - try { - settings = JSON.parse(readFileSync(settingsPath, "utf8")); - } catch { - return 0; // Corrupt or unreadable — nothing to do - } - - if (!settings?.hooks) return 0; - - let hooksRemoved = 0; - - for (const eventType of Object.keys(settings.hooks)) { - const matchers = settings.hooks[eventType]; - if (!Array.isArray(matchers)) continue; - - for (let i = matchers.length - 1; i >= 0; i--) { - const matcher = matchers[i]; - if (!matcher.hooks) continue; - - const before = matcher.hooks.length; - matcher.hooks = matcher.hooks.filter((h) => { - if (h[FAILPROOFAI_HOOK_MARKER] === true) return false; // marked entry - // Fallback for legacy installs that predate the marker - const cmd = typeof h.command === "string" ? h.command : ""; - if (cmd.includes("failproofai") && cmd.includes("--hook")) return false; - return true; - }); - hooksRemoved += before - matcher.hooks.length; - - // Remove now-empty matchers - if (matcher.hooks.length === 0) { - matchers.splice(i, 1); - } - } - - // Remove now-empty event type arrays - if (matchers.length === 0) { - delete settings.hooks[eventType]; - } - } - - // Remove now-empty hooks object - if (Object.keys(settings.hooks).length === 0) { - delete settings.hooks; - } - - if (hooksRemoved > 0) { - try { - writeFileSync(settingsPath, JSON.stringify(settings, null, 2) + "\n", "utf8"); - } catch { - // Best-effort — don't block uninstall - } - } - - return hooksRemoved; -} - -let totalRemoved = 0; - -try { - const home = homedir(); - const projectCwd = process.cwd(); - - // Build list of settings files to clean, deduped in case cwd === home - const candidates = [ - resolve(home, ".claude", "settings.json"), // user scope - resolve(projectCwd, ".claude", "settings.json"), // project scope - resolve(projectCwd, ".claude", "settings.local.json"), // local scope - ]; - const seen = new Set(); - const settingsPaths = candidates.filter((p) => { - if (seen.has(p)) return false; - seen.add(p); - return true; - }); - - for (const settingsPath of settingsPaths) { - const removed = removeHooksFromFile(settingsPath); - if (removed > 0) { - console.log(`[failproofai] Removed ${removed} hook(s) from ${settingsPath}.`); - totalRemoved += removed; - } - } - - if (totalRemoved === 0) { - console.log("[failproofai] No hook entries found to remove."); - } -} catch { - // Never block uninstall -} - -// Telemetry — best-effort, awaited so the process stays alive long enough to send -try { - await trackInstallEvent("package_uninstalled", { - platform: platform(), - arch: arch(), - hooks_removed: totalRemoved, - }); -} catch {} From 479d33e08ddebb8a93f9b8eab2ee3361037453b1 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Fri, 17 Jul 2026 15:47:29 +0530 Subject: [PATCH 2/5] fix(audit): report the onboarding auto-audit to PostHog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runPostSetupAudit() — the audit that runs automatically at the end of first-run setup, and therefore the first audit any new user ever runs — emitted no telemetry at all, while an explicit `failproofai audit` reported cli_audit_started / cli_audit_completed / cli_audit_failed. First-run audits were invisible, so the audit funnel silently undercounted exactly the activation moment it exists to measure. Confirmed live: a fresh install's auto-audit wrote its dashboard cache and PostHog saw nothing. It now emits the same three events tagged `source: "onboarding"`, against the existing `source: "cli"`, so the two paths stay distinguishable. - cli_audit_completed fires before the empty-history return, matching runAuditCli, so a fresh user with no agent history is still counted. - cli_audit_failed is awaited: this function returns straight into the dashboard boot, which would race a fire-and-forget send. - The completed-event properties now come from one shared helper so the two entry points cannot drift apart. Onboarding stays best-effort — never throws, never exits. Verified by driving the real (unmocked) runPostSetupAudit against a local capture server: cli_audit_started and cli_audit_completed both land with source=onboarding and real counts (7,468 tool calls / 210 sessions), and the function returns without throwing or exiting. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 1 + __tests__/audit/audit-cli-telemetry.test.ts | 92 ++++++++++++++++++++- src/audit/cli.ts | 50 +++++++++-- 3 files changed, 133 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ef0231a..717d68a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ ### Fixes - Stop this repo's dogfood hooks from silently no-op'ing when `bun` isn't on the hook's PATH. All 75 hook commands across the 8 project-level agent-CLI configs fired `bun bin/failproofai.mjs --hook ` directly, so whenever the agent CLI was handed a PATH without bun — `npm i -g bun` installs into a single nvm version's bin dir, so `nvm use ` drops it; a macOS GUI launch inherits a launchd PATH built without `~/.zshrc` — every event died with exit 127 and the session ran with **zero** policy enforcement, saying nothing. The configs now call a dev-only `node scripts/dev-hook.mjs` launcher that locates bun across `PATH`, `$BUN_INSTALL/bin`, `~/.bun/bin`, the node execPath sibling, Homebrew/`/usr/local`, and every `~/.nvm/versions/node/*/bin`; installs it via npm if it is genuinely absent; builds `dist/index.js` when missing so `.failproofai/policies/*.mjs` can resolve `import ... from 'failproofai'`; then re-execs the real binary with `stdio: "inherit"` (byte-exact stdout — the deny contracts are JSON on stdout) and propagates the exit code verbatim (2 still means deny; signals map to 128+signum as `sh` did). A `command -v node` pre-check fronts each command so a missing node is a loud one-liner rather than silence — exit 2 on tool events, exit 1 on stop-class events, where exit 2 would mean "retry" and loop forever. The `.opencode` dev shim shares the same resolver and no longer reports a never-run hook as `exitCode: 0` (a silent allow). Production users on `npx -y failproofai` are unaffected. A new drift-guard test reads every committed dogfood config and asserts the launcher form, guard, exit-code split, and per-file command counts — these configs are generated by nothing and read by nothing, which is how #337's opencode shim drifted and silently no-op'd `block-read-outside-cwd` repo-wide. (#564) +- Report the onboarding auto-audit to PostHog. `runPostSetupAudit()` — the audit that runs automatically at the end of first-run setup, and therefore the **first audit every new user ever runs** — emitted no telemetry at all, while an explicit `failproofai audit` reported `cli_audit_started` / `cli_audit_completed` / `cli_audit_failed`. First-run audits were invisible, so the audit funnel silently undercounted exactly the activation moment it exists to measure (confirmed live: a fresh install's auto-audit wrote its dashboard cache and reached PostHog with nothing). It now emits the same three events, tagged `source: "onboarding"` against the existing `source: "cli"`, so the two paths stay distinguishable. `cli_audit_completed` fires before the empty-history return, matching `runAuditCli`, so a fresh user with no agent history is still counted; `cli_audit_failed` is awaited because the function returns straight into the dashboard boot, which would otherwise race a fire-and-forget send. The completed-event properties are now built by one shared helper so the two entry points cannot drift. Onboarding remains best-effort: it never throws and never exits. (#562) - Delete translated pages whose English source no longer exists, and stop them coming back: `translate-docs` gained a `--prune` mode that runs by default on every translation pass (`--no-prune` opts out) and as an explicit step in the `consolidate` job — that job re-checks-out `main` and *overlays* the artifacts, so a prune done only in the per-language jobs would be undone. Translation only ever moved forward, so the 11 pages the AgentEye syncs removed upstream left 154 orphans (11 × 14 locales) that `--update-nav` dropped from the sidebar but Mintlify still served and indexed — non-English readers could land on docs for a deleted feature with no way out. A repo invariant test now fails if any translation outlives its English source. (#556) - Move the docs auto-translation daily cron from 06:00 UTC to 11:05 AM IST (05:35 UTC, encoded as `35 5 * * *` since GitHub Actions cron is always UTC). (#553) diff --git a/__tests__/audit/audit-cli-telemetry.test.ts b/__tests__/audit/audit-cli-telemetry.test.ts index f000423c..06a15be3 100644 --- a/__tests__/audit/audit-cli-telemetry.test.ts +++ b/__tests__/audit/audit-cli-telemetry.test.ts @@ -32,7 +32,7 @@ vi.mock("../../src/audit/open-browser", () => ({ openWhenReady: h.openWhenReady vi.mock("../../scripts/launch", () => ({ launch: h.launch })); vi.mock("../../lib/telemetry-id", () => ({ getInstanceId: () => "test-instance" })); -import { runAuditCli } from "../../src/audit/cli"; +import { runAuditCli, runPostSetupAudit } from "../../src/audit/cli"; function result(over: Partial): AuditResult { return { @@ -129,3 +129,93 @@ describe("failproofai audit telemetry", () => { expect(exitInfo?.resolvedAtExit.has("cli_audit_failed")).toBe(true); }); }); + +/** + * The onboarding auto-audit — the FIRST audit any new user runs — previously + * emitted nothing at all, so first-run audits were invisible in PostHog while + * explicit `failproofai audit` runs reported normally. + */ +describe("post-setup (onboarding) audit telemetry", () => { + afterEach(() => { + delete process.env.FAILPROOFAI_NO_AUTO_AUDIT; + }); + + it("emits cli_audit_started then cli_audit_completed, tagged source=onboarding", async () => { + h.runAudit.mockResolvedValue(result({ eventsScanned: 100, totals: { hits: 2, projectsWithHits: 1 } })); + + await runPostSetupAudit(); + + expect(names()).toEqual(["cli_audit_started", "cli_audit_completed"]); + expect(h.trackHookEvent).toHaveBeenCalledWith("test-instance", "cli_audit_started", { + source: "onboarding", + }); + expect(h.trackHookEvent).toHaveBeenCalledWith("test-instance", "cli_audit_completed", { + source: "onboarding", + events_scanned: 100, + sessions_scanned: 3, + total_hits: 2, + findings: 0, + }); + }); + + // source is what separates the onboarding run from a deliberate one in PostHog. + it("distinguishes itself from an explicit `failproofai audit` run", async () => { + h.runAudit.mockResolvedValue(result({})); + + await runPostSetupAudit(); + const onboarding = h.trackHookEvent.mock.calls.map((c) => (c[2] as { source: string }).source); + + expect(new Set(onboarding)).toEqual(new Set(["onboarding"])); + }); + + it("emits cli_audit_failed when the scan throws, and still does not throw", async () => { + h.runAudit.mockRejectedValue(new TypeError("disk exploded")); + + await expect(runPostSetupAudit()).resolves.toBeUndefined(); + + expect(names()).toEqual(["cli_audit_started", "cli_audit_failed"]); + expect(h.trackHookEvent).toHaveBeenCalledWith("test-instance", "cli_audit_failed", { + source: "onboarding", + error_type: "TypeError", + error_message: "disk exploded", + }); + }); + + // Mirrors runAuditCli, which reports completed regardless of what it found — + // a fresh user with no agent history is exactly who we want counted. + it("still emits cli_audit_completed when there is no history to scan", async () => { + h.runAudit.mockResolvedValue( + result({ eventsScanned: 0, transcripts: { scanned: 0, skipped: 0, errors: 0, durationMs: 0 } }), + ); + + await runPostSetupAudit(); + + expect(names()).toEqual(["cli_audit_started", "cli_audit_completed"]); + expect(h.trackHookEvent).toHaveBeenCalledWith("test-instance", "cli_audit_completed", { + source: "onboarding", + events_scanned: 0, + sessions_scanned: 0, + total_hits: 0, + findings: 0, + }); + // Nothing found — no point warming a cache or opening a dashboard. + expect(h.writeDashboardCache).not.toHaveBeenCalled(); + }); + + it("emits nothing when the auto-audit is opted out", async () => { + process.env.FAILPROOFAI_NO_AUTO_AUDIT = "1"; + + await runPostSetupAudit(); + + expect(names()).toEqual([]); + expect(h.runAudit).not.toHaveBeenCalled(); + }); + + it("never exits the process — onboarding must continue to the dashboard", async () => { + h.runAudit.mockResolvedValue(result({})); + + await runPostSetupAudit(); + + expect(exitInfo).toBeNull(); + }); +}); diff --git a/src/audit/cli.ts b/src/audit/cli.ts index a4ed2855..c1831974 100644 --- a/src/audit/cli.ts +++ b/src/audit/cli.ts @@ -237,6 +237,26 @@ function printSummary(result: AuditResult): void { for (const line of buildSummary(result)) process.stdout.write(` ${line}\n`); } +// ── Audit telemetry ────────────────────────────────────────────────────────── + +/** + * Which entry point ran the audit. `onboarding` is the automatic post-setup run; + * `cli` is an explicit `failproofai audit`. Carried on every cli_audit_* event so + * the first audit a user ever runs is distinguishable from later deliberate ones. + */ +type AuditSource = "cli" | "onboarding"; + +/** Shared so both entry points report cli_audit_completed identically. */ +function auditCompletedProps(source: AuditSource, result: AuditResult) { + return { + source, + events_scanned: result.eventsScanned, + sessions_scanned: result.transcripts.scanned, + total_hits: result.totals.hits, + findings: result.results.length, + }; +} + // ── Post-setup background audit ──────────────────────────────────────────────── /** @@ -252,7 +272,13 @@ function printSummary(result: AuditResult): void { */ export async function runPostSetupAudit(): Promise { if (process.env.FAILPROOFAI_NO_AUTO_AUDIT === "1") return; - + + const instanceId = getInstanceId(); + // Fire-and-forget, as in runAuditCli: the multi-second scan below keeps the + // process alive long enough for this to land, and the completed/failed event + // that follows is awaited. + void trackHookEvent(instanceId, "cli_audit_started", { source: "onboarding" }); + process.stdout.write( `\n ${c(PINK, "✦")} ${c(BOLD, "failproofai audit now running")} ${c(DIM, "· ctrl+c to stop")}\n\n`, ); @@ -260,13 +286,25 @@ export async function runPostSetupAudit(): Promise { let result: AuditResult; try { result = await runWithProgress({}); - } catch { + } catch (err) { + // Awaited: this function returns straight into the dashboard boot, and a + // fire-and-forget fetch would race it. + await trackHookEvent(instanceId, "cli_audit_failed", { + source: "onboarding", + error_type: err instanceof Error ? err.name : "unknown", + error_message: sanitizeErrorMessage(err), + }); process.stdout.write( ` ${c(PINK, "!")} ${c(DIM, "audit couldn't finish — run")} ${c(CYAN, "failproofai audit")} ${c(DIM, "later.")}\n\n`, ); return; } + // Reported before the empty-history return below, so an onboarding audit that + // finds nothing is still counted — matching runAuditCli, which reports + // completed regardless of what the scan turned up. + await trackHookEvent(instanceId, "cli_audit_completed", auditCompletedProps("onboarding", result)); + if (result.eventsScanned === 0) { process.stdout.write( `\n ${c(DIM, "no agent sessions to audit yet — come back after using your agent.")}\n\n`, @@ -328,13 +366,7 @@ export async function runAuditCli(args: string[]): Promise { // would otherwise drop this event. On the dashboard path launch() keeps the // process alive, but awaiting makes delivery reliable on every exit path. // Bounded (5s) and never throws. - await trackHookEvent(instanceId, "cli_audit_completed", { - source: "cli", - events_scanned: result.eventsScanned, - sessions_scanned: result.transcripts.scanned, - total_hits: result.totals.hits, - findings: result.results.length, - }); + await trackHookEvent(instanceId, "cli_audit_completed", auditCompletedProps("cli", result)); // No sessions on disk — guide the user instead of opening an empty dashboard. if (result.eventsScanned === 0) { From b7e24d3078f79f7cf8b98872fc16699b71875d1f Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Fri, 17 Jul 2026 16:00:24 +0530 Subject: [PATCH 3/5] fix(dashboard): report only our own errors, not browser extensions' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GlobalErrorListeners registers page-global error/unhandledrejection handlers. Browser extensions inject content scripts into the same page and share the same window, so their failures reached our listeners and went to PostHog stamped $lib: failproofai-web, as though the dashboard had thrown them. Observed live: MetaMask's "Failed to connect to MetaMask" (error_name "i", its minified class) arriving as a failproofai unhandled_rejection on /policies. Extensions are user-installed and open-ended, so the noise was unbounded and depended on which extensions a user happens to run rather than on our code. Left alone it would drown the real signal these listeners exist to catch, and make the dashboard's error rate a function of users' browser setups. Both handlers now attribute an error before reporting it (new lib/error-origin.ts) and report only what traces back to our own origin: - Positive attribution, not a denylist of known extensions, so an unrecognised extension is filtered by default. - The match is on the shared `-extension://` suffix, so a new browser's scheme is covered without a code change. - A stack passing through an extension is treated as the extension's, even when one of our frames appears further down. - Unattributable errors (cross-origin "Script error.", rejections of non-Error values) are dropped — there is nothing in them to debug. React render errors are unaffected: the error boundaries report client_error directly, and React only invokes those for errors thrown inside our own tree. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 + .../global-error-listeners.test.tsx | 112 ++++++++++++++++++ __tests__/lib/error-origin.test.ts | 111 +++++++++++++++++ app/components/global-error-listeners.tsx | 27 +++++ lib/error-origin.ts | 67 +++++++++++ 5 files changed, 319 insertions(+) create mode 100644 __tests__/components/global-error-listeners.test.tsx create mode 100644 __tests__/lib/error-origin.test.ts create mode 100644 lib/error-origin.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 717d68a9..3a2f546e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ ### Fixes - Stop this repo's dogfood hooks from silently no-op'ing when `bun` isn't on the hook's PATH. All 75 hook commands across the 8 project-level agent-CLI configs fired `bun bin/failproofai.mjs --hook ` directly, so whenever the agent CLI was handed a PATH without bun — `npm i -g bun` installs into a single nvm version's bin dir, so `nvm use ` drops it; a macOS GUI launch inherits a launchd PATH built without `~/.zshrc` — every event died with exit 127 and the session ran with **zero** policy enforcement, saying nothing. The configs now call a dev-only `node scripts/dev-hook.mjs` launcher that locates bun across `PATH`, `$BUN_INSTALL/bin`, `~/.bun/bin`, the node execPath sibling, Homebrew/`/usr/local`, and every `~/.nvm/versions/node/*/bin`; installs it via npm if it is genuinely absent; builds `dist/index.js` when missing so `.failproofai/policies/*.mjs` can resolve `import ... from 'failproofai'`; then re-execs the real binary with `stdio: "inherit"` (byte-exact stdout — the deny contracts are JSON on stdout) and propagates the exit code verbatim (2 still means deny; signals map to 128+signum as `sh` did). A `command -v node` pre-check fronts each command so a missing node is a loud one-liner rather than silence — exit 2 on tool events, exit 1 on stop-class events, where exit 2 would mean "retry" and loop forever. The `.opencode` dev shim shares the same resolver and no longer reports a never-run hook as `exitCode: 0` (a silent allow). Production users on `npx -y failproofai` are unaffected. A new drift-guard test reads every committed dogfood config and asserts the launcher form, guard, exit-code split, and per-file command counts — these configs are generated by nothing and read by nothing, which is how #337's opencode shim drifted and silently no-op'd `block-read-outside-cwd` repo-wide. (#564) - Report the onboarding auto-audit to PostHog. `runPostSetupAudit()` — the audit that runs automatically at the end of first-run setup, and therefore the **first audit every new user ever runs** — emitted no telemetry at all, while an explicit `failproofai audit` reported `cli_audit_started` / `cli_audit_completed` / `cli_audit_failed`. First-run audits were invisible, so the audit funnel silently undercounted exactly the activation moment it exists to measure (confirmed live: a fresh install's auto-audit wrote its dashboard cache and reached PostHog with nothing). It now emits the same three events, tagged `source: "onboarding"` against the existing `source: "cli"`, so the two paths stay distinguishable. `cli_audit_completed` fires before the empty-history return, matching `runAuditCli`, so a fresh user with no agent history is still counted; `cli_audit_failed` is awaited because the function returns straight into the dashboard boot, which would otherwise race a fire-and-forget send. The completed-event properties are now built by one shared helper so the two entry points cannot drift. Onboarding remains best-effort: it never throws and never exits. (#562) +- Report the onboarding auto-audit to PostHog. `runPostSetupAudit()` — the audit that runs automatically at the end of first-run setup, and therefore the **first audit every new user ever runs** — emitted no telemetry at all, while an explicit `failproofai audit` reported `cli_audit_started` / `cli_audit_completed` / `cli_audit_failed`. First-run audits were invisible, so the audit funnel silently undercounted exactly the activation moment it exists to measure (confirmed live: a fresh install's auto-audit wrote its dashboard cache and reached PostHog with nothing). It now emits the same three events, tagged `source: "onboarding"` against the existing `source: "cli"`, so the two paths stay distinguishable. `cli_audit_completed` fires before the empty-history return, matching `runAuditCli`, so a fresh user with no agent history is still counted; `cli_audit_failed` is awaited because the function returns straight into the dashboard boot, which would otherwise race a fire-and-forget send. The completed-event properties are now built by one shared helper so the two entry points cannot drift. Onboarding remains best-effort: it never throws and never exits. (#560) +- Report only our own dashboard failures, not browser extensions'. `GlobalErrorListeners` registers page-global `error` / `unhandledrejection` handlers, and browser extensions inject content scripts into the same page and share the same `window` — so their failures reached our listeners and went to PostHog stamped `$lib: failproofai-web`, as though the dashboard had thrown them. Observed live: MetaMask's "Failed to connect to MetaMask" (`error_name: "i"`, its minified class) arriving as a failproofai `unhandled_rejection` on `/policies`. Extensions are user-installed and open-ended, so the noise was unbounded, depended on which extensions a user happened to run rather than on our code, and would eventually have drowned the real signal these listeners exist to catch. Both handlers now attribute the error before reporting (new `lib/error-origin.ts`) and report it only when it traces back to our own origin. Attribution is positive rather than a denylist of known extensions, so an unrecognised extension is filtered by default; the match is on the shared `-extension://` suffix, so a new browser's scheme needs no code change. Unattributable errors (cross-origin `"Script error."`, rejections of non-Error values) are dropped too — there is nothing in them to debug. React render errors are unaffected: the error boundaries report `client_error` directly, and React only invokes those for errors thrown inside our own tree. (#560) - Delete translated pages whose English source no longer exists, and stop them coming back: `translate-docs` gained a `--prune` mode that runs by default on every translation pass (`--no-prune` opts out) and as an explicit step in the `consolidate` job — that job re-checks-out `main` and *overlays* the artifacts, so a prune done only in the per-language jobs would be undone. Translation only ever moved forward, so the 11 pages the AgentEye syncs removed upstream left 154 orphans (11 × 14 locales) that `--update-nav` dropped from the sidebar but Mintlify still served and indexed — non-English readers could land on docs for a deleted feature with no way out. A repo invariant test now fails if any translation outlives its English source. (#556) - Move the docs auto-translation daily cron from 06:00 UTC to 11:05 AM IST (05:35 UTC, encoded as `35 5 * * *` since GitHub Actions cron is always UTC). (#553) diff --git a/__tests__/components/global-error-listeners.test.tsx b/__tests__/components/global-error-listeners.test.tsx new file mode 100644 index 00000000..0b981eee --- /dev/null +++ b/__tests__/components/global-error-listeners.test.tsx @@ -0,0 +1,112 @@ +/** + * End-to-end wiring for the dashboard's global error listeners: real window + * events in, capture decisions out. + * + * The bug this guards: these listeners are page-global, so browser extensions + * sharing the page had their failures reported as failproofai's. A live + * MetaMask rejection reached PostHog as an unhandled_rejection. + */ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { render } from "@testing-library/react"; +import { GlobalErrorListeners } from "@/app/components/global-error-listeners"; + +const h = vi.hoisted(() => ({ captureClientEvent: vi.fn() })); +vi.mock("@/lib/client-telemetry", () => ({ captureClientEvent: h.captureClientEvent })); + +const APP = window.location.origin; + +/** Dispatch a real unhandledrejection event with the given reason. */ +function rejectWith(reason: unknown) { + const event = new Event("unhandledrejection") as Event & { reason: unknown }; + event.reason = reason; + window.dispatchEvent(event); +} + +/** Dispatch a real error event carrying `error` and `filename`. */ +function throwWith(error: Error | undefined, filename: string) { + const event = new Event("error") as Event & { + error?: Error; + filename: string; + message: string; + lineno: number; + colno: number; + }; + event.error = error; + event.filename = filename; + event.message = error?.message ?? "boom"; + event.lineno = 1; + event.colno = 1; + window.dispatchEvent(event); +} + +function errorFrom(origin: string, message: string): Error { + const err = new Error(message); + err.stack = `Error: ${message}\n at fn (${origin}/_next/static/chunks/main.js:1:1)`; + return err; +} + +describe("GlobalErrorListeners", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("reports an unhandled rejection thrown by our own code", () => { + render(); + + rejectWith(errorFrom(APP, "state is undefined")); + + expect(h.captureClientEvent).toHaveBeenCalledWith( + "unhandled_rejection", + expect.objectContaining({ error_message: "state is undefined" }), + ); + }); + + // The exact production case that prompted this filter. + it("ignores MetaMask's rejection instead of reporting it as ours", () => { + render(); + + const metamask = new Error("Failed to connect to MetaMask"); + metamask.name = "i"; + metamask.stack = + "i: Failed to connect to MetaMask\n at chrome-extension://nkbihfbeogaeaoehlefnkodbefgpgknn/inpage.js:1:1"; + rejectWith(metamask); + + expect(h.captureClientEvent).not.toHaveBeenCalled(); + }); + + it("ignores a rejection with no stack to attribute it by", () => { + render(); + + rejectWith("just a string"); + + expect(h.captureClientEvent).not.toHaveBeenCalled(); + }); + + it("reports an uncaught exception from our own script", () => { + render(); + + throwWith(errorFrom(APP, "render failed"), `${APP}/_next/static/chunks/main.js`); + + expect(h.captureClientEvent).toHaveBeenCalledWith( + "unhandled_exception", + expect.objectContaining({ error_message: "render failed" }), + ); + }); + + it("ignores an uncaught exception from an extension's script", () => { + render(); + + throwWith(undefined, "chrome-extension://abc/inpage.js"); + + expect(h.captureClientEvent).not.toHaveBeenCalled(); + }); + + it("stops listening once unmounted", () => { + const { unmount } = render(); + unmount(); + + rejectWith(errorFrom(APP, "after unmount")); + + expect(h.captureClientEvent).not.toHaveBeenCalled(); + }); +}); diff --git a/__tests__/lib/error-origin.test.ts b/__tests__/lib/error-origin.test.ts new file mode 100644 index 00000000..450b9ae1 --- /dev/null +++ b/__tests__/lib/error-origin.test.ts @@ -0,0 +1,111 @@ +// @vitest-environment node +/** + * Attribution for browser errors. + * + * The bug this guards: window's error/unhandledrejection listeners are + * page-global, so browser extensions injected into the dashboard reported their + * failures as ours. A live MetaMask rejection ("Failed to connect to MetaMask") + * arrived in PostHog as a failproofai unhandled_rejection. + */ +import { describe, it, expect } from "vitest"; +import { classifyErrorOrigin, isAppError } from "../../lib/error-origin"; + +const APP = "http://localhost:8020"; + +describe("classifyErrorOrigin", () => { + describe("browser extensions", () => { + // The exact shape observed in production. + it("classifies the real MetaMask rejection as an extension error", () => { + const stack = + "i: Failed to connect to MetaMask\n" + + " at chrome-extension://nkbihfbeogaeaoehlefnkodbefgpgknn/inpage.js:1:12345"; + + expect(classifyErrorOrigin({ stack, appOrigin: APP })).toBe("extension"); + }); + + it.each([ + ["chrome", "chrome-extension://abc/inpage.js:1:1"], + ["firefox", "moz-extension://abc/content.js:1:1"], + ["safari", "safari-web-extension://abc/injected.js:1:1"], + ["edge", "ms-browser-extension://abc/script.js:1:1"], + ])("recognises %s extension frames", (_browser, frame) => { + expect(classifyErrorOrigin({ stack: `Error: x\n at ${frame}`, appOrigin: APP })).toBe( + "extension", + ); + }); + + // Matching the shared `-extension://` suffix, not a vendor list, so a new + // browser's scheme is filtered without a code change. + it("recognises an unknown vendor's extension scheme", () => { + expect( + classifyErrorOrigin({ stack: "at future-extension://abc/x.js:1:1", appOrigin: APP }), + ).toBe("extension"); + }); + + it("uses the filename when there is no stack", () => { + expect( + classifyErrorOrigin({ filename: "chrome-extension://abc/inpage.js", appOrigin: APP }), + ).toBe("extension"); + }); + + // An error routed through an extension is the extension's problem, even + // though one of our frames appears further down the stack. + it("treats a mixed stack as an extension error", () => { + const stack = + "Error: boom\n" + + " at chrome-extension://abc/inpage.js:1:1\n" + + ` at fn (${APP}/_next/static/chunks/main.js:2:2)`; + + expect(classifyErrorOrigin({ stack, appOrigin: APP })).toBe("extension"); + }); + }); + + describe("our own dashboard", () => { + it("classifies a stack from our origin as an app error", () => { + const stack = `TypeError: x is not a function\n at Page (${APP}/_next/static/chunks/page.js:1:1)`; + + expect(classifyErrorOrigin({ stack, appOrigin: APP })).toBe("app"); + }); + + it("classifies a filename from our origin as an app error", () => { + expect( + classifyErrorOrigin({ filename: `${APP}/_next/static/chunks/main.js`, appOrigin: APP }), + ).toBe("app"); + }); + + it("attributes correctly on a non-default origin", () => { + const origin = "http://127.0.0.1:3999"; + + expect(classifyErrorOrigin({ stack: `at fn (${origin}/x.js:1:1)`, appOrigin: origin })).toBe( + "app", + ); + }); + }); + + describe("unattributable", () => { + it("returns unknown when there is no location information at all", () => { + expect(classifyErrorOrigin({ appOrigin: APP })).toBe("unknown"); + expect(classifyErrorOrigin({ stack: "", filename: "", appOrigin: APP })).toBe("unknown"); + expect(classifyErrorOrigin({ stack: " ", appOrigin: APP })).toBe("unknown"); + }); + + // The opaque cross-origin error — nothing in it to debug. + it("returns unknown for a bare cross-origin 'Script error.'", () => { + expect(classifyErrorOrigin({ stack: "Script error.", appOrigin: APP })).toBe("unknown"); + }); + + it("returns unknown for a third-party host that is neither ours nor an extension", () => { + expect( + classifyErrorOrigin({ stack: "at https://cdn.example.com/x.js:1:1", appOrigin: APP }), + ).toBe("unknown"); + }); + }); +}); + +describe("isAppError", () => { + it("reports only positively-attributed app errors", () => { + expect(isAppError({ stack: `at fn (${APP}/x.js:1:1)`, appOrigin: APP })).toBe(true); + expect(isAppError({ stack: "at chrome-extension://abc/x.js:1:1", appOrigin: APP })).toBe(false); + expect(isAppError({ appOrigin: APP })).toBe(false); + }); +}); diff --git a/app/components/global-error-listeners.tsx b/app/components/global-error-listeners.tsx index e50fe076..ecfa35f1 100644 --- a/app/components/global-error-listeners.tsx +++ b/app/components/global-error-listeners.tsx @@ -2,10 +2,27 @@ import { useEffect } from "react"; import { captureClientEvent } from "@/lib/client-telemetry"; +import { isAppError } from "@/lib/error-origin"; +/** + * Reports uncaught dashboard errors. + * + * Both listeners are page-global, so they also see failures from browser + * extensions injected into the page (MetaMask and friends). Those are filtered + * out — see lib/error-origin.ts — so this reports only our own failures. + */ export function GlobalErrorListeners() { useEffect(() => { const handleError = (event: ErrorEvent) => { + if ( + !isAppError({ + stack: event.error?.stack, + filename: event.filename, + appOrigin: window.location.origin, + }) + ) { + return; + } captureClientEvent("unhandled_exception", { error_message: event.message, error_name: event.error?.name, @@ -17,6 +34,16 @@ export function GlobalErrorListeners() { const handleUnhandledRejection = (event: PromiseRejectionEvent) => { const reason = event.reason; + // A rejection carries no filename, so the stack is the only evidence of + // where it came from. A non-Error reason has none and stays unattributed. + if ( + !isAppError({ + stack: reason instanceof Error ? reason.stack : undefined, + appOrigin: window.location.origin, + }) + ) { + return; + } captureClientEvent("unhandled_rejection", { error_message: reason instanceof Error ? reason.message : String(reason), error_name: reason instanceof Error ? reason.name : undefined, diff --git a/lib/error-origin.ts b/lib/error-origin.ts new file mode 100644 index 00000000..4025fa96 --- /dev/null +++ b/lib/error-origin.ts @@ -0,0 +1,67 @@ +/** + * Works out whether a browser error actually came from the dashboard. + * + * `window`'s "error" and "unhandledrejection" events are page-global: browser + * extensions inject content scripts into the same page and share the same + * window, so their failures reach our listeners and get reported as ours. The + * observed case was MetaMask's "Failed to connect to MetaMask" arriving as a + * failproofai `unhandled_rejection`. Extensions are user-installed and + * open-ended, so the noise is unbounded and unrelated to our code. + * + * We therefore attribute positively — report an error only when it can be + * traced to our own origin — rather than maintaining a denylist of extensions + * we happen to know about. React render errors don't depend on this: the error + * boundaries (`app/global-error.tsx`, `app/components/error-fallback.tsx`) + * report `client_error` directly, and React only invokes those for errors thrown + * inside our own tree. + */ + +/** + * Any `-extension://` URL — `chrome-extension://`, `moz-extension://`, + * `safari-web-extension://`, `ms-browser-extension://`. Matching the shared + * suffix rather than an explicit vendor list means a new browser's scheme is + * filtered the day it ships. + */ +const EXTENSION_URL = /\b[\w-]+-extension:\/\//i; + +export type ErrorOrigin = + /** Traceable to our own origin — a real dashboard failure. */ + | "app" + /** Came from a browser extension sharing the page. */ + | "extension" + /** Not attributable: no stack, or a cross-origin "Script error.". */ + | "unknown"; + +export interface ErrorOriginInput { + /** `error.stack` / `reason.stack`, when the thrown value was an Error. */ + stack?: string | null; + /** `ErrorEvent.filename` — the script URL. Absent on rejections. */ + filename?: string | null; + /** Typically `window.location.origin`. */ + appOrigin: string; +} + +/** + * Classify where an error came from, using whatever location information the + * browser gave us. + * + * Extension frames are checked first: an error whose stack passes through an + * extension is that extension's problem even if one of our frames appears + * further down. + */ +export function classifyErrorOrigin({ stack, filename, appOrigin }: ErrorOriginInput): ErrorOrigin { + const text = [filename, stack].filter(Boolean).join("\n"); + if (!text.trim()) return "unknown"; + if (EXTENSION_URL.test(text)) return "extension"; + if (appOrigin && text.includes(appOrigin)) return "app"; + return "unknown"; +} + +/** + * Whether an error is ours to report. Only positively-attributed errors qualify: + * `unknown` covers cross-origin "Script error." and stack-less rejections, which + * are unactionable anyway — there's nothing in them to debug. + */ +export function isAppError(input: ErrorOriginInput): boolean { + return classifyErrorOrigin(input) === "app"; +} From 5daad39893f0e2fc21d3c5b40f1c78f86153ccac Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Fri, 17 Jul 2026 17:55:22 +0530 Subject: [PATCH 4/5] fix(telemetry): give install reporting a 5s ceiling, not 2s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The install events are the first thing a fresh process sends, so they always pay the DNS + TLS handshake. Measured against production PostHog they deliver in ~1.1s from cold, with a 1.85s worst case observed. A 2s budget left almost no headroom over that, for a saving nobody experiences: the timeout is a ceiling, not a cost — delivery resolves as soon as PostHog answers. A tight budget only ever converts a slow network into a silently dropped report, which is exactly what this module exists to prevent. 5s matches hook-telemetry and the postinstall script this replaces, and install-telemetry's own default. Verified by driving trackInstallEvent against real PostHog from a cold process at both budgets, plus a transparent proxy confirming production returns HTTP 200 {"status":"Ok"} for every event (14/14). Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 5 ++--- __tests__/lib/install-check.test.ts | 15 +++++++++++++++ lib/install-check.ts | 16 ++++++++++++++-- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a2f546e..782044ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,10 @@ - Redirect the 13 AgentEye pages the upstream syncs deleted (`/agenteye/deployment`, `/agenteye/kubernetes-deployment`, `/agenteye/troubleshooting`, …) to a live page instead of hard-404ing. (#556) ### Fixes +- Stop shipping npm lifecycle scripts, removing the `npm warn` every install printed. npm 12 (`allowScripts`, released 2026-07-08 and now the `latest` dist-tag) blocks dependency install scripts by default, so `npm install failproofai` warned `1 package had install scripts blocked … failproofai@… (postinstall: node scripts/postinstall.mjs)` and silently skipped the script; npm 11.16+ prints the advisory `allow-scripts` variant and still runs it. `allowScripts` is purely consumer-side — a package cannot opt itself in (verified: a package declaring `allowScripts`/`trustedDependencies` for *itself* is ignored) — so the only fix is to ship no install scripts. The `postinstall` script's install telemetry (`first_install` / `version_changed` / `package_installed`, with identical event names and properties) now fires from the CLI's first non-hook invocation via `lib/install-check.ts`, reporting at most once per version and no-op'ing on the steady-state path; it is deliberately kept out of `bin/failproofai.mjs`'s `--hook` fast path, which runs on every tool call. This also *recovers* telemetry already being dropped for bun, pnpm, and Yarn ≥4.14 users, which have blocked install scripts for some time. The script's `server.js` check and shadowed-PATH diagnosis were redundant — `scripts/launch.ts` already performs both at launch, with a better error message. `package_installed` now measures install→activation rather than raw installs, and same-version reinstalls (`direction: "reinstall"`) are no longer reported, as the CLI has no signal to detect them. The install-time welcome message is dropped; the `config` wizard's first-run redirect already handles onboarding. (#560) +- Remove `scripts/preuninstall.mjs`, which never ran. npm honours no uninstall lifecycle scripts (verified with a probe package: `postinstall` fired, `preuninstall` did not), so the hook cleanup it appeared to perform has never happened — uninstalling failproofai leaves `__failproofai_hook__` entries behind in settings files. Removing the dead code makes the gap visible; cleanup needs an explicit `failproofai policies --uninstall` before removing the package. (#560) - Stop this repo's dogfood hooks from silently no-op'ing when `bun` isn't on the hook's PATH. All 75 hook commands across the 8 project-level agent-CLI configs fired `bun bin/failproofai.mjs --hook ` directly, so whenever the agent CLI was handed a PATH without bun — `npm i -g bun` installs into a single nvm version's bin dir, so `nvm use ` drops it; a macOS GUI launch inherits a launchd PATH built without `~/.zshrc` — every event died with exit 127 and the session ran with **zero** policy enforcement, saying nothing. The configs now call a dev-only `node scripts/dev-hook.mjs` launcher that locates bun across `PATH`, `$BUN_INSTALL/bin`, `~/.bun/bin`, the node execPath sibling, Homebrew/`/usr/local`, and every `~/.nvm/versions/node/*/bin`; installs it via npm if it is genuinely absent; builds `dist/index.js` when missing so `.failproofai/policies/*.mjs` can resolve `import ... from 'failproofai'`; then re-execs the real binary with `stdio: "inherit"` (byte-exact stdout — the deny contracts are JSON on stdout) and propagates the exit code verbatim (2 still means deny; signals map to 128+signum as `sh` did). A `command -v node` pre-check fronts each command so a missing node is a loud one-liner rather than silence — exit 2 on tool events, exit 1 on stop-class events, where exit 2 would mean "retry" and loop forever. The `.opencode` dev shim shares the same resolver and no longer reports a never-run hook as `exitCode: 0` (a silent allow). Production users on `npx -y failproofai` are unaffected. A new drift-guard test reads every committed dogfood config and asserts the launcher form, guard, exit-code split, and per-file command counts — these configs are generated by nothing and read by nothing, which is how #337's opencode shim drifted and silently no-op'd `block-read-outside-cwd` repo-wide. (#564) - Report the onboarding auto-audit to PostHog. `runPostSetupAudit()` — the audit that runs automatically at the end of first-run setup, and therefore the **first audit every new user ever runs** — emitted no telemetry at all, while an explicit `failproofai audit` reported `cli_audit_started` / `cli_audit_completed` / `cli_audit_failed`. First-run audits were invisible, so the audit funnel silently undercounted exactly the activation moment it exists to measure (confirmed live: a fresh install's auto-audit wrote its dashboard cache and reached PostHog with nothing). It now emits the same three events, tagged `source: "onboarding"` against the existing `source: "cli"`, so the two paths stay distinguishable. `cli_audit_completed` fires before the empty-history return, matching `runAuditCli`, so a fresh user with no agent history is still counted; `cli_audit_failed` is awaited because the function returns straight into the dashboard boot, which would otherwise race a fire-and-forget send. The completed-event properties are now built by one shared helper so the two entry points cannot drift. Onboarding remains best-effort: it never throws and never exits. (#562) -- Report the onboarding auto-audit to PostHog. `runPostSetupAudit()` — the audit that runs automatically at the end of first-run setup, and therefore the **first audit every new user ever runs** — emitted no telemetry at all, while an explicit `failproofai audit` reported `cli_audit_started` / `cli_audit_completed` / `cli_audit_failed`. First-run audits were invisible, so the audit funnel silently undercounted exactly the activation moment it exists to measure (confirmed live: a fresh install's auto-audit wrote its dashboard cache and reached PostHog with nothing). It now emits the same three events, tagged `source: "onboarding"` against the existing `source: "cli"`, so the two paths stay distinguishable. `cli_audit_completed` fires before the empty-history return, matching `runAuditCli`, so a fresh user with no agent history is still counted; `cli_audit_failed` is awaited because the function returns straight into the dashboard boot, which would otherwise race a fire-and-forget send. The completed-event properties are now built by one shared helper so the two entry points cannot drift. Onboarding remains best-effort: it never throws and never exits. (#560) - Report only our own dashboard failures, not browser extensions'. `GlobalErrorListeners` registers page-global `error` / `unhandledrejection` handlers, and browser extensions inject content scripts into the same page and share the same `window` — so their failures reached our listeners and went to PostHog stamped `$lib: failproofai-web`, as though the dashboard had thrown them. Observed live: MetaMask's "Failed to connect to MetaMask" (`error_name: "i"`, its minified class) arriving as a failproofai `unhandled_rejection` on `/policies`. Extensions are user-installed and open-ended, so the noise was unbounded, depended on which extensions a user happened to run rather than on our code, and would eventually have drowned the real signal these listeners exist to catch. Both handlers now attribute the error before reporting (new `lib/error-origin.ts`) and report it only when it traces back to our own origin. Attribution is positive rather than a denylist of known extensions, so an unrecognised extension is filtered by default; the match is on the shared `-extension://` suffix, so a new browser's scheme needs no code change. Unattributable errors (cross-origin `"Script error."`, rejections of non-Error values) are dropped too — there is nothing in them to debug. React render errors are unaffected: the error boundaries report `client_error` directly, and React only invokes those for errors thrown inside our own tree. (#560) - Delete translated pages whose English source no longer exists, and stop them coming back: `translate-docs` gained a `--prune` mode that runs by default on every translation pass (`--no-prune` opts out) and as an explicit step in the `consolidate` job — that job re-checks-out `main` and *overlays* the artifacts, so a prune done only in the per-language jobs would be undone. Translation only ever moved forward, so the 11 pages the AgentEye syncs removed upstream left 154 orphans (11 × 14 locales) that `--update-nav` dropped from the sidebar but Mintlify still served and indexed — non-English readers could land on docs for a deleted feature with no way out. A repo invariant test now fails if any translation outlives its English source. (#556) - Move the docs auto-translation daily cron from 06:00 UTC to 11:05 AM IST (05:35 UTC, encoded as `35 5 * * *` since GitHub Actions cron is always UTC). (#553) @@ -41,8 +42,6 @@ - Surface OpenClaw sessions in the dashboard's audit + history browser: reads the real JSONL transcripts at `~/.openclaw/agents//sessions/.jsonl` (skipping the heavy `.trajectory.jsonl` OTel traces) via `lib/openclaw-sessions.ts` (a pure, unit-tested type-discriminated parser that pairs assistant `toolCall` blocks with their `toolResult` by `toolCallId`) and `lib/openclaw-projects.ts` (reads the `sessions.json` index, groups by agentId into `openclaw-` projects, parses channel metadata from the sessionKey). Wired through `src/audit/cli-adapters/openclaw.ts`, `lib/cli-registry.ts` (teal badge), `lib/projects.ts`, `lib/download-session.ts` (real-file download — no synthesis needed), and the `app/project/[name]` routes. (#489) ### Fixes -- Stop shipping npm lifecycle scripts, removing the `npm warn` every install printed. npm 12 (`allowScripts`, released 2026-07-08 and now the `latest` dist-tag) blocks dependency install scripts by default, so `npm install failproofai` warned `1 package had install scripts blocked … failproofai@… (postinstall: node scripts/postinstall.mjs)` and silently skipped the script; npm 11.16+ prints the advisory `allow-scripts` variant and still runs it. `allowScripts` is purely consumer-side — a package cannot opt itself in (verified: a package declaring `allowScripts`/`trustedDependencies` for *itself* is ignored) — so the only fix is to ship no install scripts. The `postinstall` script's install telemetry (`first_install` / `version_changed` / `package_installed`, with identical event names and properties) now fires from the CLI's first non-hook invocation via `lib/install-check.ts`, reporting at most once per version and no-op'ing on the steady-state path; it is deliberately kept out of `bin/failproofai.mjs`'s `--hook` fast path, which runs on every tool call. This also *recovers* telemetry already being dropped for bun, pnpm, and Yarn ≥4.14 users, which have blocked install scripts for some time. The script's `server.js` check and shadowed-PATH diagnosis were redundant — `scripts/launch.ts` already performs both at launch, with a better error message. `package_installed` now measures install→activation rather than raw installs, and same-version reinstalls (`direction: "reinstall"`) are no longer reported, as the CLI has no signal to detect them. The install-time welcome message is dropped; the `config` wizard's first-run redirect already handles onboarding. (#560) -- Remove `scripts/preuninstall.mjs`, which never ran. npm honours no uninstall lifecycle scripts (verified with a probe package: `postinstall` fired, `preuninstall` did not), so the hook cleanup it appeared to perform has never happened — uninstalling failproofai leaves `__failproofai_hook__` entries behind in settings files. Removing the dead code makes the gap visible; cleanup needs an explicit `failproofai policies --uninstall` before removing the package. (#560) - Move the docs auto-translation off the per-push trigger onto a daily scheduled run (`cron: 0 6 * * *`, 06:00 UTC): running the full 14-language matrix on every doc commit to `main` was expensive, so a day's English-source edits are now coalesced into one run. The content-hash cache (`scripts/translate-docs/.translation-cache.json`) still limits token spend to the pages whose source actually changed since the last successful run — so most days translate only a handful of pages (or none) — and manual `workflow_dispatch` (with `force`/`languages`) is unchanged. (#547) - Fix the auto-translate job failing repeatedly on the large AgentEye docs. `translateContent` capped `max_tokens` at 16384, but the biggest English pages (e.g. `agenteye/kubernetes-deployment.mdx` at ~1400 lines) translate to well beyond that for verbose target languages — so the response was silently truncated mid-MDX, leaving an unbalanced `{` or an unterminated JSX expression that failed `mintlify validate` in the `consolidate` job (and, for the very largest pages, tripped the proxy with a "stream ended" error in the per-language `translate` job). Worse, the truncated output was written to disk and cached as if complete, so a re-run served the same broken page. Raises the ceiling to 64000 (Claude Haiku 4.5's max output for Tier 2/3 languages; well within Claude Sonnet 4.6's 128000 for Tier 1), overridable via the new `TRANSLATE_MAX_TOKENS` env var, and — as a backstop — throws when `stop_reason === "max_tokens"` so a truncated page is never written or cached and its language is excluded from the publish step instead of shipping malformed MDX. (#546) - Fix the Mintlify docs deployment failing on every `docs/i18n/README.*.md` translation. Mintlify parses each page as MDX, where the README's top-level HTML comment (``) is a hard syntax error — it rejects the leading `!` of `` inside code fences literal (e.g. the AgentEye collector plist sample) — and the 14 committed translations are regenerated to match. Also aligns `readme-translator.ts` with `translateMdxPage` by running `sanitizeJsxAttributes` over its JSX-bearing output. (#535) diff --git a/__tests__/lib/install-check.test.ts b/__tests__/lib/install-check.test.ts index cbb8cc43..ae3fe483 100644 --- a/__tests__/lib/install-check.test.ts +++ b/__tests__/lib/install-check.test.ts @@ -104,6 +104,21 @@ describe("maybeReportInstall", () => { expect(writeFileSync).toHaveBeenCalledWith(LAST_VERSION, "1.0.0", "utf8"); }); + // Delivery measured at ~1.1s from a cold process against production PostHog, + // with a 1.85s worst case. The timeout is a ceiling, not a cost, so it needs + // real headroom over that — a tight budget only ever drops the report. + it("allows enough time for a cold-start delivery", async () => { + await setupFs({ lastVersion: null }); + const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); + const { maybeReportInstall } = await import("../../lib/install-check"); + + await maybeReportInstall("1.0.0"); + + for (const call of vi.mocked(trackInstallEvent).mock.calls) { + expect((call[2] as { timeoutMs: number }).timeoutMs).toBeGreaterThanOrEqual(5000); + } + }); + it("stamps the caller's version on the event, never 'unknown'", async () => { await setupFs({ lastVersion: null }); const { trackInstallEvent } = await import("../../scripts/install-telemetry.mjs"); diff --git a/lib/install-check.ts b/lib/install-check.ts index a5a089fb..d0a9e864 100644 --- a/lib/install-check.ts +++ b/lib/install-check.ts @@ -27,8 +27,20 @@ import { trackInstallEvent } from "../scripts/install-telemetry.mjs"; const FAILPROOFAI_HOOK_MARKER = "__failproofai_hook__"; const NAMESPACE = "failproofai-telemetry-v1"; -/** Cap on the once-per-version report so a hanging network never stalls a command. */ -const REPORT_TIMEOUT_MS = 2000; +/** + * Cap on the once-per-version report so a hanging network never stalls a command. + * + * A ceiling, not a cost: delivery resolves as soon as PostHog answers, measured + * at ~1.1s from a cold process (these events are the first thing a fresh process + * sends, so they always pay the DNS + TLS handshake). Raising this only matters + * when the network is slow — precisely when a tight budget would silently drop + * the report instead. + * + * 5s matches hook-telemetry and the postinstall script this replaces. An earlier + * 2s left barely any headroom over a 1.85s worst case seen in testing, for a + * saving nobody experiences on a healthy network. + */ +const REPORT_TIMEOUT_MS = 5000; function hashToId(raw: string): string { return createHmac("sha256", NAMESPACE).update(raw).digest("hex"); From ced8e1318b5fb094ddbd83bcc5ccbbc4a39c8266 Mon Sep 17 00:00:00 2001 From: chhhee10 Date: Fri, 17 Jul 2026 18:20:56 +0530 Subject: [PATCH 5/5] fix(install-check): split prerelease identifiers on dots only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Semver §9 separates prerelease identifiers with dots; a hyphen is a legal character *inside* one. So `beta-2` is a single non-numeric identifier and compares lexically against `beta-10` — making it GREATER ("2" > "1"). Splitting on /[.-]/ shredded it into ["beta", "2"] and compared numerically, producing the opposite order: 1.0.0-beta-2 vs 1.0.0-beta-10 → spec: greater · was: less Inherited verbatim from postinstall.mjs, so not a regression, and harmless for our own dot-separated versions — but wrong, and this module is the place it lives now. Caught by CodeRabbit on #560; verified against the spec and pinned with a regression test that fails on the old split. Co-Authored-By: Claude Opus 4.8 (1M context) --- __tests__/lib/install-check.test.ts | 10 ++++++++++ lib/install-check.ts | 8 ++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/__tests__/lib/install-check.test.ts b/__tests__/lib/install-check.test.ts index ae3fe483..f1060425 100644 --- a/__tests__/lib/install-check.test.ts +++ b/__tests__/lib/install-check.test.ts @@ -319,6 +319,16 @@ describe("compareSemver", () => { expect(compareSemver("1.0.0-beta.2", "1.0.0-beta.10")).toBeLessThan(0); }); + // Semver §9: identifiers are dot-separated; a hyphen is legal *inside* one. + // So `beta-2` is a single non-numeric identifier and compares lexically — + // making it GREATER than `beta-10` ("2" > "1"), the opposite of the numeric + // ordering you'd get by splitting on the hyphen. + it("treats a hyphen inside a prerelease identifier as part of it, not a separator", async () => { + const { compareSemver } = await import("../../lib/install-check"); + expect(compareSemver("1.0.0-beta-2", "1.0.0-beta-10")).toBeGreaterThan(0); + expect(compareSemver("1.0.0-beta-10", "1.0.0-beta-2")).toBeLessThan(0); + }); + it("compares major/minor/patch numerically", async () => { const { compareSemver } = await import("../../lib/install-check"); expect(compareSemver("1.9.0", "1.10.0")).toBeLessThan(0); diff --git a/lib/install-check.ts b/lib/install-check.ts index d0a9e864..e3ed8745 100644 --- a/lib/install-check.ts +++ b/lib/install-check.ts @@ -54,6 +54,10 @@ function lastVersionPath(): string { * Semver ordering. A release outranks the same version carrying a prerelease * tag (semver §11); within a prerelease, numeric identifiers rank below * non-numeric ones. + * + * Prerelease identifiers are separated by dots only (semver §9). A hyphen is a + * legal character *inside* an identifier, so `beta-2` is one non-numeric + * identifier compared lexically — not `beta` and `2`. */ export function compareSemver(a: string, b: string): number { const parse = (v: string) => { @@ -70,8 +74,8 @@ export function compareSemver(a: string, b: string): number { if (pa.pre === null && pb.pre === null) return 0; if (pa.pre === null) return 1; if (pb.pre === null) return -1; - const ax = pa.pre.split(/[.-]/); - const bx = pb.pre.split(/[.-]/); + const ax = pa.pre.split("."); + const bx = pb.pre.split("."); for (let i = 0; i < Math.max(ax.length, bx.length); i++) { const ai = ax[i], bi = bx[i]; if (ai === undefined) return -1;