Fix desktop CI + concern AI opt-in, faithfulness actions, cycle stats, 1:1 themes

.gitignore

@@ -9,3 +9,4 @@ apps/ui/dist
 apps/api/dist
 apps/desktop/src-tauri/target
 apps/desktop/src-tauri/sidecars/
+*.tsbuildinfo

apps/api/src/faithfulness.test.ts

@@ -1,31 +1,117 @@
 import assert from "node:assert/strict";
 import { describe, it } from "node:test";
-import { mergeLintFindings, runFaithfulnessLint } from "../src/faithfulness.js";
+import {
+  applyFaithfulnessFix,
+  mergeLintFindings,
+  runFaithfulnessLint,
+} from "../src/faithfulness.js";
+
+/** Golden drafts with known invention patterns — regression set for faithfulness depth. */
+const GOLDEN = {
+  clean: {
+    text: "Led the checkout migration [ach_1] and coached two engineers through the rollout [fb_2].",
+    allowlist: ["ach_1", "fb_2"],
+    expectInvention: false,
+  },
+  inventedCite: {
+    text: "Shipped the migration [ach_real] and invented support [ach_fake].",
+    allowlist: ["ach_real"],
+    expectIds: ["unknown_cite_ach_fake"],
+  },
+  ungroundedAbsolutes: {
+    text: [
+      "They completely transformed the platform architecture across every squad this half.",
+      "Customer satisfaction improved dramatically while latency dropped for all regions.",
+      "See [ach_1] for the migration they led.",
+    ].join(" "),
+    allowlist: ["ach_1"],
+    expectIds: ["ungrounded_claims"],
+  },
+  bareMetric: {
+    text: "Reduced p99 latency by 40% across checkout.",
+    allowlist: [] as string[],
+    expectMetric: "40%",
+  },
+  metricWithCite: {
+    text: "Reduced p99 latency by 40% across checkout [ach_1].",
+    allowlist: ["ach_1"],
+    expectInvention: false,
+  },
+};
 
 describe("faithfulness lint", () => {
   it("flags unknown citations", () => {
-    const findings = runFaithfulnessLint("Shipped [ach_real] and [ach_fake].", ["ach_real"]);
+    const findings = runFaithfulnessLint(GOLDEN.inventedCite.text, GOLDEN.inventedCite.allowlist);
     assert.ok(findings.some((f) => f.category === "invention" && /ach_fake/.test(f.message)));
+    assert.equal(findings.find((f) => f.id.startsWith("unknown_cite_"))?.suggestedFix?.action, "strip_cite");
   });
 
   it("flags ungrounded long sentences", () => {
-    const text = [
-      "They completely transformed the platform architecture across every squad this half.",
-      "Customer satisfaction improved dramatically while latency dropped for all regions.",
-      "See [ach_1] for the migration they led.",
-    ].join(" ");
-    const findings = runFaithfulnessLint(text, ["ach_1"]);
+    const findings = runFaithfulnessLint(
+      GOLDEN.ungroundedAbsolutes.text,
+      GOLDEN.ungroundedAbsolutes.allowlist,
+    );
     assert.ok(findings.some((f) => f.id === "ungrounded_claims"));
+    assert.ok(findings.find((f) => f.id === "ungrounded_claims")?.suggestedFix);
   });
 
   it("flags bare metrics", () => {
-    const findings = runFaithfulnessLint("Reduced p99 latency by 40% across checkout.", []);
+    const findings = runFaithfulnessLint(GOLDEN.bareMetric.text, GOLDEN.bareMetric.allowlist);
     assert.ok(findings.some((f) => f.category === "invention" && /40%/.test(f.message)));
   });
 
+  it("accepts clean cited prose", () => {
+    const findings = runFaithfulnessLint(GOLDEN.clean.text, GOLDEN.clean.allowlist);
+    assert.equal(
+      findings.filter((f) => f.category === "invention" && f.severity === "warn").length,
+      0,
+    );
+  });
+
+  it("does not flag metrics next to a citation", () => {
+    const findings = runFaithfulnessLint(GOLDEN.metricWithCite.text, GOLDEN.metricWithCite.allowlist);
+    assert.ok(!findings.some((f) => f.id.startsWith("metric_")));
+  });
+
   it("merges findings without dupes", () => {
     const a = runFaithfulnessLint("Reduced cost by 25% somehow.", []);
     const merged = mergeLintFindings(a, a);
     assert.equal(merged.length, a.length);
   });
+
+  it("applies soften and cite fixes", () => {
+    const findings = runFaithfulnessLint(GOLDEN.bareMetric.text, ["ach_1"]);
+    const metric = findings.find((f) => f.id.startsWith("metric_"));
+    assert.ok(metric?.suggestedFix);
+    const next = applyFaithfulnessFix(GOLDEN.bareMetric.text, metric!);
+    assert.ok(next);
+    assert.match(next!, /\[ach_1\]/);
+  });
+
+  it("strips unknown cites via suggested fix", () => {
+    const findings = runFaithfulnessLint(GOLDEN.inventedCite.text, GOLDEN.inventedCite.allowlist);
+    const bad = findings.find((f) => f.id === "unknown_cite_ach_fake");
+    assert.ok(bad);
+    const next = applyFaithfulnessFix(GOLDEN.inventedCite.text, bad!);
+    assert.ok(next);
+    assert.ok(!next!.includes("[ach_fake]"));
+    assert.ok(next!.includes("[ach_real]"));
+  });
+});
+
+describe("faithfulness golden eval set", () => {
+  it("scores known invention drafts", () => {
+    const cases = [
+      GOLDEN.inventedCite,
+      GOLDEN.ungroundedAbsolutes,
+      GOLDEN.bareMetric,
+    ];
+    for (const c of cases) {
+      const findings = runFaithfulnessLint(c.text, c.allowlist);
+      assert.ok(
+        findings.some((f) => f.category === "invention"),
+        `expected invention in: ${c.text.slice(0, 60)}`,
+      );
+    }
+  });
 });

apps/api/src/faithfulness.ts

@@ -3,6 +3,28 @@ import { extractCitationIds } from "./citations.js";
 
 const SENTENCE_SPLIT = /(?<=[.!?])\s+(?=[A-Z("[])|(?:\n+)/;
 
+const ABSOLUTE_WORDS =
+  /\b(always|never|completely|entirely|dramatically|everyone|no one|all regions|every squad)\b/gi;
+
+function softenText(excerpt: string): string {
+  return excerpt
+    .replace(/\balways\b/gi, "often")
+    .replace(/\bnever\b/gi, "rarely")
+    .replace(/\bcompletely\b/gi, "substantially")
+    .replace(/\bentirely\b/gi, "largely")
+    .replace(/\bdramatically\b/gi, "notably")
+    .replace(/\beveryone\b/gi, "many teammates")
+    .replace(/\bno one\b/gi, "few people")
+    .replace(/\ball regions\b/gi, "several regions")
+    .replace(/\bevery squad\b/gi, "multiple squads")
+    .replace(/(\d+%)/g, "about $1 (verify)")
+    .replace(/(\$\d[\d,]*(?:\.\d+)?)/g, "about $1 (verify)");
+}
+
+function pickCiteId(allowlist: string[]): string | undefined {
+  return allowlist.find((id) => /^(ach|fb|goal|doc|rexp)_/.test(id));
+}
+
 /** Claims that look substantive but lack any [id] citation nearby. */
 export function runFaithfulnessLint(text: string, allowlist: string[] = []): BiasFinding[] {
   const findings: BiasFinding[] = [];
@@ -19,6 +41,11 @@ export function runFaithfulnessLint(text: string, allowlist: string[] = []): Bia
         category: "invention",
         message: `Citation [${id}] is not in the evidence allowlist — remove or replace with a real evidence id.`,
         excerpt: `[${id}]`,
+        suggestedFix: {
+          action: "strip_cite",
+          label: "Remove unknown cite",
+          newText: "",
+        },
       });
     }
   }
@@ -33,9 +60,7 @@ export function runFaithfulnessLint(text: string, allowlist: string[] = []): Bia
   for (const sentence of sentences) {
     if (/^(summary|strengths|growth|overall|note|private)\b/i.test(sentence)) continue;
     if (!/\[[a-z]+_[A-Za-z0-9_-]+\]/.test(sentence)) {
-      // Skip questions / instructions
       if (/\?$/.test(sentence) || /^(please|consider|remember)\b/i.test(sentence)) continue;
-      // Skip name-ish lists (mostly Title Case tokens, almost no verbs/stopwords)
       const words = sentence.split(/\s+/).filter(Boolean);
       const titleish = words.filter((w) => /^[A-Z][a-z]+/.test(w.replace(/[.,]/g, ""))).length;
       if (words.length >= 3 && titleish / words.length >= 0.7) continue;
@@ -45,33 +70,83 @@ export function runFaithfulnessLint(text: string, allowlist: string[] = []): Bia
   }
 
   if (ungrounded >= 2) {
+    const sample = samples[0];
+    const citeId = pickCiteId(allowlist);
+    const softened = sample ? softenText(sample) : undefined;
     findings.push({
       id: "ungrounded_claims",
       severity: "warn",
       category: "invention",
       message: `${ungrounded} long sentence(s) lack evidence citations. Prefer claims tied to [ach_…]/[fb_…]/[goal_…] ids.`,
-      excerpt: samples[0],
+      excerpt: sample,
+      suggestedFix: citeId
+        ? {
+            action: "cite",
+            label: `Cite [${citeId}]`,
+            citeId,
+            newText: sample ? `${sample.replace(/\.*$/, "")} [${citeId}].` : undefined,
+          }
+        : softened && sample
+          ? {
+              action: "soften",
+              label: "Soften absolute language",
+              newText: softened,
+            }
+          : undefined,
     });
   }
 
-  // Numeric claims without citations are high invention risk (% is non-word — no trailing \b)
   const metricClaims = [
     ...body.matchAll(/(\d+%|\$\d[\d,]*(?:\.\d+)?|\b\d{2,}\s*(?:users|PRs|tickets|customers)\b)/gi),
   ];
   for (const m of metricClaims.slice(0, 5)) {
     const idx = m.index ?? 0;
     const window = body.slice(Math.max(0, idx - 80), idx + 80);
     if (!/\[[a-z]+_[A-Za-z0-9_-]+\]/.test(window)) {
+      const citeId = pickCiteId(allowlist);
+      const excerpt = m[0];
       findings.push({
         id: `metric_${idx}`,
         severity: "warn",
         category: "invention",
-        message: `Metric “${m[0]}” appears without a nearby evidence citation — verify or cite the source.`,
-        excerpt: m[0],
+        message: `Metric “${excerpt}” appears without a nearby evidence citation — verify or cite the source.`,
+        excerpt,
+        suggestedFix: citeId
+          ? {
+              action: "cite",
+              label: `Cite [${citeId}] near metric`,
+              citeId,
+              newText: `${excerpt} [${citeId}]`,
+            }
+          : {
+              action: "soften",
+              label: "Soften / qualify metric",
+              newText: softenText(excerpt),
+            },
       });
     }
   }
 
+  // Absolute phrasing without cites
+  for (const m of body.matchAll(ABSOLUTE_WORDS)) {
+    const idx = m.index ?? 0;
+    const window = body.slice(Math.max(0, idx - 40), idx + (m[0]?.length ?? 0) + 40);
+    if (/\[[a-z]+_[A-Za-z0-9_-]+\]/.test(window)) continue;
+    const word = m[0];
+    findings.push({
+      id: `absolute_${idx}`,
+      severity: "info",
+      category: "invention",
+      message: `Absolute wording “${word}” is easy to overclaim without evidence.`,
+      excerpt: word,
+      suggestedFix: {
+        action: "soften",
+        label: "Soften wording",
+        newText: softenText(word),
+      },
+    });
+  }
+
   return findings;
 }
 
@@ -88,3 +163,20 @@ export function mergeLintFindings(...groups: BiasFinding[][]): BiasFinding[] {
   }
   return out;
 }
+
+/** Apply a suggested faithfulness fix to draft text. Returns null if excerpt not found. */
+export function applyFaithfulnessFix(
+  text: string,
+  finding: Pick<BiasFinding, "excerpt" | "suggestedFix">,
+): string | null {
+  const fix = finding.suggestedFix;
+  const excerpt = finding.excerpt;
+  if (!fix || !excerpt) return null;
+  if (fix.action === "strip_cite") {
+    if (!text.includes(excerpt)) return null;
+    return text.replace(excerpt, "").replace(/  +/g, " ").replace(/ \./g, ".");
+  }
+  if (!fix.newText) return null;
+  if (!text.includes(excerpt)) return null;
+  return text.replace(excerpt, fix.newText);
+}