Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
name: CI

on:
push:
branches: [main]
pull_request:
branches: [main]

concurrency:
group: ci-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

jobs:
test:
name: Test (Ubuntu)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 10.33.3
- uses: actions/setup-node@v4
with:
node-version: 22
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: pnpm rebuild better-sqlite3
- run: pnpm test

desktop:
name: Desktop ${{ matrix.platform }}
needs: test
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
strategy:
fail-fast: false
matrix:
include:
- platform: ubuntu-22.04
args: --target x86_64-unknown-linux-gnu
rust_target: x86_64-unknown-linux-gnu
- platform: macos-latest
args: --target aarch64-apple-darwin
rust_target: aarch64-apple-darwin
- platform: windows-latest
args: --target x86_64-pc-windows-msvc
rust_target: x86_64-pc-windows-msvc
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 10.33.3
- uses: actions/setup-node@v4
with:
node-version: 22
cache: pnpm
- name: Install Linux WebKit deps
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf
- uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.rust_target }}
- uses: Swatinem/rust-cache@v2
with:
workspaces: apps/desktop/src-tauri -> target
- run: pnpm install --frozen-lockfile
- run: pnpm rebuild better-sqlite3
- name: Bundle API sidecar + Node
run: node apps/desktop/scripts/bundle-api-sidecar.mjs --node
env:
PRM_BUNDLE_NODE: "1"
- uses: tauri-apps/tauri-action@v0
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
projectPath: apps/desktop
args: ${{ matrix.args }}
includeUpdaterJson: false
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,4 @@ data/
apps/ui/dist
apps/api/dist
apps/desktop/src-tauri/target
apps/desktop/src-tauri/sidecars/
31 changes: 31 additions & 0 deletions apps/api/src/faithfulness.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import { mergeLintFindings, runFaithfulnessLint } from "../src/faithfulness.js";

describe("faithfulness lint", () => {
it("flags unknown citations", () => {
const findings = runFaithfulnessLint("Shipped [ach_real] and [ach_fake].", ["ach_real"]);
assert.ok(findings.some((f) => f.category === "invention" && /ach_fake/.test(f.message)));
});

it("flags ungrounded long sentences", () => {
const text = [
"They completely transformed the platform architecture across every squad this half.",
"Customer satisfaction improved dramatically while latency dropped for all regions.",
"See [ach_1] for the migration they led.",
].join(" ");
const findings = runFaithfulnessLint(text, ["ach_1"]);
assert.ok(findings.some((f) => f.id === "ungrounded_claims"));
});

it("flags bare metrics", () => {
const findings = runFaithfulnessLint("Reduced p99 latency by 40% across checkout.", []);
assert.ok(findings.some((f) => f.category === "invention" && /40%/.test(f.message)));
});

it("merges findings without dupes", () => {
const a = runFaithfulnessLint("Reduced cost by 25% somehow.", []);
const merged = mergeLintFindings(a, a);
assert.equal(merged.length, a.length);
});
});
90 changes: 90 additions & 0 deletions apps/api/src/faithfulness.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import type { BiasFinding } from "@prm/shared";
import { extractCitationIds } from "./citations.js";

const SENTENCE_SPLIT = /(?<=[.!?])\s+(?=[A-Z("[])|(?:\n+)/;

/** Claims that look substantive but lack any [id] citation nearby. */
export function runFaithfulnessLint(text: string, allowlist: string[] = []): BiasFinding[] {
const findings: BiasFinding[] = [];
const body = text.trim();
if (!body) return findings;

const allowed = new Set(allowlist);
const cited = extractCitationIds(body);
for (const id of cited) {
if (allowlist.length && !allowed.has(id)) {
findings.push({
id: `unknown_cite_${id}`,
severity: "warn",
category: "invention",
message: `Citation [${id}] is not in the evidence allowlist — remove or replace with a real evidence id.`,
excerpt: `[${id}]`,
});
}
}

const sentences = body
.split(SENTENCE_SPLIT)
.map((s) => s.trim())
.filter((s) => s.length > 40);

let ungrounded = 0;
const samples: string[] = [];
for (const sentence of sentences) {
if (/^(summary|strengths|growth|overall|note|private)\b/i.test(sentence)) continue;
if (!/\[[a-z]+_[A-Za-z0-9_-]+\]/.test(sentence)) {
// Skip questions / instructions
if (/\?$/.test(sentence) || /^(please|consider|remember)\b/i.test(sentence)) continue;
// Skip name-ish lists (mostly Title Case tokens, almost no verbs/stopwords)
const words = sentence.split(/\s+/).filter(Boolean);
const titleish = words.filter((w) => /^[A-Z][a-z]+/.test(w.replace(/[.,]/g, ""))).length;
if (words.length >= 3 && titleish / words.length >= 0.7) continue;
ungrounded += 1;
if (samples.length < 3) samples.push(sentence.slice(0, 120));
}
}

if (ungrounded >= 2) {
findings.push({
id: "ungrounded_claims",
severity: "warn",
category: "invention",
message: `${ungrounded} long sentence(s) lack evidence citations. Prefer claims tied to [ach_…]/[fb_…]/[goal_…] ids.`,
excerpt: samples[0],
});
}

// Numeric claims without citations are high invention risk (% is non-word — no trailing \b)
const metricClaims = [
...body.matchAll(/(\d+%|\$\d[\d,]*(?:\.\d+)?|\b\d{2,}\s*(?:users|PRs|tickets|customers)\b)/gi),
];
for (const m of metricClaims.slice(0, 5)) {
const idx = m.index ?? 0;
const window = body.slice(Math.max(0, idx - 80), idx + 80);
if (!/\[[a-z]+_[A-Za-z0-9_-]+\]/.test(window)) {
findings.push({
id: `metric_${idx}`,
severity: "warn",
category: "invention",
message: `Metric “${m[0]}” appears without a nearby evidence citation — verify or cite the source.`,
excerpt: m[0],
});
}
}

return findings;
}

export function mergeLintFindings(...groups: BiasFinding[][]): BiasFinding[] {
const seen = new Set<string>();
const out: BiasFinding[] = [];
for (const group of groups) {
for (const f of group) {
const key = `${f.category}:${f.id}:${f.message}`;
if (seen.has(key)) continue;
seen.add(key);
out.push(f);
}
}
return out;
}
Loading
Loading