feat(memory): add retention model test gaps and health diagnostics

sdwolf4103 · sdwolf4103 · commit 36b78ea91c25 · 2026-04-29T15:26:44.000+08:00
Wave 1 - P0 Test Gaps:
- Add hard stale prune removed regression test
- Add dormant overlap tests (entry created during dormancy)
- Add invalid timestamp NaN protection test
- Add reinforcement ordering test with reference type
- Add dedupe same-session/under-1hr guard tests
- Fix NaN handling with Number.isFinite check

Wave 2 - Helper Functions:
- Add timestampMs() for safe timestamp conversion
- Add isSafetyCriticalForDiag() aligned with runtime

Wave 3 - Health Output Format:
- Fix top rendered candidates sorted by strength (not text length)
- Add stored vs rendered counts breakdown
- Add type caps and global cap overflow display
- Track globalCapped array explicitly
- Add dormant status section

Wave 4 - Monitoring Metrics:
- Add high_importance_ratio (alert &gt; 30%)
- Add safety_critical_count (alert &gt; 5)
- Add max_reinforced_count (alert &gt; 10% active)

Wave 5 - Integration Fixture:
- Add 34-entry over-cap test
- Add mixed retention regression fixture
- Test TYPE_MAX caps, safety-critical exemption, reinforcement ordering

Tests: 224 → 237
diff --git a/scripts/memory-diag.ts b/scripts/memory-diag.ts
@@ -12,7 +12,7 @@ import { dataHome, extractionRejectionLogPath, migrationLogPath, workspaceKey, w
 import { assessMemoryQuality, HARD_QUALITY_REASONS } from "../src/memory-quality.ts";
 import { redactCredentials } from "../src/redaction.ts";
 import { scanWorkspaceResidues } from "../src/workspace-cleanup.ts";
-import { renderWorkspaceMemory } from "../src/workspace-memory.ts";
+import { calculateRetentionStrength, renderWorkspaceMemory } from "../src/workspace-memory.ts";
 import type { LongTermMemoryEntry, LongTermSource, LongTermType, PendingMemoryJournalStore, WorkspaceMemoryStore } from "../src/types.ts";
 import { LONG_TERM_LIMITS, PROMOTION_RETRY_LIMITS } from "../src/types.ts";
 
@@ -65,6 +65,14 @@ type MigrationLogRecord = {
 };
 
 const TYPES: LongTermType[] = ["feedback", "decision", "project", "reference"];
+const TYPE_MAX_FOR_DIAG: Record<LongTermType, number> = {
+  feedback: 10,
+  decision: 10,
+  project: 8,
+  reference: 6,
+};
+const WORKSPACE_DORMANT_AFTER_DAYS_FOR_DIAG = 14;
+const DORMANT_DECAY_MULTIPLIER_FOR_DIAG = 0.25;
 const SUSPICIOUS_REASONS = [
   "progress_snapshot",
   "active_file_snapshot",
@@ -229,6 +237,68 @@ function ageDays(entry: LongTermMemoryEntry): number | null {
   return Math.floor((Date.now() - time) / 86_400_000);
 }
 
+function formatStrength(value: number): string {
+  return Number.isFinite(value) ? value.toFixed(3) : "0.000";
+}
+
+function daysSinceIso(value: string | undefined, now = Date.now()): number | null {
+  if (!value) return null;
+  const ms = new Date(value).getTime();
+  if (!Number.isFinite(ms)) return null;
+  return Math.max(0, (now - ms) / 86_400_000);
+}
+
+function formatPercent(ratio: number): string {
+  return `${(ratio * 100).toFixed(1)}%`;
+}
+
+type RetentionDiagItem = {
+  entry: LongTermMemoryEntry;
+  strength: number;
+};
+
+function isSafetyCriticalForDiag(entry: LongTermMemoryEntry): boolean {
+  return entry.safetyCritical === true;
+}
+
+function retentionCandidatesForDiag(store: WorkspaceMemoryStore): {
+  sorted: RetentionDiagItem[];
+  rendered: RetentionDiagItem[];
+  typeCapped: RetentionDiagItem[];
+  globalCapped: RetentionDiagItem[];
+} {
+  const now = Date.now();
+  const active = store.entries.filter(entry => entry.status !== "superseded");
+  const sorted = active
+    .map(entry => ({ entry, strength: calculateRetentionStrength(entry, now, store.lastActivityAt) }))
+    .sort((a, b) => b.strength - a.strength || a.entry.id.localeCompare(b.entry.id));
+
+  const rendered: RetentionDiagItem[] = [];
+  const typeCapped: RetentionDiagItem[] = [];
+  const globalCapped: RetentionDiagItem[] = [];
+  const typeCounts: Partial<Record<LongTermType, number>> = {};
+
+  for (const item of sorted) {
+    if (!isSafetyCriticalForDiag(item.entry)) {
+      const count = typeCounts[item.entry.type] ?? 0;
+      const max = TYPE_MAX_FOR_DIAG[item.entry.type] ?? Infinity;
+      if (count >= max) {
+        typeCapped.push(item);
+        continue;
+      }
+      typeCounts[item.entry.type] = count + 1;
+    }
+
+    if (rendered.length < LONG_TERM_LIMITS.maxEntries) {
+      rendered.push(item);
+    } else {
+      globalCapped.push(item);
+    }
+  }
+
+  return { sorted, rendered, typeCapped, globalCapped };
+}
+
 function promotionLimit(source: LongTermSource): number {
   if (source === "manual") return PROMOTION_RETRY_LIMITS.maxManualAttempts;
   return PROMOTION_RETRY_LIMITS.maxExplicitAttempts;
@@ -333,10 +403,13 @@ async function printWorkspaceHealth(input: {
 
   const active = store.entries.filter(entry => entry.status !== "superseded");
   const superseded = store.entries.filter(entry => entry.status === "superseded");
+  const retention = retentionCandidatesForDiag(store);
+  const renderedEntries = retention.rendered.map(item => item.entry);
   const renderedEstimate = renderWorkspaceMemory(store).length;
 
-  console.log(`Active memories: ${active.length}`);
+  console.log(`Stored active memories: ${active.length}`);
   console.log(`Superseded memories: ${superseded.length}`);
+  console.log(`Rendered candidates: ${renderedEntries.length}`);
   console.log(`Rendered estimate: ${renderedEstimate.toLocaleString()} chars`);
   console.log("");
 
@@ -356,12 +429,18 @@ async function printWorkspaceHealth(input: {
 
   console.log("By type:");
   for (const type of TYPES) {
-    const activeCount = active.filter(entry => entry.type === type).length;
+    const storedCount = active.filter(entry => entry.type === type).length;
+    const renderedCount = renderedEntries.filter(entry => entry.type === type).length;
     const supersededCount = superseded.filter(entry => entry.type === type).length;
-    console.log(`  ${type.padEnd(9)} active=${String(activeCount).padEnd(3)} superseded=${supersededCount}`);
+    console.log(`  ${type.padEnd(9)} stored=${String(storedCount).padEnd(3)} rendered=${String(renderedCount).padEnd(3)} typeCap=${TYPE_MAX_FOR_DIAG[type]} superseded=${supersededCount}`);
   }
   console.log("");
 
+  console.log("Retention caps:");
+  console.log(`  type-capped entries: ${retention.typeCapped.length}`);
+  console.log(`  global-cap overflow: ${retention.globalCapped.length}`);
+  console.log("");
+
   const olderThan30 = active.filter(entry => (ageDays(entry) ?? 0) > 30).length;
   const olderThan90 = active.filter(entry => (ageDays(entry) ?? 0) > 90).length;
   const staleMarked = active.filter(entry => {
@@ -374,6 +453,33 @@ async function printWorkspaceHealth(input: {
   console.log(`  older than 90d: ${olderThan90}`);
   console.log("");
 
+  const wallDaysSinceActivity = daysSinceIso(store.lastActivityAt);
+  const dormantDiscountActive = wallDaysSinceActivity !== null && wallDaysSinceActivity > WORKSPACE_DORMANT_AFTER_DAYS_FOR_DIAG;
+  const dormantDaysPastGrace = wallDaysSinceActivity === null
+    ? 0
+    : Math.max(0, wallDaysSinceActivity - WORKSPACE_DORMANT_AFTER_DAYS_FOR_DIAG);
+  console.log("Dormancy:");
+  console.log(`  lastActivityAt: ${store.lastActivityAt ?? "(missing)"}`);
+  console.log(`  wall days since activity: ${wallDaysSinceActivity === null ? "unknown" : wallDaysSinceActivity.toFixed(1)}`);
+  console.log(`  dormant discount active: ${dormantDiscountActive ? "yes" : "no"}`);
+  console.log(`  dormant days past grace: ${dormantDaysPastGrace.toFixed(1)}`);
+  console.log(`  dormant multiplier: ${DORMANT_DECAY_MULTIPLIER_FOR_DIAG}`);
+  console.log("");
+
+  const highImportanceCount = active.filter(entry => entry.userImportance === "high").length;
+  const safetyCriticalCount = active.filter(isSafetyCriticalForDiag).length;
+  const maxReinforcedCount = active.filter(entry => (entry.reinforcementCount ?? 0) >= 6).length;
+  const highImportanceRatio = active.length === 0 ? 0 : highImportanceCount / active.length;
+  const maxReinforcedRatio = active.length === 0 ? 0 : maxReinforcedCount / active.length;
+  const highImportanceAlert = highImportanceRatio > 0.3;
+  const safetyCriticalAlert = safetyCriticalCount > 5;
+  const maxReinforcedAlert = maxReinforcedRatio > 0.1;
+  console.log("Retention monitoring:");
+  console.log(`  high_importance_ratio: ${formatPercent(highImportanceRatio)} (alert > 30%)${highImportanceAlert ? " ALERT" : ""}`);
+  console.log(`  safety_critical_count: ${safetyCriticalCount} (alert > 5)${safetyCriticalAlert ? " ALERT" : ""}`);
+  console.log(`  max_reinforced_count: ${maxReinforcedAlert ? `${maxReinforcedCount} (${formatPercent(maxReinforcedRatio)}, alert > 10%) ALERT` : `${maxReinforcedCount} (alert > 10% active)`}`);
+  console.log("");
+
   const qualityByEntry = active.map(entry => ({ entry, quality: assessMemoryQuality(entry) }));
   const duplicateCounts = countBy(active.map(entry => `${entry.type}:${canonicalMemoryText(entry.text)}`));
   const duplicateExtras = [...duplicateCounts.values()].reduce((sum, count) => sum + Math.max(0, count - 1), 0);
@@ -400,12 +506,23 @@ async function printWorkspaceHealth(input: {
 
   console.log("");
   console.log("Top rendered candidates:");
-  const top = [...active].sort((a, b) => b.text.length - a.text.length).slice(0, 5);
+  const top = retention.rendered.slice(0, 5);
   if (top.length === 0) {
     console.log("  (none)");
   } else {
-    for (const entry of top) {
-      console.log(`  - [${entry.type}] ${truncate(cleanText(entry.text, input.raw))}`);
+    for (const item of top) {
+      console.log(`  - strength=${formatStrength(item.strength)} [${item.entry.type}] ${truncate(cleanText(item.entry.text, input.raw))}`);
+    }
+  }
+
+  console.log("");
+  console.log("Weakest active memories:");
+  const weakest = retention.sorted.slice(-5).reverse();
+  if (weakest.length === 0) {
+    console.log("  (none)");
+  } else {
+    for (const item of weakest) {
+      console.log(`  - strength=${formatStrength(item.strength)} [${item.entry.type}] ${truncate(cleanText(item.entry.text, input.raw))}`);
     }
   }
 }
diff --git a/src/workspace-memory.ts b/src/workspace-memory.ts
@@ -67,6 +67,11 @@ export function calculateEffectiveHalfLife(memory: LongTermMemoryEntry): number
   return BASE_HALF_LIFE_DAYS / factor;
 }
 
+function timestampMs(value: unknown, fallback: number): number {
+  const ms = typeof value === "number" ? value : new Date(String(value)).getTime();
+  return Number.isFinite(ms) ? ms : fallback;
+}
+
 export function calculateRetentionStrength(
   memory: LongTermMemoryEntry,
   now: number,
@@ -76,14 +81,16 @@ export function calculateRetentionStrength(
   const effectiveHalfLife = calculateEffectiveHalfLife(memory);
 
   // Use retentionClock if available, fallback to updatedAt.
-  const retentionStart = memory.retentionClock ?? memory.updatedAt;
-  const createdAtMs = new Date(retentionStart).getTime();
+  const retentionStart = Number.isFinite(memory.retentionClock)
+    ? memory.retentionClock
+    : memory.updatedAt ?? memory.createdAt;
+  const createdAtMs = timestampMs(retentionStart, now);
   const effectiveAgeDays = calculateEffectiveAgeDays(createdAtMs, now, lastActivityAt);
 
   // Calculate strength using exponential decay.
   const strength = initialStrength * Math.pow(2, -effectiveAgeDays / effectiveHalfLife);
 
-  return Math.max(0, strength);
+  return Number.isFinite(strength) ? Math.max(0, strength) : 0;
 }
 
 export function calculateDormantDays(store: WorkspaceMemoryStore, now: number): number {
diff --git a/tests/memory-diag.test.ts b/tests/memory-diag.test.ts
@@ -0,0 +1,143 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import type { LongTermMemoryEntry, WorkspaceMemoryStore } from "../src/types.ts";
+import { LONG_TERM_LIMITS } from "../src/types.ts";
+import { workspaceKey, workspaceMemoryPath } from "../src/paths.ts";
+
+const execFileAsync = promisify(execFile);
+const repoRoot = join(dirname(fileURLToPath(import.meta.url)), "..");
+
+function entry(id: string, text: string, type: LongTermMemoryEntry["type"]): LongTermMemoryEntry {
+  const now = new Date().toISOString();
+  return {
+    id,
+    type,
+    text,
+    source: "compaction",
+    confidence: 0.75,
+    status: "active",
+    createdAt: now,
+    updatedAt: now,
+  };
+}
+
+async function writeWorkspaceStore(root: string, entries: LongTermMemoryEntry[], options: { lastActivityAt?: string; omitLastActivityAt?: boolean } = {}): Promise<void> {
+  const key = await workspaceKey(root);
+  const path = await workspaceMemoryPath(root);
+  const now = new Date().toISOString();
+  const store: WorkspaceMemoryStore = {
+    version: 1,
+    workspace: { root, key },
+    limits: { maxRenderedChars: LONG_TERM_LIMITS.maxRenderedChars, maxEntries: LONG_TERM_LIMITS.maxEntries },
+    entries,
+    migrations: [],
+    updatedAt: now,
+  };
+  if (!options.omitLastActivityAt) store.lastActivityAt = options.lastActivityAt ?? now;
+
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(path, JSON.stringify(store, null, 2), "utf8");
+}
+
+async function runMemoryDiagHealth(root: string): Promise<string> {
+  const { stdout } = await execFileAsync(process.execPath, [
+    "--experimental-strip-types",
+    "scripts/memory-diag.ts",
+    "health",
+    "--workspace",
+    root,
+  ], { cwd: repoRoot });
+
+  return stdout;
+}
+
+test("memory health reports stored vs rendered retention counts", async () => {
+  const root = await mkdtemp(join(tmpdir(), "opencode-memory-diag-"));
+  try {
+    const entries: LongTermMemoryEntry[] = [
+      ...Array.from({ length: 17 }, (_, i) => entry(`feedback-${i}`, `Unique feedback preference for memory health ${i}`, "feedback")),
+      ...Array.from({ length: 11 }, (_, i) => entry(`decision-${i}`, `Unique durable decision for memory health ${i}`, "decision")),
+    ];
+    await writeWorkspaceStore(root, entries);
+
+    const stdout = await runMemoryDiagHealth(root);
+
+    assert.match(stdout, /Stored active memories:/);
+    assert.match(stdout, /Rendered candidates:/);
+    assert.match(stdout, /feedback\s+stored=17\s+rendered=10/);
+    assert.match(stdout, /Top rendered candidates:\n\s+- strength=/);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("memory health reports dormancy and retention monitoring alerts", async () => {
+  const root = await mkdtemp(join(tmpdir(), "opencode-memory-diag-"));
+  try {
+    const lastActivityAt = new Date(Date.now() - 19 * 24 * 60 * 60 * 1000).toISOString();
+    const entries = Array.from({ length: 10 }, (_, i) => ({
+      ...entry(`monitoring-${i}`, `Unique monitoring memory ${i} for retention health`, i % 2 === 0 ? "feedback" : "decision"),
+      userImportance: i < 4 ? "high" as const : "normal" as const,
+      safetyCritical: i < 6,
+      reinforcementCount: i < 2 ? 6 : 0,
+    }));
+    await writeWorkspaceStore(root, entries, { lastActivityAt });
+
+    const stdout = await runMemoryDiagHealth(root);
+
+    assert.match(stdout, /Dormancy:/);
+    assert.match(stdout, /wall days since activity: 19\.0/);
+    assert.match(stdout, /dormant discount active: yes/);
+    assert.match(stdout, /dormant days past grace: 5\.0/);
+    assert.match(stdout, /high_importance_ratio: 40\.0% .* ALERT/);
+    assert.match(stdout, /safety_critical_count: 6 .* ALERT/);
+    assert.match(stdout, /max_reinforced_count: 2 \(20\.0%, alert > 10%\) ALERT/);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("memory health reports global cap overflow separately from type caps", async () => {
+  const root = await mkdtemp(join(tmpdir(), "opencode-memory-diag-"));
+  try {
+    const entries: LongTermMemoryEntry[] = [
+      ...Array.from({ length: 10 }, (_, i) => entry(`global-feedback-${i}`, `Unique global feedback preference ${i}`, "feedback")),
+      ...Array.from({ length: 10 }, (_, i) => entry(`global-decision-${i}`, `Unique global durable decision ${i}`, "decision")),
+      ...Array.from({ length: 8 }, (_, i) => entry(`global-project-${i}`, `Unique global project fact ${i}`, "project")),
+      ...Array.from({ length: 6 }, (_, i) => entry(`global-reference-${i}`, `Unique global reference fact ${i}`, "reference")),
+    ];
+    await writeWorkspaceStore(root, entries);
+
+    const stdout = await runMemoryDiagHealth(root);
+
+    assert.match(stdout, /Rendered candidates: 28/);
+    assert.match(stdout, /type-capped entries: 0/);
+    assert.match(stdout, /global-cap overflow: 6/);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("memory health reports missing dormancy and non-alert monitoring defaults", async () => {
+  const root = await mkdtemp(join(tmpdir(), "opencode-memory-diag-"));
+  try {
+    await writeWorkspaceStore(root, [], { omitLastActivityAt: true });
+
+    const stdout = await runMemoryDiagHealth(root);
+
+    assert.match(stdout, /lastActivityAt: \(missing\)/);
+    assert.match(stdout, /wall days since activity: unknown/);
+    assert.match(stdout, /dormant discount active: no/);
+    assert.match(stdout, /high_importance_ratio: 0\.0% \(alert > 30%\)\n/);
+    assert.match(stdout, /safety_critical_count: 0 \(alert > 5\)\n/);
+    assert.match(stdout, /max_reinforced_count: 0 \(alert > 10% active\)/);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
diff --git a/tests/workspace-memory.test.ts b/tests/workspace-memory.test.ts
