Skip to content

Commit f863939

Browse files
committed
fix(usage): persist scan-cache predecessor freshness across restart
The v4 codec rewrites every retained row to the current 18-field shape, so a predecessor entry (rows written before numeric quality metadata existed) decoded as `declared` after the first persist. A fallback entry kept while its transcript was temporarily unreadable was therefore promoted to current metadata on save; after a restart it was served warm (unchanged file) or resumed from its stale prefix (grown file), skipping the cold re-parse that asserts quality. Add a file-level `lq` provenance marker alongside the existing `li` legacy marker. Encode writes `lq: 1` for a predecessor entry; decode treats `lq === 1` as predecessor regardless of the rewritten row length. The marker is dropped only when a successful full parse replaces the entry, and a current parser's genuinely partial measurement (carrying its own completeness field) stays `declared`. Recovered from the M3E local worktree (uncommitted at reviewed head af70495); codec fix and codec/service regressions transferred unchanged in substance, plus an explicit stale-prefix resume regression. Refs: METRICS-1:M3F
1 parent af70495 commit f863939

4 files changed

Lines changed: 297 additions & 2 deletions

File tree

‎apps/server/src/usage/UsageService.test.ts‎

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -799,6 +799,235 @@ describe("UsageService", () => {
799799
}).pipe(Effect.scoped),
800800
);
801801

802+
it.live(
803+
"keeps a persisted predecessor fallback predecessor, then cold re-classifies on recovery",
804+
() =>
805+
Effect.gen(function* () {
806+
const { transcript, settings, home } = yield* setup;
807+
const secondPath = NodePath.join(home, "claude", "projects", "proj", "second.jsonl");
808+
// The target parses under the current parser as `invalid` (a recognised
809+
// field with an invalid value). The predecessor cache stored it as an
810+
// observed/partial zero because numeric quality metadata did not exist
811+
// then; a warm or resumed read would keep that stale classification.
812+
const invalidContent =
813+
encodeUnknownJsonString({
814+
type: "assistant",
815+
timestamp: "2026-08-01T10:00:00Z",
816+
requestId: "req_live",
817+
sessionId: "session-1",
818+
message: { id: "msg_live", model: "claude-fable-5", usage: { input_tokens: null } },
819+
}) + "\n";
820+
yield* Effect.promise(() => NodeFSP.writeFile(transcript, invalidContent));
821+
yield* Effect.promise(() => NodeFSP.writeFile(secondPath, claudeLine(2, 7)));
822+
823+
yield* Effect.gen(function* () {
824+
const config = yield* ServerConfig.ServerConfig;
825+
const livePath = yield* Effect.promise(() => NodeFSP.realpath(transcript));
826+
const liveStat = yield* Effect.promise(() => NodeFSP.stat(livePath));
827+
const scanCachePath = NodePath.join(config.stateDir, "usage-scan-cache.json");
828+
const TS = Date.parse("2026-08-01T10:00:00Z");
829+
// Seed a predecessor v4 entry for the target, matching the file exactly
830+
// so only the stale quality metadata (not size/mtime) forces the cold
831+
// re-parse.
832+
yield* Effect.promise(() =>
833+
NodeFSP.writeFile(
834+
scanCachePath,
835+
encodeUnknownJsonString({
836+
version: 4,
837+
models: ["claude-fable-5"],
838+
sessions: ["session-1"],
839+
files: {
840+
[livePath]: {
841+
s: liveStat.size,
842+
m: liveStat.mtimeMs,
843+
p: "claude",
844+
r: [
845+
[
846+
TS,
847+
0,
848+
0,
849+
0,
850+
0,
851+
0,
852+
0,
853+
0,
854+
"msg_live:",
855+
null,
856+
"req_live",
857+
"msg_live",
858+
null,
859+
0,
860+
0,
861+
],
862+
],
863+
t: [],
864+
o: liveStat.size,
865+
gl: 0,
866+
gh: 0,
867+
cs: null,
868+
},
869+
},
870+
}),
871+
),
872+
);
873+
// Deterministic failure seam: make the target unreadable, then let a
874+
// second readable file change so the cache is persisted while the
875+
// target's retained fallback row is still the predecessor one.
876+
yield* Effect.promise(() => NodeFSP.chmod(livePath, 0o000));
877+
878+
const service = yield* UsageService.make;
879+
const failed = yield* service.readSummary(WINDOW);
880+
assert.strictEqual(totalOutputTokens(failed), 7);
881+
882+
const afterFail = decodeUnknownJsonString(
883+
yield* Effect.promise(() => NodeFSP.readFile(scanCachePath, "utf8")),
884+
) as { files: Record<string, { lq?: number; r: unknown[][] }> };
885+
// The persisted fallback keeps the predecessor provenance, not just the
886+
// 15-field shape the encoder rewrites.
887+
assert.strictEqual(afterFail.files[livePath]?.lq, 1);
888+
assert.strictEqual(afterFail.files[livePath]?.r[0]?.[13], 0);
889+
890+
// Restore access without changing size or mtime, then restart.
891+
yield* Effect.promise(() => NodeFSP.chmod(livePath, 0o644));
892+
const restarted = yield* UsageService.make;
893+
const recovered = yield* restarted.readSummary(WINDOW);
894+
// One full cold parse recovers the corrected all-invalid
895+
// classification and counts it as malformed, not a measured zero.
896+
assert.strictEqual(
897+
recovered.sources.find((source) => source.fingerprint.provider === "claude")
898+
?.malformedRecords,
899+
1,
900+
);
901+
assert.strictEqual(totalOutputTokens(recovered), 7);
902+
903+
const afterRecover = decodeUnknownJsonString(
904+
yield* Effect.promise(() => NodeFSP.readFile(scanCachePath, "utf8")),
905+
) as { files: Record<string, { lq?: number; r: unknown[][] }> };
906+
// Freshness is cleared only after the successful re-parse; the row now
907+
// carries the current invalid measurement code (3).
908+
assert.strictEqual(afterRecover.files[livePath]?.lq, undefined);
909+
assert.strictEqual(afterRecover.files[livePath]?.r[0]?.[13], 3);
910+
911+
// The next unchanged scan is warm: byte-identical cache, stable totals.
912+
const beforeWarm = yield* Effect.promise(() => NodeFSP.readFile(scanCachePath, "utf8"));
913+
const warm = yield* restarted.readSummary(WINDOW);
914+
assert.strictEqual(totalOutputTokens(warm), 7);
915+
const afterWarm = yield* Effect.promise(() => NodeFSP.readFile(scanCachePath, "utf8"));
916+
assert.strictEqual(afterWarm, beforeWarm);
917+
}).pipe(
918+
Effect.provide(
919+
serviceLayers({ prefix: "usage-service-freshness-persist-test", home, settings }),
920+
),
921+
);
922+
}).pipe(Effect.scoped),
923+
);
924+
925+
it.live("does not resume a stale predecessor prefix after a restarted persist", () =>
926+
Effect.gen(function* () {
927+
const { transcript, settings, home } = yield* setup;
928+
const secondPath = NodePath.join(home, "claude", "projects", "proj", "second.jsonl");
929+
// The target parses as `invalid` under the current parser; the predecessor
930+
// entry only has a stale zero-total partial row for it.
931+
const invalidContent =
932+
encodeUnknownJsonString({
933+
type: "assistant",
934+
timestamp: "2026-08-01T10:00:00Z",
935+
requestId: "req_live",
936+
sessionId: "session-1",
937+
message: { id: "msg_live", model: "claude-fable-5", usage: { input_tokens: null } },
938+
}) + "\n";
939+
yield* Effect.promise(() => NodeFSP.writeFile(transcript, invalidContent));
940+
yield* Effect.promise(() => NodeFSP.writeFile(secondPath, claudeLine(2, 7)));
941+
942+
yield* Effect.gen(function* () {
943+
const config = yield* ServerConfig.ServerConfig;
944+
const livePath = yield* Effect.promise(() => NodeFSP.realpath(transcript));
945+
const liveStat = yield* Effect.promise(() => NodeFSP.stat(livePath));
946+
const scanCachePath = NodePath.join(config.stateDir, "usage-scan-cache.json");
947+
const TS = Date.parse("2026-08-01T10:00:00Z");
948+
// A predecessor entry whose parse position already sits at the current
949+
// end of the file, matching size/mtime exactly.
950+
yield* Effect.promise(() =>
951+
NodeFSP.writeFile(
952+
scanCachePath,
953+
encodeUnknownJsonString({
954+
version: 4,
955+
models: ["claude-fable-5"],
956+
sessions: ["session-1"],
957+
files: {
958+
[livePath]: {
959+
s: liveStat.size,
960+
m: liveStat.mtimeMs,
961+
p: "claude",
962+
r: [
963+
[
964+
TS,
965+
0,
966+
0,
967+
0,
968+
0,
969+
0,
970+
0,
971+
0,
972+
"msg_live:",
973+
null,
974+
"req_live",
975+
"msg_live",
976+
null,
977+
0,
978+
0,
979+
],
980+
],
981+
t: [],
982+
o: liveStat.size,
983+
gl: 0,
984+
gh: 0,
985+
cs: null,
986+
},
987+
},
988+
}),
989+
),
990+
);
991+
// Unreadable target plus a second changed file forces a persist of the
992+
// retained fallback while the target entry is still predecessor.
993+
yield* Effect.promise(() => NodeFSP.chmod(livePath, 0o000));
994+
const service = yield* UsageService.make;
995+
const failed = yield* service.readSummary(WINDOW);
996+
assert.strictEqual(totalOutputTokens(failed), 7);
997+
const fallback = decodeUnknownJsonString(
998+
yield* Effect.promise(() => NodeFSP.readFile(scanCachePath, "utf8")),
999+
) as { files: Record<string, { lq?: number }> };
1000+
assert.strictEqual(fallback.files[livePath]?.lq, 1);
1001+
1002+
// Restore access and append a readable line so the file is strictly
1003+
// larger. A stale-but-declared entry would resume from the cached end
1004+
// offset and never re-see the invalid prefix; the persisted
1005+
// predecessor marker must force a full re-parse instead.
1006+
yield* Effect.promise(() => NodeFSP.chmod(livePath, 0o644));
1007+
yield* Effect.promise(() => NodeFSP.appendFile(livePath, claudeLine(3, 5)));
1008+
const restarted = yield* UsageService.make;
1009+
const recovered = yield* restarted.readSummary(WINDOW);
1010+
// The invalid prefix is reclassified (not retained as a measured zero)
1011+
// and the appended line is counted: 7 + 5 = 12.
1012+
assert.strictEqual(
1013+
recovered.sources.find((source) => source.fingerprint.provider === "claude")
1014+
?.malformedRecords,
1015+
1,
1016+
);
1017+
assert.strictEqual(totalOutputTokens(recovered), 12);
1018+
const after = decodeUnknownJsonString(
1019+
yield* Effect.promise(() => NodeFSP.readFile(scanCachePath, "utf8")),
1020+
) as { files: Record<string, { lq?: number; r: unknown[][] }> };
1021+
assert.strictEqual(after.files[livePath]?.lq, undefined);
1022+
assert.strictEqual(after.files[livePath]?.r.length, 2);
1023+
}).pipe(
1024+
Effect.provide(
1025+
serviceLayers({ prefix: "usage-service-stale-prefix-test", home, settings }),
1026+
),
1027+
);
1028+
}).pipe(Effect.scoped),
1029+
);
1030+
8021031
it.live("counts Codex and Grok invalid events as malformed while keeping valid tokens", () =>
8031032
Effect.gen(function* () {
8041033
const { settings, home } = yield* setup;

‎apps/server/src/usage/usageScanCache.test.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,9 @@ describe("scan cache round trip", () => {
183183

184184
expect(again.get("/deleted.jsonl")?.records[0]?.identityAvailable).toBe(false);
185185
expect(again.get("/deleted.jsonl")?.records[0]?.measurementCompleteness).toBe("partial");
186+
// The quality provenance survives the save too, so an erased-history entry
187+
// is not promoted to a declared measurement after a restart.
188+
expect(again.get("/deleted.jsonl")?.qualityMetadata).toBe("predecessor");
186189
});
187190

188191
it("drops an entry whose persisted parse state is corrupt", () => {
@@ -339,6 +342,7 @@ describe("legacy v3 cache history", () => {
339342

340343
expect(again.get("/deleted.jsonl")?.identity).toBe("unavailable");
341344
expect(again.get("/deleted.jsonl")?.records[0]?.totals.outputTokens).toBe(50);
345+
expect(again.get("/deleted.jsonl")?.qualityMetadata).toBe("predecessor");
342346
});
343347

344348
it("marks a freshly re-parsed entry declared so it can resume and enrich", () => {
@@ -463,6 +467,39 @@ describe("predecessor v4 format policy", () => {
463467
expect(decoded.get("/a.jsonl")?.qualityMetadata).toBe("declared");
464468
});
465469

470+
it("persists predecessor freshness across a save so it is not promoted on restart", () => {
471+
// The encoder rewrites every row to the current 18-field shape, so a
472+
// predecessor entry that is persisted (e.g. kept as a fallback while its
473+
// transcript is unreadable) would decode as `declared` from row length
474+
// alone and then be served warm forever, skipping the cold re-parse that
475+
// asserts numeric quality.
476+
const once = decodeScanCache(JSON.parse(JSON.stringify(PINNED_PREDECESSOR_DOCUMENT)));
477+
expect(once.get("/pred/live.jsonl")?.qualityMetadata).toBe("predecessor");
478+
479+
const encoded = encodeScanCache(once);
480+
expect(encoded.files["/pred/live.jsonl"]?.lq).toBe(1);
481+
482+
const again = decodeScanCache(JSON.parse(JSON.stringify(encoded)));
483+
expect(again.get("/pred/live.jsonl")?.qualityMetadata).toBe("predecessor");
484+
expect(
485+
again.get("/pred/live.jsonl")?.records.map((row) => row.measurementCompleteness),
486+
).toEqual(["partial", "partial", "partial"]);
487+
});
488+
489+
it("leaves a current partial entry declared across a save so it can stay warm", () => {
490+
// A legitimately partial measurement from the current parser carries the
491+
// completeness field; it is not stale parser-format metadata and must not
492+
// be marked predecessor or it would re-parse on every scan.
493+
const encoded = encodeScanCache(
494+
cacheWith([["/a.jsonl", 100, [record({ measurementCompleteness: "partial" })]]]),
495+
);
496+
expect(encoded.files["/a.jsonl"]?.lq).toBeUndefined();
497+
498+
const again = decodeScanCache(JSON.parse(JSON.stringify(encoded)));
499+
expect(again.get("/a.jsonl")?.qualityMetadata).toBe("declared");
500+
expect(again.get("/a.jsonl")?.records[0]?.measurementCompleteness).toBe("partial");
501+
});
502+
466503
it("treats a row with no appended fields as predecessor", () => {
467504
// A hand-built or truncated row that stops after the scope code is exactly
468505
// the predecessor shape and must be treated the same way.

‎apps/server/src/usage/usageScanCache.ts‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,10 @@ import type {
4444
// - A v4 row written by the predecessor (15 fields, no quality metadata)
4545
// decodes conservatively: completeness is `partial`, never `complete`, and
4646
// the entry is `qualityMetadata: "predecessor"` so an extant file is cold
47-
// re-parsed once. No row is discarded for the format change.
47+
// re-parsed once. No row is discarded for the format change. The marker is
48+
// persisted on the file (`lq`) because the encoder rewrites rows to the
49+
// current 18-field shape, so provenance cannot be recovered from row length
50+
// after a save/restart.
4851
// - Only v1/v2 (no parse position, different fork semantics) are rejected.
4952
const USAGE_SCAN_CACHE_VERSION = 4 as const;
5053
const LEGACY_USAGE_SCAN_CACHE_VERSION = 3 as const;
@@ -185,6 +188,15 @@ interface SerializedFile {
185188
* until an extant file is cold re-parsed. Absent on a fresh entry.
186189
*/
187190
readonly li?: number;
191+
/**
192+
* `1` when this entry's rows predate the current numeric quality metadata.
193+
* Like `li`, it is a file-level provenance marker: the encoder rewrites every
194+
* row to the current 18-field shape, so row length alone cannot tell a
195+
* predecessor entry apart from a freshly parsed one after a persist. Kept on
196+
* the file so a predecessor entry stays `predecessor` across restarts until a
197+
* successful full parse replaces it. Absent on a fresh entry.
198+
*/
199+
readonly lq?: number;
188200
}
189201

190202
interface SerializedCache {
@@ -244,6 +256,11 @@ export function encodeScanCache(cache: ScanCache): SerializedCache {
244256
gh: entry.position.guardHash,
245257
cs: entry.position.codexState,
246258
...(entry.identity === "unavailable" ? { li: 1 } : {}),
259+
// Persist the freshness provenance, not just the row shape. The rows are
260+
// always rewritten to the current 18-field form, so without this marker a
261+
// predecessor entry would decode as `declared` after the first persist and
262+
// be served warm forever, skipping the cold re-parse that asserts quality.
263+
...(entry.qualityMetadata === "predecessor" ? { lq: 1 } : {}),
247264
};
248265
}
249266

@@ -451,8 +468,13 @@ export function decodeScanCache(document: unknown): ScanCache {
451468
codexState,
452469
},
453470
identity: legacy ? "unavailable" : "declared",
471+
// `lq` records provenance the row shape cannot: the encoder always emits
472+
// the current 18-field rows, so a persisted predecessor entry would
473+
// otherwise decode as `declared` on the next start without any parse.
454474
qualityMetadata:
455-
decodedRecords.qualityDeclared && decodedTail.qualityDeclared ? "declared" : "predecessor",
475+
entry.lq === 1 || !decodedRecords.qualityDeclared || !decodedTail.qualityDeclared
476+
? "predecessor"
477+
: "declared",
456478
});
457479
}
458480

‎docs/internals/usage-attribution.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,13 @@ The scan cache version is still `4`. Supported-format policy:
155155
coverage. Missing quality metadata is never silently promoted. The entry is marked
156156
`qualityMetadata: "predecessor"`, so an extant file is cold re-parsed once and then
157157
accepted warm. No row is discarded for the format change.
158+
- The freshness provenance is persisted on the entry (`lq`), not inferred from row
159+
length. The encoder always rewrites rows to the current 18-field shape, so a
160+
predecessor entry kept as a fallback (its transcript temporarily unreadable) would
161+
otherwise decode as a freshly parsed, `declared` entry after the next save and be
162+
served warm forever. `lq` is cleared only when a successful full parse replaces the
163+
entry; a current parser's genuinely partial measurement carries its completeness
164+
field and stays `declared`, so it does not re-parse on every scan.
158165
- Only `v1`/`v2` (no parse position, different fork semantics) are rejected.
159166

160167
Warm-cache acceptance therefore requires both `identity: "declared"` (native ids and

0 commit comments

Comments
 (0)