Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apps/server/src/mcp/toolkits/sim/handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,8 @@ export const layer = McpToolAccess.toLayer(
policy: params.policy,
harness: params.harness,
page: params.page,
days: params.days,
maxTokens: params.maxTokens,
})
.pipe(Effect.mapError(simFailure));
return definedOnly({
Expand Down
6 changes: 5 additions & 1 deletion apps/server/src/mcp/toolkits/sim/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,16 +53,19 @@ const attemptFields = {
const SimReadTool = Tool.make("sim_read", {
...shared,
description:
"Read the SIM record of harness changes (AGENTS.md, CLAUDE.md, notes, hooks). Actions: index, show (ids, brief), brief_all, find (query), by_file (file), check, review (policy, harness), audit (since|days, limit, includeClaudeLogs), inspect_thread (threadId). Reading records server-side read state that harness edits and SIM writes require.",
"Read the SIM record of harness changes (AGENTS.md, CLAUDE.md, notes, hooks). Actions: index, show (ids, brief), brief_all, brief (ids|file|query, maxTokens), find (query), by_file (file), check, review (policy, harness), metrics (days), roi (days), audit (since|days, limit, includeClaudeLogs), inspect_thread (threadId). Reading records server-side read state that harness edits and SIM writes require.",
parameters: Schema.Struct({
action: Schema.Literals([
"index",
"show",
"brief_all",
"brief",
"find",
"by_file",
"check",
"review",
"metrics",
"roi",
"audit",
"inspect_thread",
]),
Expand All @@ -75,6 +78,7 @@ const SimReadTool = Tool.make("sim_read", {
page: Schema.optionalKey(Schema.Int),
since: Schema.optionalKey(Schema.String),
days: Schema.optionalKey(Schema.Number),
maxTokens: Schema.optionalKey(Schema.Int),
limit: Schema.optionalKey(Schema.Int),
includeClaudeLogs: Schema.optionalKey(Schema.Boolean),
threadId: Schema.optionalKey(Schema.String),
Expand Down
1 change: 1 addition & 0 deletions apps/server/src/serverSettings.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1972,6 +1972,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
"cloudSpeechCredentials.openrouter",
"easDeviceHosts.[].accessToken",
"simlockDeviceHosts.[].agentToken",
"sim.jev.apiKey",
"usageLimitSources.*.managementKey",
"textGenerators.*.apiKey",
]);
Expand Down
10 changes: 10 additions & 0 deletions apps/server/src/settingsSecrets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
type ProviderInstanceConfig,
type ServerSettings,
type SettingsSecretValue,
SimJevSettings,
SimlockDeviceHostConfig,
TextGeneratorConfig,
UsageLimitSourceConfig,
Expand Down Expand Up @@ -51,6 +52,7 @@ const textGeneratorSecretFields = secretSettingsFields(TextGeneratorConfig);
const voiceSecretFields = secretSettingsFields(CloudSpeechCredentials);
const easDeviceHostSecretFields = secretSettingsFields(EasDeviceHostConfig);
const simlockDeviceHostSecretFields = secretSettingsFields(SimlockDeviceHostConfig);
const simJevSecretFields = secretSettingsFields(SimJevSettings);
const encode = (value: string) => Buffer.from(value, "utf8").toString("base64url");

export interface SettingsSecret {
Expand Down Expand Up @@ -254,6 +256,14 @@ function mapSecretFieldGroups(
voiceSecretFields,
(provider) => `voice-cloud-${provider}`,
) as ServerSettings["cloudSpeechCredentials"],
sim: {
...settings.sim,
jev: mapFields(
settings.sim.jev,
simJevSecretFields,
(field) => `sim-jev-${encode(field)}`,
) as ServerSettings["sim"]["jev"],
},
providerInstances,
sourceControlHosts,
usageLimitSources,
Expand Down
62 changes: 61 additions & 1 deletion apps/server/src/sim/SimAuditService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@ import {
type SimAuditOptions,
type SimAuditReport,
type SimCallerContext,
type SimMetricEvent,
type SimResolvedPaths,
type SimState,
type SimTelemetryEvent,
} from "./SimContracts.ts";
import { summarizeMetricsMatrix, summarizeRuleRoi } from "./SimMetricsStore.ts";
import { detectShellWriteTargets } from "./SimObserverReactor.ts";
import * as SimStore from "./SimStore.ts";

Expand Down Expand Up @@ -105,6 +107,7 @@ function extractUserText(content: unknown): string {
.replace(/<assistant_citations>[\s\S]*?<\/assistant_citations>/g, "")
.replace(/<system_context>[\s\S]*?<\/system_context>/g, "")
.replace(/<runtime_info>[\s\S]*?<\/runtime_info>/g, "")
.replace(/<sim_turn_nudge>[\s\S]*?<\/sim_turn_nudge>/g, "")
.trim();
const userRequest = text.match(/<user_request>([\s\S]*?)<\/user_request>/);
if (userRequest) text = (userRequest[1] ?? "").trim();
Expand Down Expand Up @@ -172,6 +175,7 @@ interface HarnessEdit {
readonly files: ReadonlyArray<string>;
readonly record: boolean;
readonly recorded: boolean;
readonly verifiedByHash?: boolean | undefined;
}

interface FlaggedSession {
Expand Down Expand Up @@ -708,6 +712,7 @@ const make = Effect.gen(function* () {
const telemetry = (yield* sim.readTelemetry(sinceMs)).filter(
(event) => !caller.threadId || !("threadId" in event) || event.threadId !== caller.threadId,
);
const metrics = typeof sim.readMetrics === "function" ? yield* sim.readMetrics(sinceMs) : [];
const observed = yield* sim.getObservedModels();
return yield* Effect.try({
try: () =>
Expand All @@ -722,6 +727,8 @@ const make = Effect.gen(function* () {
limit,
db,
telemetry,
metrics,
priceOverrides: settings.usagePriceOverrides,
observed,
advisor: settings.sim,
}),
Expand Down Expand Up @@ -841,6 +848,10 @@ interface ReportInput {
readonly forkParents: ReadonlyMap<string, string | null>;
};
readonly telemetry: ReadonlyArray<SimTelemetryEvent>;
readonly metrics: ReadonlyArray<SimMetricEvent>;
readonly priceOverrides?:
| Readonly<Record<string, import("@t3tools/contracts").UsageModelPriceOverride>>
| undefined;
readonly observed: ReadonlySet<string>;
readonly advisor: Parameters<typeof SimStore.buildAdvisorEscalation>[1];
}
Expand Down Expand Up @@ -894,6 +905,8 @@ function buildReport(input: ReportInput): SimAuditReport {
const interrupts = new Map<string, Interrupt>();
const reviews = new Map<string, Review>();
const freshModels = new Map<string, { providerInstanceId: string; model: string }>();
const jevCorrections: Array<Extract<SimTelemetryEvent, { kind: "jev_correction_signal" }>> = [];
const turnEndings: Array<Extract<SimTelemetryEvent, { kind: "turn_ending_classified" }>> = [];
let threadsScanned = 0;
let excludedSelf = 0;

Expand Down Expand Up @@ -1061,6 +1074,8 @@ function buildReport(input: ReportInput): SimAuditReport {
case "unrecorded_harness_edit":
case "direct_sim_record_edit": {
const isRecord = event.kind === "direct_sim_record_edit";
const verifiedByHash =
event.kind === "unrecorded_harness_edit" ? event.verifiedByHash : undefined;
const known = edits.some(
(edit) =>
edit.session === event.threadId &&
Expand All @@ -1077,6 +1092,7 @@ function buildReport(input: ReportInput): SimAuditReport {
files: event.files,
record: isRecord,
recorded: false,
...(verifiedByHash ? { verifiedByHash: true } : {}),
});
}
break;
Expand Down Expand Up @@ -1123,6 +1139,14 @@ function buildReport(input: ReportInput): SimAuditReport {
}
break;
}
case "jev_correction_signal":
if (event.flagged) jevCorrections.push(event);
break;
case "turn_ending_classified":
if (event.label !== "finished") turnEndings.push(event);
break;
case "detector_evaluated":
break;
}
}

Expand Down Expand Up @@ -1305,8 +1329,13 @@ function buildReport(input: ReportInput): SimAuditReport {
out.push(`== 3. Harness Edits (${harnessEdits.length}) ==`);
if (!harnessEdits.length) out.push(" none");
for (const edit of harnessEdits) {
const tag = edit.recorded
? "recorded"
: edit.verifiedByHash
? "UNRECORDED [hash-verified]"
: "UNRECORDED?";
out.push(
` ${edit.date} ${edit.source} ${edit.session} [${edit.provider}/${edit.model}] ${edit.recorded ? "recorded" : "UNRECORDED?"}: ${edit.files.join(", ")}`,
` ${edit.date} ${edit.source} ${edit.session} [${edit.provider}/${edit.model}] ${tag}: ${edit.files.join(", ")}`,
);
}
for (const edit of recordEdits) {
Expand Down Expand Up @@ -1366,6 +1395,37 @@ function buildReport(input: ReportInput): SimAuditReport {
out.push(` ${candidate.kind}: ${candidate.summary}`);
out.push(` ${advisorEscalation.guidance}`);
}
const metricsMatrix = summarizeMetricsMatrix(input.metrics);
const roiSummary = summarizeRuleRoi(input.metrics, { priceOverrides: input.priceOverrides });
if (metricsMatrix.length > 0 || roiSummary.rows.length > 0) {
out.push("");
out.push("== 9. Opportunity Matrix & Rule Tax (Token-ROI) ==");
out.push(
`Opportunities: ${roiSummary.totalOpportunities} | Violations: ${roiSummary.totalViolations} | Injected: ${roiSummary.totalTokensInjected.toLocaleString("en-US")} tokens (~$${roiSummary.totalEstimatedCostUsd.toFixed(4)})`,
);
for (const row of roiSummary.rows.slice(0, 12)) {
const rate = row.complianceRate === null ? "n/a" : `${Math.round(row.complianceRate * 100)}%`;
out.push(
` ${row.ruleKey} (${row.problemId ?? "-"}) opps=${row.opportunities} pass=${row.passed} viol=${row.violated} rate=${rate} tokens=${row.tokensInjected} [${row.recommendation}]`,
);
}
}
if (jevCorrections.length > 0 || turnEndings.length > 0) {
out.push("");
out.push(
`== 10. Jev Semantic Signals (corrections=${jevCorrections.length}, non-finished endings=${turnEndings.length}) ==`,
);
for (const signal of jevCorrections.slice(0, limit)) {
out.push(
` JEV-CORRECTION ${signal.ts.slice(0, 10)} ${signal.threadId} [${signal.providerInstanceId ?? "?"}/${signal.model ?? "?"}]: "${squashText(signal.excerpt ?? "", 180)}"`,
);
}
for (const ending of turnEndings.slice(0, limit)) {
out.push(
` TURN-END ${ending.ts.slice(0, 10)} ${ending.threadId} [${ending.providerInstanceId ?? "?"}/${ending.model ?? "?"}] ${ending.label} (wait=${ending.waitingForUser.toFixed(2)}, manual=${ending.manualHandoff.toFixed(2)}, unverified=${ending.unverifiedDoneClaim.toFixed(2)})`,
);
}
}

return {
text: out.join("\n"),
Expand Down
97 changes: 96 additions & 1 deletion apps/server/src/sim/SimContracts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,8 @@ export interface SimResolvedPaths {
readonly seenDir: string;
readonly epochsPath: string;
readonly telemetryPath: string;
readonly metricsPath: string;
readonly jevUsagePath: string;
readonly auditStatePath: string;
}

Expand Down Expand Up @@ -264,6 +266,8 @@ export function resolveSimPaths(
seenDir: NodePath.join(simDir, "seen"),
epochsPath: NodePath.join(simDir, "t3-epochs.json"),
telemetryPath: NodePath.join(simDir, "telemetry.jsonl"),
metricsPath: NodePath.join(simDir, "metrics.jsonl"),
jevUsagePath: NodePath.join(simDir, "jev-usage.json"),
auditStatePath: NodePath.join(harnessDir, "state", "sim-audit-state.json"),
};
}
Expand Down Expand Up @@ -342,6 +346,8 @@ const RECORD_FILES = new Set([
"sim.jsonl",
"sim.lock",
"telemetry.jsonl",
"metrics.jsonl",
"jev-usage.json",
"t3-epochs.json",
]);

Expand Down Expand Up @@ -464,6 +470,7 @@ export type SimTelemetryEvent =
readonly providerInstanceId?: string | undefined;
readonly model?: string | undefined;
readonly files: ReadonlyArray<string>;
readonly verifiedByHash?: boolean | undefined;
}
| {
readonly ts: string;
Expand Down Expand Up @@ -518,17 +525,103 @@ export type SimTelemetryEvent =
readonly model?: string | undefined;
readonly title?: string | undefined;
readonly excerpt?: string | undefined;
}
| {
readonly ts: string;
readonly kind: "detector_evaluated";
readonly threadId: string;
readonly runId?: string | undefined;
readonly providerInstanceId?: string | undefined;
readonly model?: string | undefined;
readonly project?: string | undefined;
readonly detectorId: string;
readonly problemId?: string | undefined;
readonly attemptId?: string | undefined;
readonly verdict: "passed" | "violated" | "uncertain" | "misfired";
readonly source: "deterministic" | "jev" | "replay";
readonly confidence?: number | undefined;
readonly evidence?: string | undefined;
}
| {
readonly ts: string;
readonly kind: "jev_correction_signal";
readonly threadId: string;
readonly runId?: string | undefined;
readonly providerInstanceId?: string | undefined;
readonly model?: string | undefined;
readonly project?: string | undefined;
readonly scores: Readonly<Record<string, number>>;
readonly flagged: boolean;
readonly excerpt?: string | undefined;
}
| {
readonly ts: string;
readonly kind: "turn_ending_classified";
readonly threadId: string;
readonly runId?: string | undefined;
readonly providerInstanceId?: string | undefined;
readonly model?: string | undefined;
readonly project?: string | undefined;
readonly waitingForUser: number;
readonly manualHandoff: number;
readonly unverifiedDoneClaim: number;
readonly label: "finished" | "waiting_for_user" | "manual_handoff" | "unverified_done";
};

export type SimMetricEvent =
| {
readonly ts: string;
readonly kind: "observation";
readonly detectorId: string;
readonly ruleKey: string;
readonly problemId?: string | undefined;
readonly attemptId?: string | undefined;
readonly verdict: "passed" | "violated" | "uncertain" | "misfired";
readonly source: "deterministic" | "jev" | "replay";
readonly confidence?: number | undefined;
readonly providerInstanceId: string;
readonly model: string;
readonly project?: string | undefined;
readonly threadId?: string | undefined;
readonly runId?: string | undefined;
readonly evidence?: string | undefined;
}
| {
readonly ts: string;
readonly kind: "injection";
readonly ruleKey: string;
readonly problemId?: string | undefined;
readonly attemptId?: string | undefined;
readonly delivery: "base" | "jit" | "candidate";
readonly bytes: number;
readonly estimatedTokens: number;
readonly providerInstanceId: string;
readonly model: string;
readonly project?: string | undefined;
readonly threadId?: string | undefined;
};

export interface SimReadRequest {
readonly action: "index" | "show" | "brief_all" | "find" | "by_file" | "check" | "review";
readonly action:
| "index"
| "show"
| "brief_all"
| "brief"
| "find"
| "by_file"
| "check"
| "review"
| "metrics"
| "roi";
readonly ids?: ReadonlyArray<string> | undefined;
readonly brief?: boolean | undefined;
readonly query?: string | undefined;
readonly file?: string | undefined;
readonly policy?: string | undefined;
readonly harness?: string | undefined;
readonly page?: number | undefined;
readonly days?: number | undefined;
readonly maxTokens?: number | undefined;
}

export interface SimReadResult {
Expand Down Expand Up @@ -707,6 +800,8 @@ export class SimService extends Context.Service<
readonly getThreadSeenEpoch: (threadId: ThreadId) => Effect.Effect<number>;
readonly recordTelemetry: (event: SimTelemetryEvent) => Effect.Effect<void>;
readonly readTelemetry: (sinceMs?: number) => Effect.Effect<ReadonlyArray<SimTelemetryEvent>>;
readonly recordMetrics: (events: ReadonlyArray<SimMetricEvent>) => Effect.Effect<void>;
readonly readMetrics: (sinceMs?: number) => Effect.Effect<ReadonlyArray<SimMetricEvent>>;
readonly recordRecentAttemptInTurn: (input: {
readonly threadId: ThreadId;
readonly runId?: RunId | undefined;
Expand Down
Loading