From 32ab07d9ecf7559be184a69bc3edb6e930174a60 Mon Sep 17 00:00:00 2001 From: Sanjay Date: Wed, 29 Jul 2026 03:28:28 -0400 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20Plan=206=20PR=20B=20=E2=80=94=20sco?= =?UTF-8?q?re=20recompute=20scheduler=20+=20live=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add single-flight RecomputeScheduler with dirty latch and deterministic shutdown drain, hook markDirty after attestation ingest, and require trust-engine in control-plane integration CI for score-recompute tests. Co-authored-by: Cursor --- .github/workflows/ci.yml | 35 +++ .../integration/score-recompute.test.ts | 221 ++++++++++++++++++ control-plane/src/db/client.ts | 27 ++- control-plane/src/db/transaction.ts | 1 + .../domains/attestation/attestationService.ts | 6 +- .../domains/graph/recomputeScheduler.test.ts | 120 ++++++++++ .../src/domains/graph/recomputeScheduler.ts | 177 ++++++++++++++ control-plane/src/index.ts | 16 ++ control-plane/src/testutil/appHarness.ts | 11 +- control-plane/src/testutil/seedData.ts | 39 +++- control-plane/src/testutil/testDb.ts | 4 + docs/superpowers/plans/HANDOVER.md | 19 +- 12 files changed, 650 insertions(+), 26 deletions(-) create mode 100644 control-plane/src/__tests__/integration/score-recompute.test.ts create mode 100644 control-plane/src/domains/graph/recomputeScheduler.test.ts create mode 100644 control-plane/src/domains/graph/recomputeScheduler.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 115e513..cca535e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,6 +144,12 @@ jobs: with: persist-credentials: false + - name: Set up Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff + with: + go-version-file: go.mod + cache: true + - name: Set up Node uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 with: @@ -151,6 +157,25 @@ jobs: cache: npm cache-dependency-path: control-plane/package-lock.json + - name: Build trust-engine + working-directory: . + run: go build -o /tmp/verilink-trust-engine ./cmd/trust-engine + + - name: Start trust-engine + working-directory: . + run: | + /tmp/verilink-trust-engine -grpc-port 9091 -http-port 8086 & + echo $! > /tmp/verilink-trust-engine.pid + for i in $(seq 1 30); do + if curl -sf http://127.0.0.1:8086/healthz >/dev/null; then + echo "trust-engine healthy" + exit 0 + fi + sleep 0.5 + done + echo "trust-engine failed to become healthy" >&2 + exit 1 + - name: Install dependencies run: npm ci @@ -158,4 +183,14 @@ jobs: env: DATABASE_URL: postgresql://verilink:verilink@127.0.0.1:5432/verilink_test API_KEY_HMAC_SECRET: test-hmac-secret-for-integration + TRUST_ENGINE_ADDR: 127.0.0.1:9091 + CI: "true" run: npm run test:integration + + - name: Stop trust-engine + if: always() + working-directory: . + run: | + if [ -f /tmp/verilink-trust-engine.pid ]; then + kill "$(cat /tmp/verilink-trust-engine.pid)" || true + fi diff --git a/control-plane/src/__tests__/integration/score-recompute.test.ts b/control-plane/src/__tests__/integration/score-recompute.test.ts new file mode 100644 index 0000000..f3e33ee --- /dev/null +++ b/control-plane/src/__tests__/integration/score-recompute.test.ts @@ -0,0 +1,221 @@ +/** + * Plan 6 PR B — live RunVeriRank score recompute (mandatory in CI). + * + * Requires TRUST_ENGINE_ADDR. Locally you may leave it unset to skip; + * CI / GITHUB_ACTIONS must set it (workflow starts trust-engine). + */ +process.env.DATABASE_URL ||= + 'postgresql://verilink:verilink@127.0.0.1:15432/verilink_test'; +process.env.API_KEY_HMAC_SECRET ||= 'test-hmac-secret-for-integration'; + +import { describe, it, before, after, beforeEach } from 'node:test'; +import assert from 'node:assert/strict'; +import type pg from 'pg'; +import { setupTestDb, teardownTestDb, resetTestData } from '../../testutil/testDb.js'; +import { + seedTenant, + seedIssuer, + seedSubject, + seedBootstrapIssuer, + seedApiKey, + authHeaders, + signAttestationToken, +} from '../../testutil/seedData.js'; +import { startControlPlane, type ControlPlaneHarness } from '../../testutil/appHarness.js'; +import { recomputeNow } from '../../domains/graph/scoreComputationService.js'; +import { + RecomputeScheduler, + setRecomputeSchedulerForTests, +} from '../../domains/graph/recomputeScheduler.js'; + +const trustEngineAddr = process.env.TRUST_ENGINE_ADDR; +const inCi = process.env.CI === 'true' || process.env.GITHUB_ACTIONS === 'true'; +const skipLive = !trustEngineAddr && !inCi; + +if (!trustEngineAddr && inCi) { + throw new Error( + 'TRUST_ENGINE_ADDR is required for score-recompute integration in CI (start cmd/trust-engine)' + ); +} + +function delay(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +describe('Score recompute (live RunVeriRank)', { skip: skipLive }, () => { + let pool: pg.Pool; + let harness: ControlPlaneHarness; + let tenantId: string; + let apiKey: string; + + before(async () => { + pool = await setupTestDb(); + harness = await startControlPlane(); + }); + + after(async () => { + setRecomputeSchedulerForTests(null); + await harness.stop(); + await teardownTestDb(pool); + }); + + beforeEach(async () => { + setRecomputeSchedulerForTests(null); + await resetTestData(pool); + const tenant = await seedTenant(pool, `tenant-score-${Date.now()}`); + tenantId = tenant.id; + apiKey = await seedApiKey(pool, tenantId, ['attest:write', 'attest:read']); + }); + + async function seedGraph() { + const issuer = await seedIssuer(pool, tenantId); + await seedBootstrapIssuer(pool, issuer.id); + const subject = await seedSubject(pool, tenantId); + return { issuer, subject }; + } + + async function submitAttestation(opts: { + issuer: Awaited>; + subjectId: string; + }): Promise<{ id: string }> { + const token = await signAttestationToken({ + issuerId: opts.issuer.id, + subjectId: opts.subjectId, + privateKey: opts.issuer.privateKey, + keyId: opts.issuer.keyId, + }); + const resp = await fetch(`${harness.url}/v1/attestations/submit`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...authHeaders(apiKey), + }, + body: JSON.stringify({ token }), + }); + const bodyText = await resp.text(); + assert.equal(resp.status, 201, bodyText); + const body = JSON.parse(bodyText) as { ok: boolean; data: { id: string } }; + return body.data; + } + + it('seeds bootstrap + attestation → scores + score.upsert', async () => { + const { issuer, subject } = await seedGraph(); + await submitAttestation({ issuer, subjectId: subject.id }); + + const result = await recomputeNow(new Date(), { trustEngineAddr }); + assert.equal(result.status, 'applied'); + assert.ok(result.upserts >= 1); + + const { rows: scores } = await pool.query( + 'SELECT principal_id, score FROM network_scores ORDER BY principal_id' + ); + assert.ok(scores.length >= 1); + + const { rows: events } = await pool.query( + `SELECT event_type FROM sync_events WHERE event_type = 'score.upsert'` + ); + assert.ok(events.length >= 1); + }); + + it('excludes expired attestations from scoring', async () => { + const { issuer, subject } = await seedGraph(); + const submitted = await submitAttestation({ issuer, subjectId: subject.id }); + await pool.query( + `UPDATE attestations SET expires_at = NOW() - INTERVAL '1 hour' WHERE id = $1`, + [submitted.id] + ); + + await recomputeNow(new Date(), { trustEngineAddr }); + const { rows } = await pool.query( + 'SELECT principal_id FROM network_scores WHERE principal_id = $1', + [subject.id] + ); + assert.equal(rows.length, 0, 'expired attestation must not score the subject'); + }); + + it('deactivated principal does not regain a score', async () => { + const { issuer, subject } = await seedGraph(); + await submitAttestation({ issuer, subjectId: subject.id }); + let result = await recomputeNow(new Date(), { trustEngineAddr }); + assert.equal(result.status, 'applied'); + + await pool.query(`UPDATE principals SET status = 'deactivated' WHERE id = $1`, [ + subject.id, + ]); + result = await recomputeNow(new Date(), { trustEngineAddr }); + assert.equal(result.status, 'applied'); + + const { rows } = await pool.query( + 'SELECT principal_id FROM network_scores WHERE principal_id = $1', + [subject.id] + ); + assert.equal(rows.length, 0); + }); + + it('removing last bootstrap root clears existing scores + score.delete', async () => { + const { issuer, subject } = await seedGraph(); + await submitAttestation({ issuer, subjectId: subject.id }); + const first = await recomputeNow(new Date(), { trustEngineAddr }); + assert.equal(first.status, 'applied'); + assert.ok(first.upserts >= 1); + + await pool.query('DELETE FROM bootstrap_issuers'); + const cleared = await recomputeNow(new Date(), { trustEngineAddr }); + assert.equal(cleared.status, 'cleared_stale'); + assert.ok(cleared.deletes >= 1); + + const { rows: scores } = await pool.query('SELECT principal_id FROM network_scores'); + assert.equal(scores.length, 0); + + const { rows: deletes } = await pool.query( + `SELECT event_type FROM sync_events WHERE event_type = 'score.delete'` + ); + assert.ok(deletes.length >= 1); + }); + + it('second submit converges via dirty latch (single-flight)', async () => { + const { issuer, subject } = await seedGraph(); + const subject2 = await seedSubject(pool, tenantId); + const calls: number[] = []; + + let resolveGate!: () => void; + const gate = new Promise((r) => { + resolveGate = r; + }); + let passedGate = false; + + const scheduler = new RecomputeScheduler({ + debounceMs: 15, + intervalMs: 60_000, + recompute: async (t) => { + if (!passedGate) { + await gate; + passedGate = true; + } + calls.push(Date.now()); + return recomputeNow(t, { trustEngineAddr }); + }, + }); + scheduler.start(); + setRecomputeSchedulerForTests(scheduler); + + const runP = scheduler.kick(); + await delay(10); + await submitAttestation({ issuer, subjectId: subject.id }); + await submitAttestation({ issuer, subjectId: subject2.id }); + scheduler.markDirty(); + resolveGate(); + await runP; + await delay(80); + await scheduler.stop(); + setRecomputeSchedulerForTests(null); + + assert.ok( + calls.length >= 2 && calls.length <= 3, + `expected 2–3 coalesced runs, got ${calls.length}` + ); + + const { rows } = await pool.query('SELECT COUNT(*)::int AS n FROM network_scores'); + assert.ok((rows[0].n as number) >= 1); + }); +}); diff --git a/control-plane/src/db/client.ts b/control-plane/src/db/client.ts index 4cb501b..fef55ce 100644 --- a/control-plane/src/db/client.ts +++ b/control-plane/src/db/client.ts @@ -3,13 +3,22 @@ import pg from 'pg'; import { config } from '../config.js'; import { logger } from '../shared/logger.js'; -export const pool = new pg.Pool({ - connectionString: config.database.url, - max: config.database.poolMax, - idleTimeoutMillis: config.database.poolIdleTimeoutMillis, - connectionTimeoutMillis: config.database.poolConnectionTimeoutMillis, -}); +function createPool(): pg.Pool { + const p = new pg.Pool({ + connectionString: config.database.url, + max: config.database.poolMax, + idleTimeoutMillis: config.database.poolIdleTimeoutMillis, + connectionTimeoutMillis: config.database.poolConnectionTimeoutMillis, + }); + p.on('error', (err) => { + logger.error({ err }, 'Unexpected idle client error'); + }); + return p; +} -pool.on('error', (err) => { - logger.error({ err }, 'Unexpected idle client error'); -}); +export let pool = createPool(); + +/** Recreate the singleton after `pool.end()` (multi-suite integration harness). */ +export function recreatePool(): void { + pool = createPool(); +} \ No newline at end of file diff --git a/control-plane/src/db/transaction.ts b/control-plane/src/db/transaction.ts index 54671ed..4ad0b34 100644 --- a/control-plane/src/db/transaction.ts +++ b/control-plane/src/db/transaction.ts @@ -7,6 +7,7 @@ export { pool }; /** * Execute work inside a transaction. The client is automatically released * (even if work throws). Commit happens only if work returns without error. + * Uses the live `pool` export so recreatePool() is visible after suite teardown. */ export async function withTransaction( work: (client: pg.PoolClient) => Promise diff --git a/control-plane/src/domains/attestation/attestationService.ts b/control-plane/src/domains/attestation/attestationService.ts index dbaceec..9a6e3b3 100644 --- a/control-plane/src/domains/attestation/attestationService.ts +++ b/control-plane/src/domains/attestation/attestationService.ts @@ -9,6 +9,7 @@ import { isV0AllowedForIssuer } from './legacyConfig.js'; import { verifyAttestation, type KeyCandidate } from '../../grpc/trustEngineClient.js'; import { AppError, CODES } from '../../shared/errors/AppError.js'; import { withTransaction } from '../../db/transaction.js'; +import { getRecomputeScheduler } from '../graph/recomputeScheduler.js'; function sha256hex(input: string): string { return createHash('sha256').update(input).digest('hex'); @@ -183,7 +184,7 @@ export async function submitAttestation(opts: { // 10. Store attestation transactionally with concurrency-safe dedup + observation_id pairing try { - return await withTransaction(async (client) => { + const row = await withTransaction(async (client) => { // Dedup check inside transaction const existing = await attestationRepo.findByTokenDigest(tokenDigest, client); if (existing) { @@ -230,6 +231,9 @@ export async function submitAttestation(opts: { verifiedKeyId: verifyResult.verifiedKeyId!, }, client); }); + // Plan 6: enqueue score recompute after durable ingest (outside TX). + getRecomputeScheduler().markDirty(); + return row; } catch (err: any) { if (err instanceof AppError) throw err; if (err.code === '23505') { diff --git a/control-plane/src/domains/graph/recomputeScheduler.test.ts b/control-plane/src/domains/graph/recomputeScheduler.test.ts new file mode 100644 index 0000000..c7b058c --- /dev/null +++ b/control-plane/src/domains/graph/recomputeScheduler.test.ts @@ -0,0 +1,120 @@ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { RecomputeScheduler } from './recomputeScheduler.js'; + +function delay(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +describe('RecomputeScheduler', () => { + let calls: Date[]; + let gate: { release: () => void; wait: Promise } | null; + let scheduler: RecomputeScheduler; + + beforeEach(() => { + calls = []; + gate = null; + scheduler = new RecomputeScheduler({ + debounceMs: 30, + intervalMs: 60_000, + recompute: async (t) => { + calls.push(t); + if (gate) await gate.wait; + }, + }); + scheduler.start(); + }); + + afterEach(async () => { + if (gate) gate.release(); + await scheduler.stop(); + }); + + it('coalesces concurrent markDirty while running into one follow-up run', async () => { + let resolveGate!: () => void; + gate = { + wait: new Promise((r) => { + resolveGate = r; + }), + release: () => resolveGate(), + }; + + const first = scheduler.kick(); + await delay(5); + assert.equal(scheduler.isRunning, true); + scheduler.markDirty(); + scheduler.markDirty(); + scheduler.markDirty(); + resolveGate(); + await first; + // Allow follow-up loop to finish + await delay(20); + assert.equal(calls.length, 2); + }); + + it('stop finishes active run, drains one dirty, ignores further dirty', async () => { + await scheduler.stop(); + const runOrder: string[] = []; + let resolveFirst!: () => void; + const firstWait = new Promise((r) => { + resolveFirst = r; + }); + let drainStarted!: () => void; + const drainSeen = new Promise((r) => { + drainStarted = r; + }); + let n = 0; + + scheduler = new RecomputeScheduler({ + debounceMs: 30, + intervalMs: 60_000, + recompute: async () => { + n += 1; + runOrder.push(`run-${n}`); + if (n === 1) { + await firstWait; + } else if (n === 2) { + drainStarted(); + // During drain, further triggers set dirty but must not cause a third run. + void scheduler.kick(); + await delay(5); + } + }, + }); + scheduler.start(); + + const kickP = scheduler.kick(); + await delay(5); + scheduler.markDirty(); + const stopP = scheduler.stop(); + resolveFirst(); + await Promise.all([kickP, stopP, drainSeen]); + await delay(20); + + assert.deepEqual(runOrder, ['run-1', 'run-2']); + assert.equal(scheduler.isDirty, false); + assert.equal(scheduler.isRunning, false); + }); + + it('markDirty is no-op while stopping', async () => { + await scheduler.stop(); + scheduler.markDirty(); + await delay(50); + assert.equal(calls.length, 0); + }); + + it('markDirty is no-op before start', async () => { + await scheduler.stop(); + const idle = new RecomputeScheduler({ + debounceMs: 10, + intervalMs: 60_000, + recompute: async (t) => { + calls.push(t); + }, + }); + idle.markDirty(); + await delay(40); + assert.equal(calls.length, 0); + await idle.stop(); + }); +}); diff --git a/control-plane/src/domains/graph/recomputeScheduler.ts b/control-plane/src/domains/graph/recomputeScheduler.ts new file mode 100644 index 0000000..966ef24 --- /dev/null +++ b/control-plane/src/domains/graph/recomputeScheduler.ts @@ -0,0 +1,177 @@ +// control-plane/src/domains/graph/recomputeScheduler.ts +import { logger } from '../../shared/logger.js'; +import { config } from '../../config.js'; +import { recomputeNow } from './scoreComputationService.js'; + +export type RecomputeFn = (evaluationTime: Date) => Promise; + +export interface RecomputeSchedulerOptions { + debounceMs?: number; + intervalMs?: number; + recompute?: RecomputeFn; +} + +/** + * In-process single-flight score recompute scheduler (Plan 6). + * Debounced ingest dirty + hourly tick; stop drains ≤1 dirty rerun. + */ +export class RecomputeScheduler { + private running = false; + private dirty = false; + private stopping = false; + private started = false; + private debounceTimer: ReturnType | null = null; + private intervalTimer: ReturnType | null = null; + private activeRun: Promise | null = null; + + private readonly debounceMs: number; + private readonly intervalMs: number; + private readonly recompute: RecomputeFn; + + constructor(opts: RecomputeSchedulerOptions = {}) { + this.debounceMs = opts.debounceMs ?? config.scoreRecompute.debounceMs; + this.intervalMs = opts.intervalMs ?? config.scoreRecompute.intervalMs; + this.recompute = opts.recompute ?? ((t) => recomputeNow(t)); + } + + start(): void { + this.stopping = false; + this.started = true; + if (this.intervalTimer) return; + this.intervalTimer = setInterval(() => { + if (this.stopping) return; + void this.kick(); + }, this.intervalMs); + // Allow Node test runners to exit if a suite forgets stop(). + this.intervalTimer.unref?.(); + } + + /** After successful attestation ingest — coalesce via debounce when idle. */ + markDirty(): void { + // No-op until start() so integration suites that only exercise ingest + // do not schedule background recomputes against a test pool. + if (this.stopping || !this.started) return; + if (this.running) { + this.dirty = true; + return; + } + this.scheduleDebounce(); + } + + private scheduleDebounce(): void { + if (this.stopping || this.debounceTimer) return; + this.debounceTimer = setTimeout(() => { + this.debounceTimer = null; + if (this.stopping) return; + void this.kick(); + }, this.debounceMs); + } + + /** Entry used by hourly tick and debounce. */ + kick(): Promise { + if (this.stopping && !this.running) { + return Promise.resolve(); + } + if (this.running) { + this.dirty = true; + return this.activeRun ?? Promise.resolve(); + } + this.running = true; + this.activeRun = this.runLoop().finally(() => { + this.running = false; + this.activeRun = null; + }); + return this.activeRun; + } + + private async runLoop(): Promise { + // Normal loop: repeat while dirty and not stopping. + for (;;) { + this.dirty = false; + const evaluationTime = new Date(); + try { + await this.recompute(evaluationTime); + } catch (err) { + logger.error({ err }, 'score recompute failed'); + } + + if (this.dirty && !this.stopping) { + continue; + } + break; + } + + // Shutdown drain: at most one already-dirty rerun. + if (this.stopping && this.dirty) { + this.dirty = false; + try { + await this.recompute(new Date()); + } catch (err) { + logger.error({ err }, 'score recompute failed (shutdown drain)'); + } + // Discard dirty set during the drain rerun. + this.dirty = false; + } + } + + /** + * stop accepting triggers → cancel timers → await active → drain ≤1 dirty → exit. + */ + async stop(): Promise { + this.stopping = true; + this.started = false; + if (this.debounceTimer) { + clearTimeout(this.debounceTimer); + this.debounceTimer = null; + } + if (this.intervalTimer) { + clearInterval(this.intervalTimer); + this.intervalTimer = null; + } + + if (this.activeRun) { + await this.activeRun; + return; + } + + // Idle but dirty (e.g. debounce cancelled before fire): drain one. + if (this.dirty) { + this.running = true; + this.dirty = false; + try { + await this.recompute(new Date()); + } catch (err) { + logger.error({ err }, 'score recompute failed (shutdown idle drain)'); + } finally { + this.dirty = false; + this.running = false; + } + } + } + + /** Test helpers */ + get isRunning(): boolean { + return this.running; + } + + get isDirty(): boolean { + return this.dirty; + } + + get isStopping(): boolean { + return this.stopping; + } +} + +let defaultScheduler: RecomputeScheduler | null = null; + +export function getRecomputeScheduler(): RecomputeScheduler { + if (!defaultScheduler) { + defaultScheduler = new RecomputeScheduler(); + } + return defaultScheduler; +} + +export function setRecomputeSchedulerForTests(scheduler: RecomputeScheduler | null): void { + defaultScheduler = scheduler; +} diff --git a/control-plane/src/index.ts b/control-plane/src/index.ts index d98eebb..54ff9c6 100644 --- a/control-plane/src/index.ts +++ b/control-plane/src/index.ts @@ -3,6 +3,7 @@ import { pool } from './db/client.js'; import { runMigrations } from './db/migrate.js'; import { createApp } from './app.js'; import { logger } from './shared/logger.js'; +import { getRecomputeScheduler } from './domains/graph/recomputeScheduler.js'; async function main() { // Validate config @@ -13,6 +14,16 @@ async function main() { logger.info('Running migrations...'); await runMigrations(); + const scheduler = getRecomputeScheduler(); + scheduler.start(); + logger.info( + { + debounceMs: config.scoreRecompute.debounceMs, + intervalMs: config.scoreRecompute.intervalMs, + }, + 'Score recompute scheduler started' + ); + // Create and start server const app = createApp(); const server = app.listen(config.server.port, () => { @@ -25,6 +36,11 @@ async function main() { if (shuttingDown) return; shuttingDown = true; logger.info('Shutting down...'); + try { + await scheduler.stop(); + } catch (err) { + logger.error({ err }, 'score recompute scheduler stop failed'); + } server.close(async () => { await pool.end(); logger.info('Bye.'); diff --git a/control-plane/src/testutil/appHarness.ts b/control-plane/src/testutil/appHarness.ts index 8f00760..19ad743 100644 --- a/control-plane/src/testutil/appHarness.ts +++ b/control-plane/src/testutil/appHarness.ts @@ -16,6 +16,11 @@ export async function startControlPlane( ): Promise { // Dynamic import so config freezes after env is configured by the caller. const { createApp } = await import('../app.js'); + const { pool, recreatePool } = await import('../db/client.js'); + // Prior suite may have ended the singleton pool in stop(). + if (pool.ended) { + recreatePool(); + } const app: Express = createApp(); const server: Server = createServer(app); @@ -34,8 +39,10 @@ export async function startControlPlane( server.close((err) => (err ? reject(err) : resolve())); }); // End the singleton app pool so node:test can exit cleanly. - const { pool } = await import('../db/client.js'); - await pool.end(); + const { pool: livePool } = await import('../db/client.js'); + if (!livePool.ended) { + await livePool.end(); + } }, }; } diff --git a/control-plane/src/testutil/seedData.ts b/control-plane/src/testutil/seedData.ts index 31125ca..72519f4 100644 --- a/control-plane/src/testutil/seedData.ts +++ b/control-plane/src/testutil/seedData.ts @@ -58,6 +58,26 @@ export async function seedSubject(pool: pg.Pool, tenantId: string): Promise<{ id return { id }; } +/** Mark an existing issuer as a bootstrap root for score recompute tests. */ +export async function seedBootstrapIssuer( + pool: pg.Pool, + principalId: string, + opts: { name?: string; weight?: number } = {} +): Promise { + const weight = opts.weight ?? 1.0; + await pool.query(`UPDATE issuers SET is_bootstrap = true WHERE principal_id = $1`, [ + principalId, + ]); + await pool.query( + `INSERT INTO bootstrap_issuers (principal_id, name, current_weight) + VALUES ($1, $2, $3) + ON CONFLICT (principal_id) DO UPDATE SET + name = EXCLUDED.name, + current_weight = EXCLUDED.current_weight`, + [principalId, opts.name ?? 'test-bootstrap', weight] + ); +} + /** Creates a vrl_ API key (68 chars) with HMAC hash matching auth middleware. */ export async function seedApiKey( pool: pg.Pool, @@ -90,8 +110,13 @@ export async function signAttestationToken(opts: { trustLevelDelta?: number; attestationType?: string; facts?: Record; + /** Unix seconds; defaults to now. */ + issuedAtUnix?: number; + /** Unix seconds; defaults to issuedAt+3600. Pass 0 to omit JWT exp (DB may still set expires). */ + expiresAtUnix?: number; }): Promise { const now = Math.floor(Date.now() / 1000); + const issuedAt = opts.issuedAtUnix ?? now; const facts = opts.facts ?? { start: new Date(Date.now() - 3600_000).toISOString(), end: new Date().toISOString(), @@ -100,7 +125,7 @@ export async function signAttestationToken(opts: { dispute_count: 0, }; - return new SignJWT({ + let builder = new SignJWT({ vli: { type: opts.attestationType ?? 'transaction_summary', facts, @@ -112,8 +137,12 @@ export async function signAttestationToken(opts: { .setProtectedHeader({ alg: 'EdDSA', kid: opts.keyId }) .setIssuer(opts.issuerId) .setSubject(opts.subjectId) - .setIssuedAt(now) - .setExpirationTime(now + 3600) - .setJti(crypto.randomUUID()) - .sign(opts.privateKey); + .setIssuedAt(issuedAt) + .setJti(crypto.randomUUID()); + + if (opts.expiresAtUnix !== 0) { + builder = builder.setExpirationTime(opts.expiresAtUnix ?? issuedAt + 3600); + } + + return builder.sign(opts.privateKey); } diff --git a/control-plane/src/testutil/testDb.ts b/control-plane/src/testutil/testDb.ts index 60ba7ca..2fa557a 100644 --- a/control-plane/src/testutil/testDb.ts +++ b/control-plane/src/testutil/testDb.ts @@ -134,6 +134,10 @@ export async function resetTestData(pool: pg.Pool): Promise { assertSafeVerilinkTestDb(url); await pool.query(` TRUNCATE + sync_events, + network_score_history, + network_scores, + bootstrap_issuers, attestations, principal_keys, issuers, diff --git a/docs/superpowers/plans/HANDOVER.md b/docs/superpowers/plans/HANDOVER.md index e3c40c8..b0204a2 100644 --- a/docs/superpowers/plans/HANDOVER.md +++ b/docs/superpowers/plans/HANDOVER.md @@ -1,9 +1,9 @@ # Handover Note — VeriLink Productization > **Updated:** 2026-07-29 -> **Repo HEAD:** `main` @ Plan 6 docs merged (PR #11) -> **Status:** Plans 1–5 + Plan 6 plan doc on `main`. Implementing Plan 6 **PR A** (loader/writer/gRPC/migrations) -> **Next session:** Finish/land Plan 6 PR A then PR B (scheduler + mandatory CI trust-engine) +> **Repo HEAD:** `main` @ Plan 6 PR A merged (PR #12) +> **Status:** Plan 6 PR A on `main`. Implementing Plan 6 **PR B** (scheduler + ingest hook + mandatory CI trust-engine) +> **Next session:** Land Plan 6 PR B → Plan 6 complete → Plan 7 (SSE / edge sync) --- @@ -15,7 +15,7 @@ VeriLink is past the “toolkit only” MVP. The monorepo now has: |---------|----------|--------| | Trust engine (gRPC) | `cmd/trust-engine`, `internal/trustengine` | `RunVeriRank`, `VerifyAttestation`, `GetFingerprint` | | Edge verifier | `cmd/edge-verifier`, `internal/edgeverifier` | RFC 9421 three-way outcomes + trust annotations | -| Control plane (TS) | `control-plane/` | Express + Postgres; attestation ingest E2E | +| Control plane (TS) | `control-plane/` | Express + Postgres; attestation ingest E2E; score writer (PR A) | | Clients | `client/go`, `client/node` | Signing helpers present | | Proto | `proto/verilink/trust/v1/trust.proto` → `pkg/trustpb` | Buf pipeline in CI | @@ -58,13 +58,13 @@ User judgment: **Plans 1–4 are covered well enough to move on.** Remaining wor --- -## Plan 6 — Network score computation — READY TO EXECUTE +## Plan 6 — Network score computation — IN PROGRESS - Plan doc: `docs/superpowers/plans/2026-07-28-network-score-computation.md` - Design: §4.5 + §13 step 10 -- Locked: single-flight debounce (60s) + hourly + dirty latch; deterministic shutdown (finish active + drain ≤1 dirty); live gRPC `RunVeriRank`; Verify stays in-process Node; scores + `sync_events` in one TX with `pg_advisory_xact_lock`; mandatory expiry + active-principal filters; shared `evaluationTime`; empty-roots clears existing scores (cold-start no-op only); weight columns + API `[0,1]`; `entity_kind` on history + change detection (`score`/`blacklisted`/`score_reason`/`entity_kind`); PR B mandatory live-engine CI -- Review closure matrix lives in the Plan 6 doc (human + CodeRabbit + Qodo) -- Suggested split: **PR A** migrations/loader/writer/gRPC client + units; **PR B** scheduler + ingest hook + mandatory CI trust-engine integration +- **PR A (merged #12):** migrations `009`–`012`, loader (expiry + active principals + shared `evaluationTime`), writer (`pg_advisory_xact_lock`, history `entity_kind`), empty-roots cold-start vs clear-stale, live gRPC `RunVeriRank` client + units +- **PR B (this branch):** single-flight `RecomputeScheduler` (deterministic stop), ingest `markDirty`, `index.ts` start/stop, mandatory CI trust-engine + `score-recompute` integration +- Locked: single-flight debounce (60s) + hourly + dirty latch; deterministic shutdown (finish active + drain ≤1 dirty); live gRPC `RunVeriRank`; Verify stays in-process Node; scores + `sync_events` in one TX with `pg_advisory_xact_lock`; mandatory expiry + active-principal filters; shared `evaluationTime`; empty-roots clears existing scores (cold-start no-op only); weight columns + API `[0,1]`; `entity_kind` on history + change detection; PR B mandatory live-engine CI ### What’s after Plan 6 (§13) @@ -87,7 +87,7 @@ Also useful: refresh stale remote branches (`origin/feat/engine-trust-engine`, ` | `docs/superpowers/plans/2026-07-27-verilink-2-control-plane-foundation.md` | Plan 2 (detailed) | | `docs/superpowers/plans/2026-07-27-verilink-3-request-auth-protocol.md` | Plan 3 | | `docs/superpowers/plans/2026-07-27-verilink-4-attestation-ingest.md` | Plan 4 | -| `docs/superpowers/plans/2026-07-28-network-score-computation.md` | Plan 6 (next) | +| `docs/superpowers/plans/2026-07-28-network-score-computation.md` | Plan 6 | | `docs/gate-contract.md` | Local + CI gate contract | | `internal/testutil/` | Go service harnesses | | `control-plane/src/testutil/` | TS DB/app harnesses | @@ -106,6 +106,7 @@ go test -tags=integration -count=1 ./... # Control plane cd control-plane && npm run test:unit # Needs local Postgres verilink_test (default URL uses 127.0.0.1:15432) +# Score recompute integration also needs: go run ./cmd/trust-engine && TRUST_ENGINE_ADDR=127.0.0.1:9091 cd control-plane && npm run test:integration # Edge / trust-engine local From 8a840cef58030eeb737674e8c17c354aaa86c153 Mon Sep 17 00:00:00 2001 From: Sanjay Date: Wed, 29 Jul 2026 03:36:59 -0400 Subject: [PATCH 2/3] fix: address CodeRabbit findings on Plan 6 PR B Poll for scheduler follow-up runs instead of fixed sleeps; guard recreatePool; isolate markDirty from write-path errors; arm shutdown watchdog before drain; clarify trust-engine test docs. Skip debounce-between-follow-ups: Plan 6 Decision 2 requires immediate dirty latch reruns (spacing remains via idle debounce only). Co-authored-by: Cursor --- .../__tests__/integration/score-recompute.test.ts | 14 +++++++++++++- control-plane/src/db/client.ts | 5 +++++ .../src/domains/attestation/attestationService.ts | 9 ++++++++- .../src/domains/graph/recomputeScheduler.test.ts | 15 +++++++++++++-- control-plane/src/index.ts | 12 ++++++------ docs/superpowers/plans/HANDOVER.md | 5 +++-- 6 files changed, 48 insertions(+), 12 deletions(-) diff --git a/control-plane/src/__tests__/integration/score-recompute.test.ts b/control-plane/src/__tests__/integration/score-recompute.test.ts index f3e33ee..67ee65c 100644 --- a/control-plane/src/__tests__/integration/score-recompute.test.ts +++ b/control-plane/src/__tests__/integration/score-recompute.test.ts @@ -42,6 +42,18 @@ function delay(ms: number): Promise { return new Promise((r) => setTimeout(r, ms)); } +async function waitUntil( + pred: () => boolean, + opts: { timeoutMs?: number; intervalMs?: number } = {} +): Promise { + const timeoutMs = opts.timeoutMs ?? 5000; + const intervalMs = opts.intervalMs ?? 20; + const deadline = Date.now() + timeoutMs; + while (!pred() && Date.now() < deadline) { + await delay(intervalMs); + } +} + describe('Score recompute (live RunVeriRank)', { skip: skipLive }, () => { let pool: pg.Pool; let harness: ControlPlaneHarness; @@ -206,7 +218,7 @@ describe('Score recompute (live RunVeriRank)', { skip: skipLive }, () => { scheduler.markDirty(); resolveGate(); await runP; - await delay(80); + await waitUntil(() => calls.length >= 2); await scheduler.stop(); setRecomputeSchedulerForTests(null); diff --git a/control-plane/src/db/client.ts b/control-plane/src/db/client.ts index fef55ce..ac4b9b9 100644 --- a/control-plane/src/db/client.ts +++ b/control-plane/src/db/client.ts @@ -20,5 +20,10 @@ export let pool = createPool(); /** Recreate the singleton after `pool.end()` (multi-suite integration harness). */ export function recreatePool(): void { + if (!pool.ended) { + throw new Error( + 'recreatePool() called while the current pool is still open; call pool.end() first' + ); + } pool = createPool(); } \ No newline at end of file diff --git a/control-plane/src/domains/attestation/attestationService.ts b/control-plane/src/domains/attestation/attestationService.ts index 9a6e3b3..24b6ca8 100644 --- a/control-plane/src/domains/attestation/attestationService.ts +++ b/control-plane/src/domains/attestation/attestationService.ts @@ -10,6 +10,7 @@ import { verifyAttestation, type KeyCandidate } from '../../grpc/trustEngineClie import { AppError, CODES } from '../../shared/errors/AppError.js'; import { withTransaction } from '../../db/transaction.js'; import { getRecomputeScheduler } from '../graph/recomputeScheduler.js'; +import { logger } from '../../shared/logger.js'; function sha256hex(input: string): string { return createHash('sha256').update(input).digest('hex'); @@ -232,7 +233,13 @@ export async function submitAttestation(opts: { }, client); }); // Plan 6: enqueue score recompute after durable ingest (outside TX). - getRecomputeScheduler().markDirty(); + // Isolate from write-path error mapping so a scheduler throw cannot + // turn a committed submit into CONFLICT / failed response. + try { + getRecomputeScheduler().markDirty(); + } catch (err) { + logger.error({ err }, 'failed to enqueue score recompute after ingest'); + } return row; } catch (err: any) { if (err instanceof AppError) throw err; diff --git a/control-plane/src/domains/graph/recomputeScheduler.test.ts b/control-plane/src/domains/graph/recomputeScheduler.test.ts index c7b058c..4ee89c8 100644 --- a/control-plane/src/domains/graph/recomputeScheduler.test.ts +++ b/control-plane/src/domains/graph/recomputeScheduler.test.ts @@ -6,6 +6,18 @@ function delay(ms: number): Promise { return new Promise((r) => setTimeout(r, ms)); } +async function waitUntil( + pred: () => boolean, + opts: { timeoutMs?: number; intervalMs?: number } = {} +): Promise { + const timeoutMs = opts.timeoutMs ?? 1000; + const intervalMs = opts.intervalMs ?? 5; + const deadline = Date.now() + timeoutMs; + while (!pred() && Date.now() < deadline) { + await delay(intervalMs); + } +} + describe('RecomputeScheduler', () => { let calls: Date[]; let gate: { release: () => void; wait: Promise } | null; @@ -47,8 +59,7 @@ describe('RecomputeScheduler', () => { scheduler.markDirty(); resolveGate(); await first; - // Allow follow-up loop to finish - await delay(20); + await waitUntil(() => calls.length >= 2); assert.equal(calls.length, 2); }); diff --git a/control-plane/src/index.ts b/control-plane/src/index.ts index 54ff9c6..a8d0640 100644 --- a/control-plane/src/index.ts +++ b/control-plane/src/index.ts @@ -36,6 +36,11 @@ async function main() { if (shuttingDown) return; shuttingDown = true; logger.info('Shutting down...'); + // Arm the watchdog before awaiting scheduler drain (gRPC / lock can stall). + const forceExit = setTimeout(() => { + logger.error('Forced shutdown after timeout'); + process.exit(1); + }, 10000); try { await scheduler.stop(); } catch (err) { @@ -43,15 +48,10 @@ async function main() { } server.close(async () => { await pool.end(); + clearTimeout(forceExit); logger.info('Bye.'); process.exit(0); }); - - // Force close after 10s - setTimeout(() => { - logger.error('Forced shutdown after timeout'); - process.exit(1); - }, 10000); }; process.on('SIGTERM', shutdown); diff --git a/docs/superpowers/plans/HANDOVER.md b/docs/superpowers/plans/HANDOVER.md index b0204a2..9bd5cc7 100644 --- a/docs/superpowers/plans/HANDOVER.md +++ b/docs/superpowers/plans/HANDOVER.md @@ -106,8 +106,9 @@ go test -tags=integration -count=1 ./... # Control plane cd control-plane && npm run test:unit # Needs local Postgres verilink_test (default URL uses 127.0.0.1:15432) -# Score recompute integration also needs: go run ./cmd/trust-engine && TRUST_ENGINE_ADDR=127.0.0.1:9091 -cd control-plane && npm run test:integration +# Score recompute integration also needs trust-engine running in another shell: +# go run ./cmd/trust-engine -grpc-port 9091 -http-port 8086 +cd control-plane && TRUST_ENGINE_ADDR=127.0.0.1:9091 npm run test:integration # Edge / trust-engine local go run ./cmd/edge-verifier From 34d964186d1c9b8ce952c095a6e2206d9ec47df8 Mon Sep 17 00:00:00 2001 From: Sanjay Date: Wed, 29 Jul 2026 03:42:54 -0400 Subject: [PATCH 3/3] fix: drain pending debounce on stop; harden score tests Set dirty when scheduling debounce so stop drains cancelled idle work, unref debounce timers, cancel debounce on kick, and load score modules via dynamic import after env defaults. Co-authored-by: Cursor --- .../integration/score-recompute.test.ts | 21 ++++++++++++++----- .../domains/graph/recomputeScheduler.test.ts | 18 ++++++++++++++++ .../src/domains/graph/recomputeScheduler.ts | 11 ++++++---- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/control-plane/src/__tests__/integration/score-recompute.test.ts b/control-plane/src/__tests__/integration/score-recompute.test.ts index 67ee65c..68037fd 100644 --- a/control-plane/src/__tests__/integration/score-recompute.test.ts +++ b/control-plane/src/__tests__/integration/score-recompute.test.ts @@ -22,11 +22,16 @@ import { signAttestationToken, } from '../../testutil/seedData.js'; import { startControlPlane, type ControlPlaneHarness } from '../../testutil/appHarness.js'; -import { recomputeNow } from '../../domains/graph/scoreComputationService.js'; -import { - RecomputeScheduler, - setRecomputeSchedulerForTests, -} from '../../domains/graph/recomputeScheduler.js'; + +// Score modules import config/pool at load time — load them after env defaults above +// via dynamic import in before() (static ESM imports would evaluate first). +type RecomputeNow = typeof import('../../domains/graph/scoreComputationService.js').recomputeNow; +type SchedulerCtor = typeof import('../../domains/graph/recomputeScheduler.js').RecomputeScheduler; +type SetScheduler = typeof import('../../domains/graph/recomputeScheduler.js').setRecomputeSchedulerForTests; + +let recomputeNow: RecomputeNow; +let RecomputeScheduler: SchedulerCtor; +let setRecomputeSchedulerForTests: SetScheduler; const trustEngineAddr = process.env.TRUST_ENGINE_ADDR; const inCi = process.env.CI === 'true' || process.env.GITHUB_ACTIONS === 'true'; @@ -61,6 +66,12 @@ describe('Score recompute (live RunVeriRank)', { skip: skipLive }, () => { let apiKey: string; before(async () => { + const scoreMod = await import('../../domains/graph/scoreComputationService.js'); + const schedMod = await import('../../domains/graph/recomputeScheduler.js'); + recomputeNow = scoreMod.recomputeNow; + RecomputeScheduler = schedMod.RecomputeScheduler; + setRecomputeSchedulerForTests = schedMod.setRecomputeSchedulerForTests; + pool = await setupTestDb(); harness = await startControlPlane(); }); diff --git a/control-plane/src/domains/graph/recomputeScheduler.test.ts b/control-plane/src/domains/graph/recomputeScheduler.test.ts index 4ee89c8..63bf2ee 100644 --- a/control-plane/src/domains/graph/recomputeScheduler.test.ts +++ b/control-plane/src/domains/graph/recomputeScheduler.test.ts @@ -128,4 +128,22 @@ describe('RecomputeScheduler', () => { assert.equal(calls.length, 0); await idle.stop(); }); + + it('stop drains a pending debounced markDirty (idle path)', async () => { + await scheduler.stop(); + const idleCalls: Date[] = []; + const idle = new RecomputeScheduler({ + debounceMs: 5_000, + intervalMs: 60_000, + recompute: async (t) => { + idleCalls.push(t); + }, + }); + idle.start(); + idle.markDirty(); + assert.equal(idle.isDirty, true); + await idle.stop(); + assert.equal(idleCalls.length, 1); + assert.equal(idle.isDirty, false); + }); }); diff --git a/control-plane/src/domains/graph/recomputeScheduler.ts b/control-plane/src/domains/graph/recomputeScheduler.ts index 966ef24..d0da469 100644 --- a/control-plane/src/domains/graph/recomputeScheduler.ts +++ b/control-plane/src/domains/graph/recomputeScheduler.ts @@ -51,10 +51,8 @@ export class RecomputeScheduler { // No-op until start() so integration suites that only exercise ingest // do not schedule background recomputes against a test pool. if (this.stopping || !this.started) return; - if (this.running) { - this.dirty = true; - return; - } + this.dirty = true; + if (this.running) return; this.scheduleDebounce(); } @@ -65,6 +63,7 @@ export class RecomputeScheduler { if (this.stopping) return; void this.kick(); }, this.debounceMs); + this.debounceTimer.unref?.(); } /** Entry used by hourly tick and debounce. */ @@ -76,6 +75,10 @@ export class RecomputeScheduler { this.dirty = true; return this.activeRun ?? Promise.resolve(); } + if (this.debounceTimer) { + clearTimeout(this.debounceTimer); + this.debounceTimer = null; + } this.running = true; this.activeRun = this.runLoop().finally(() => { this.running = false;