diff --git a/integrationTests/upgrade/cross-version-upgrade-residuals.test.ts b/integrationTests/upgrade/cross-version-upgrade-residuals.test.ts new file mode 100644 index 0000000000..937d0e0508 --- /dev/null +++ b/integrationTests/upgrade/cross-version-upgrade-residuals.test.ts @@ -0,0 +1,685 @@ +/** + * QA-promoted regression anchor — promoted from QA-660 (P-438), a gated qa-explorer finding. + * Originating QA scenario: QA-660 (gh#1865 + QA-658/P-437 residual). + * Coverage anchored: cross-version 5.1.22 -> current upgrade stays read-consistent at 55k rows + * through a SIGKILL-dirty shutdown and boot-boundary writes. + * + * The honest remainder of the cross-version upgrade-boot read-visibility investigation. + * + * cross-version-upgrade-visibility.test.ts (QA-658/P-437) already ran the GENUINE 5.1.22 -> + * current cross-version upgrade boot (real npm install, real dataRootDir handoff, real + * 6-surface matrix vs the index-independent base-store-scan oracle) and found 0/24 misses on a + * small, UNIFORM-width dataset stopped with a CLEAN shutdown. That result narrows #1865 but + * leaves three variables untested: dataset scale/shape, a DIRTY (SIGKILL) shutdown, and writes + * racing the boot boundary. This suite varies all three, stacked onto ONE upgrade boot (not + * three isolated runs — see the "why stacked, not isolated" note below), and reuses QA-658's + * oracle-vs-surface methodology verbatim. + * + * Arms exercised on the SAME dataRootDir handoff (5.1.22 -> current build): + * (a) SCALE + SHAPE — waveA: 50,000 rows, WIDTH-HETEROGENEOUS (6 disjoint field-set shapes + * cycling by index — narrow/no-extras, arrays, long strings, nested objects, mixed + * scalars, unicode — see resources.js hetExtra()), seeded and left to complete CLEANLY + * under 5.1.22. This is the scale+shape control population within the same run. + * (b) DIRTY SHUTDOWN — waveB: a SECOND width-heterogeneous load (target 40,000 rows) is + * split into 40 independent 1,000-row HTTP requests (so partial completion is possible — + * a single giant request is one atomic transaction; see "why chunked" below) fired with + * concurrency 6, and 5.1.22 is SIGKILLed ~150ms in. The current build then boots over + * this DIRTY (mid-write, unclean-shutdown) state. + * (c) CONCURRENT WRITES ACROSS THE BOOT BOUNDARY — waveC: starting the instant the current + * build is spawned (before `startHarper()`'s promise resolves), a writer loop fires + * single-row POST /Load/ requests every ~3ms against the instance's known hostname:port, + * continuing for 2s past the moment the instance reports ready, to see whether any land + * DURING the boot window (not just immediately after). + * + * Why chunked+concurrent for (b), not one giant unawaited request: an empirical probe (see run + * history) found a single large POST /Load/ call is ATOMIC at the Resource-handler-transaction + * boundary — killing mid-handler loses either ~0 or ~all of it, never a genuine partial. Many + * independent concurrent HTTP requests, each its own request/transaction, is what QA-608 used + * to get a real mixed acked/unacked/committed-but-unacked outcome, and is reused here. + * + * Why stacked in ONE run, not three isolated arms: budget (one foreground CI-timeout run) and + * the task's own framing ("does any of (a)/(b)/(c) — alone or combined —"). Each wave's rows + * are batch-tagged, so a miss can still be attributed to whichever wave it belongs to even + * without three separate boots. This is a real methodological tradeoff — noted, not hidden. + * + * Oracle discipline (unchanged from QA-658/D-230): base-store scan (index-independent) is the + * ORACLE. search_by_value is a secondary-index scan that JOINS through the primary record and + * SKIPs on absence — it can never reveal a dangling index entry, so every surface is compared + * against the oracle's id set, never against another surface. + * + * Read-surface transport note: at this scale, per-id HTTP round trips for all 6 surfaces would + * not finish in any sane budget. Oracle scan, ops search_by_value, cached point-GET, ops + * search_by_id, and SQL-by-batch are all tested at FULL POPULATION using bulk-capable transports + * (a single scan / a bulk custom endpoint that loops the SAME per-id read call server-side / + * ops's native `ids` array / a batch-predicate SQL query) — the READ SEMANTICS under test + * (does this id resolve on this surface) are unchanged; only the transport is batched. REST + * (GET /Widget/, no bulk verb) and a point-lookup form of SQL (`WHERE id IN (...)`) are + * SAMPLED at a stratified ~2,000 ids (see buildSample()) covering all three waves and the + * region around wave B's kill point, run with bounded concurrency. + * + * Reproduction: + * cd /home/kzyp/dev/harper + * timeout 1500 npm run test:integration -- "integrationTests/upgrade/cross-version-upgrade-residuals.test.ts" + * Harper SHA under test: 07c2bbcb9 (main) + * 5.1.x version installed: harper@5.1.22 (via ~/dev/tmp/qa658-harper51, HARPER_LEGACY_51_PATH honored) + */ +import { suite, test, before, after } from 'node:test'; +import { ok, strictEqual } from 'node:assert'; +import { resolve } from 'node:path'; +import { existsSync } from 'node:fs'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { + startHarper, + setupHarperWithFixture, + killHarper, + teardownHarper, + sendOperation, + type ContextWithHarper, +} from '@harperfast/integration-testing'; +// @ts-expect-error utils/client.mjs has no type declarations; runtime resolves fine +import { createApiClient } from '../apiTests/utils/client.mjs'; + +const FIXTURE_PATH = resolve(import.meta.dirname, 'cross-version-upgrade-residuals'); +const DB = 'qa660'; +const TABLE = 'Widget'; + +// ---- workload sizing -------------------------------------------------------------------- +const WAVE_A_COUNT = 50_000; // (a) scale + width-heterogeneous shape, clean completion +const WAVE_B_TARGET = 40_000; // (b) dirty-shutdown target; actual persisted count is empirical +const WAVE_B_CHUNK = 1_000; // per-request row count -> 40 independent requests +const WAVE_B_CONCURRENCY = 6; +const WAVE_B_KILL_DELAY_MS = 150; +const WAVE_C_WRITE_INTERVAL_MS = 3; // (c) boot-boundary writer cadence +const WAVE_C_POST_READY_MS = 2_000; // keep writing this long past the ready signal + +const SAMPLE_SIZE = 2_000; // stratified REST / SQL-IN sample size +const BULK_CHUNK = 8_000; // chunk size for cachedGetBulk / opsById bulk requests +const SQL_IN_CHUNK = 200; // chunk size for SQL `WHERE id IN (...)` + +// Provisioned by: cd ~/dev/tmp/qa658-harper51 && npm install harper@5.1.22 --no-save +const LEGACY_51_BIN_PATH = + process.env.HARPER_LEGACY_51_PATH ?? + resolve(process.env.HOME ?? '', 'dev/tmp/qa658-harper51/node_modules/harper/dist/bin/harper.js'); +const legacy51Available = existsSync(LEGACY_51_BIN_PATH); +const skipSuite = process.platform === 'win32' || !legacy51Available; + +// Re-passed verbatim on every startHarper() call across every restart — never omitted. +const BOOT_CONFIG = { logging: { console: true, level: 'error' } }; + +interface ScanRow { + id: string; + batch: string; +} +interface SurfaceMatrix { + label: string; + oracle: Map; // id -> batch, from ScanAll (full population, ground truth) + opsScanIds: Set; // full population (search_by_value per known batch, unioned) + cachedGetMisses: string[]; // full population (bulk custom endpoint, same table.get() path) + opsByIdMisses: string[]; // full population (ops search_by_id, ids array, chunked) + sqlByBatchIds: Set; // full population (SQL WHERE batch=X, per batch, unioned) + sampleIds: string[]; // stratified sample used for REST + SQL-IN + restMisses: string[]; // sampled + sqlInMisses: string[]; // sampled (WHERE id IN (...)) +} + +suite( + 'QA-660 cross-version upgrade-boot read visibility: scale+shape, dirty shutdown, boot-boundary concurrency (#1865)', + { skip: skipSuite }, + (ctx: ContextWithHarper) => { + let client: ReturnType; + let httpURL: string; + const findings: string[] = []; + function log(line: string) { + findings.push(line); + console.log(`[QA-660] ${line}`); + } + + function get(path: string, baseURL = httpURL): Promise { + return fetch(`${baseURL}${path}`, { headers: { Authorization: client.headers.Authorization } }); + } + function post(path: string, body: unknown, baseURL = httpURL): Promise { + return fetch(`${baseURL}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'Authorization': client?.headers.Authorization ?? '' }, + body: JSON.stringify(body), + }); + } + + async function pollReady(path: string, maxWaitMs = 60_000, baseURL = httpURL): Promise { + const deadline = Date.now() + maxWaitMs; + while (Date.now() < deadline) { + try { + const r = await get(path, baseURL); + if (r.status !== 404) return; + } catch { + /* not up yet */ + } + await sleep(100); + } + throw new Error(`route ${path} still 404 after ${maxWaitMs}ms`); + } + + async function refreshClient() { + client = createApiClient(ctx.harper); + httpURL = ctx.harper.httpURL; + } + + function chunkArray(arr: T[], size: number): T[][] { + const out: T[][] = []; + for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size)); + return out; + } + + async function mapLimit(items: T[], limit: number, fn: (item: T) => Promise): Promise { + const results: R[] = new Array(items.length); + let idx = 0; + async function worker() { + for (;;) { + const i = idx++; + if (i >= items.length) return; + results[i] = await fn(items[i]); + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker())); + return results; + } + + // ---- (a) clean, width-heterogeneous bulk load ------------------------------------- + async function loadClean(batch: string, count: number, concurrency: number): Promise<{ completed: number }> { + const r = await post('/Load/', { count, batch, concurrency }); + strictEqual(r.status, 200, `Load(${batch}) should return 200`); + const body = (await r.json()) as { completed: number }; + return body; + } + + // ---- (b) chunked concurrent load + SIGKILL mid-write -------------------------------- + async function dirtyShutdownLoad( + batch: string, + target: number, + chunkSize: number, + concurrency: number, + killDelayMs: number + ): Promise<{ numChunks: number; ackedChunks: number }> { + const numChunks = Math.ceil(target / chunkSize); + let acked = 0; + let nextChunk = 0; + async function chunkWorker() { + for (;;) { + const c = nextChunk++; + if (c >= numChunks) return; + const startIndex = c * chunkSize; + const count = Math.min(chunkSize, target - startIndex); + try { + const r = await post('/Load/', { count, batch, startIndex, concurrency: 20 }); + if (r.status === 200) { + await r.json(); + acked++; + } + } catch { + /* in-flight at kill time */ + } + } + } + const workers = Array.from({ length: concurrency }, () => chunkWorker()); + const allDone = Promise.allSettled(workers); + + await sleep(killDelayMs); + log(`(b) dirty-shutdown trigger: ${acked}/${numChunks} chunk requests ACKed (client received 200) before kill`); + const proc = ctx.harper.process; + const pid = proc.pid!; + process.kill(-pid, 'SIGKILL'); + await new Promise((res) => proc.once('exit', res)); + await allDone; + return { numChunks, ackedChunks: acked }; + } + + // ---- (c) writer loop racing the current-build boot boundary -------------------------- + interface WriteAttempt { + t0: number; + ok: boolean; + } + function startBootBoundaryWriter(bootHttpURL: string, batch: string) { + let stop = false; + let idx = 0; + const attempts: WriteAttempt[] = []; + const loopPromise = (async () => { + while (!stop) { + const i = idx++; + const t0 = Date.now(); + try { + const r = await fetch(`${bootHttpURL}/Load/`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ count: 1, batch, startIndex: i, concurrency: 1 }), + signal: AbortSignal.timeout(2_000), + }); + attempts.push({ t0, ok: r.status === 200 }); + } catch { + attempts.push({ t0, ok: false }); + } + await sleep(WAVE_C_WRITE_INTERVAL_MS); + } + })(); + return { + stop: async () => { + stop = true; + await loopPromise; + }, + attempts, + }; + } + + // ---- full-population oracle scan ------------------------------------------------------ + async function oracleScan(): Promise> { + const r = await get('/ScanAll/'); + strictEqual(r.status, 200, `/ScanAll/ should return 200, got ${r.status}`); + const body = (await r.json()) as { count: number; rows: ScanRow[] }; + const m = new Map(); + for (const row of body.rows) m.set(row.id, row.batch); + return m; + } + + async function opsScanByValue(batches: string[]): Promise> { + const ids = new Set(); + for (const batch of batches) { + const res = await sendOperation(ctx.harper, { + operation: 'search_by_value', + schema: DB, + table: TABLE, + search_attribute: 'batch', + search_value: batch, + get_attributes: ['id'], + }); + for (const r of res as Array<{ id: string }>) ids.add(r.id); + } + return ids; + } + + async function cachedGetBulkMisses(ids: string[]): Promise { + const misses: string[] = []; + for (const chunk of chunkArray(ids, BULK_CHUNK)) { + const r = await post('/PointGetBulk/', { ids: chunk }); + strictEqual(r.status, 200, 'PointGetBulk should return 200'); + const body = (await r.json()) as { misses: string[] }; + misses.push(...body.misses); + } + return misses; + } + + async function opsByIdBulkMisses(ids: string[]): Promise { + const misses: string[] = []; + for (const chunk of chunkArray(ids, BULK_CHUNK)) { + const res = (await sendOperation(ctx.harper, { + operation: 'search_by_id', + schema: DB, + table: TABLE, + ids: chunk, + get_attributes: ['id'], + })) as Array<{ id: string }>; + const found = new Set(res.map((r) => r.id)); + for (const id of chunk) if (!found.has(id)) misses.push(id); + } + return misses; + } + + async function sqlByBatch(batches: string[]): Promise> { + const ids = new Set(); + for (const batch of batches) { + const res = (await sendOperation(ctx.harper, { + operation: 'sql', + sql: `SELECT id FROM ${DB}.${TABLE} WHERE batch = '${batch}'`, + })) as Array<{ id: string }>; + for (const r of res) ids.add(r.id); + } + return ids; + } + + async function sqlInMisses(ids: string[]): Promise { + const misses: string[] = []; + for (const chunk of chunkArray(ids, SQL_IN_CHUNK)) { + const inList = chunk.map((id) => `'${id}'`).join(','); + const res = (await sendOperation(ctx.harper, { + operation: 'sql', + sql: `SELECT id FROM ${DB}.${TABLE} WHERE id IN (${inList})`, + })) as Array<{ id: string }>; + const found = new Set(res.map((r) => r.id)); + for (const id of chunk) if (!found.has(id)) misses.push(id); + } + return misses; + } + + async function restMisses(ids: string[]): Promise { + const results = await mapLimit(ids, 40, async (id) => { + const r = await get(`/Widget/${id}`); + return { id, ok: r.status === 200 }; + }); + return results.filter((r) => !r.ok).map((r) => r.id); + } + + /** Stratified sample: proportional per-batch coverage, plus every id near wave B's kill boundary. */ + function buildSample(oracle: Map): string[] { + const byBatch = new Map(); + for (const [id, batch] of oracle) { + if (!byBatch.has(batch)) byBatch.set(batch, []); + byBatch.get(batch)!.push(id); + } + const sample: string[] = []; + for (const [, ids] of byBatch) { + ids.sort(); + const take = Math.max(1, Math.min(ids.length, Math.round((SAMPLE_SIZE * ids.length) / oracle.size))); + const step = Math.max(1, Math.floor(ids.length / take)); + for (let i = 0; i < ids.length && sample.length < SAMPLE_SIZE; i += step) sample.push(ids[i]); + } + return [...new Set(sample)]; + } + + async function buildMatrix(label: string): Promise { + const oracle = await oracleScan(); + const batches = [...new Set(oracle.values())]; + const allIds = [...oracle.keys()]; + const sampleIds = buildSample(oracle); + + const [opsScanIds, cachedMisses, opsByIdMissesResult, sqlByBatchIds, sqlInM, restM] = await Promise.all([ + opsScanByValue(batches), + cachedGetBulkMisses(allIds), + opsByIdBulkMisses(allIds), + sqlByBatch(batches), + sqlInMisses(sampleIds), + restMisses(sampleIds), + ]); + + const m: SurfaceMatrix = { + label, + oracle, + opsScanIds, + cachedGetMisses: cachedMisses, + opsByIdMisses: opsByIdMissesResult, + sqlByBatchIds, + sampleIds, + restMisses: restM, + sqlInMisses: sqlInM, + }; + log( + `${label}: oracle=${oracle.size} opsScan=${opsScanIds.size} sqlByBatch=${sqlByBatchIds.size} ` + + `cachedGET-misses=${cachedMisses.length} opsById-misses=${opsByIdMissesResult.length} ` + + `(sample=${sampleIds.length}) REST-misses=${restM.length} SQL-IN-misses=${sqlInM.length}` + ); + if (opsScanIds.size !== oracle.size) + log(`${label}: [#1865 CHECK] opsScan size (${opsScanIds.size}) != oracle size (${oracle.size})`); + if (sqlByBatchIds.size !== oracle.size) + log(`${label}: [#1865 CHECK] sqlByBatch size (${sqlByBatchIds.size}) != oracle size (${oracle.size})`); + if (cachedMisses.length) + log( + `${label}: [#1865 CHECK] cachedGET MISSES (${cachedMisses.length}): ${cachedMisses.slice(0, 20).join(',')}${cachedMisses.length > 20 ? ',...' : ''}` + ); + if (opsByIdMissesResult.length) + log( + `${label}: [#1865 CHECK] opsById MISSES (${opsByIdMissesResult.length}): ${opsByIdMissesResult.slice(0, 20).join(',')}${opsByIdMissesResult.length > 20 ? ',...' : ''}` + ); + if (restM.length) + log( + `${label}: [#1865 CHECK] REST MISSES (sampled, ${restM.length}): ${restM.slice(0, 20).join(',')}${restM.length > 20 ? ',...' : ''}` + ); + if (sqlInM.length) + log( + `${label}: [#1865 CHECK] SQL-IN MISSES (sampled, ${sqlInM.length}): ${sqlInM.slice(0, 20).join(',')}${sqlInM.length > 20 ? ',...' : ''}` + ); + return m; + } + + let matrixPreUpgrade: SurfaceMatrix; + let matrixPostUpgrade: SurfaceMatrix; + let matrixAfterHeal: SurfaceMatrix | null = null; + let matrixAfterRestart2: SurfaceMatrix; + let bWaveResult: { numChunks: number; ackedChunks: number }; + let cWaveAttempts: WriteAttempt[]; + let cWaveReadyAt: number; + + before(async () => { + log(`harper SHA under test: 07c2bbcb9; legacy 5.1.x bin: ${LEGACY_51_BIN_PATH}`); + await setupHarperWithFixture(ctx, FIXTURE_PATH, { + config: BOOT_CONFIG, + env: { TC_AGREEMENT: 'yes' }, + harperBinPath: LEGACY_51_BIN_PATH, + }); + await refreshClient(); + await pollReady('/ScanAll/'); + log(`cold boot (5.1.x) complete at ${httpURL}, dataRootDir=${ctx.harper.dataRootDir}`); + }); + + after(async () => { + await teardownHarper(ctx); + log('--- FINDINGS SUMMARY ---'); + for (const line of findings) console.log(`[QA-660][summary] ${line}`); + }); + + test('Q0 (a) seed waveA CLEANLY under 5.1.x: 50k width-heterogeneous rows', { timeout: 120_000 }, async () => { + const { completed } = await loadClean('waveA', WAVE_A_COUNT, 300); + strictEqual(completed, WAVE_A_COUNT, `PRECONDITION: waveA load must report ${WAVE_A_COUNT} completed puts`); + matrixPreUpgrade = await buildMatrix('PRE-UPGRADE (5.1.x, waveA only)'); + strictEqual( + matrixPreUpgrade.oracle.size, + WAVE_A_COUNT, + 'PRECONDITION: oracle scan must see all waveA rows pre-upgrade' + ); + strictEqual( + matrixPreUpgrade.opsScanIds.size, + WAVE_A_COUNT, + 'PRECONDITION: ops search_by_value must see all waveA rows pre-upgrade' + ); + strictEqual( + matrixPreUpgrade.cachedGetMisses.length, + 0, + 'PRECONDITION: cached point-GET must miss nothing pre-upgrade' + ); + strictEqual(matrixPreUpgrade.restMisses.length, 0, 'PRECONDITION: REST sample must miss nothing pre-upgrade'); + strictEqual( + matrixPreUpgrade.opsByIdMisses.length, + 0, + 'PRECONDITION: ops search_by_id must miss nothing pre-upgrade' + ); + strictEqual(matrixPreUpgrade.sqlInMisses.length, 0, 'PRECONDITION: SQL-IN sample must miss nothing pre-upgrade'); + strictEqual( + matrixPreUpgrade.sqlByBatchIds.size, + WAVE_A_COUNT, + 'PRECONDITION: SQL-by-batch must see all waveA rows pre-upgrade' + ); + }); + + test( + 'Q1 (b) dirty-shutdown waveB: chunked concurrent load + SIGKILL 5.1.x mid-write', + { timeout: 60_000 }, + async () => { + bWaveResult = await dirtyShutdownLoad( + 'waveB', + WAVE_B_TARGET, + WAVE_B_CHUNK, + WAVE_B_CONCURRENCY, + WAVE_B_KILL_DELAY_MS + ); + log( + `(b) 5.1.x SIGKILLed mid-write: ${bWaveResult.ackedChunks}/${bWaveResult.numChunks} of waveB's ${WAVE_B_CHUNK}-row chunk requests ACKed before kill (${WAVE_B_TARGET} rows targeted)` + ); + // PRECONDITION: the kill must have landed while write requests were genuinely still + // outstanding, not after every chunk had already been acked (a vacuous "clean-ish" kill). + ok( + bWaveResult.ackedChunks < bWaveResult.numChunks, + `PRECONDITION FAILED: all ${bWaveResult.numChunks} waveB chunks were ACKed before the kill — the SIGKILL landed after the write drained, not mid-write` + ); + } + ); + + test( + 'Q2 (a)+(b)+(c) CROSS-VERSION UPGRADE BOOT: current build over the dirty 5.1.x state, writes racing the boot boundary', + { timeout: 120_000 }, + async () => { + const hostname = ctx.harper.hostname; + const bootHttpURL = `http://${hostname}:9926`; + const startPromise = startHarper(ctx, { config: BOOT_CONFIG, env: {} }); + const writer = startBootBoundaryWriter(bootHttpURL, 'waveC'); + + await startPromise; + cWaveReadyAt = Date.now(); + await sleep(WAVE_C_POST_READY_MS); + await writer.stop(); + cWaveAttempts = writer.attempts; + + const beforeReady = cWaveAttempts.filter((a) => a.t0 < cWaveReadyAt); + const afterReady = cWaveAttempts.filter((a) => a.t0 >= cWaveReadyAt); + const beforeReadyOk = beforeReady.filter((a) => a.ok).length; + const afterReadyOk = afterReady.filter((a) => a.ok).length; + log( + `(c) boot-boundary writer: ${cWaveAttempts.length} attempts total; ` + + `before-ready=${beforeReady.length} (${beforeReadyOk} succeeded), after-ready=${afterReady.length} (${afterReadyOk} succeeded)` + ); + if (beforeReadyOk === 0) { + log( + '(c) COULD NOT ARM: 0 of ' + + beforeReady.length + + ' write attempts issued before the ready signal succeeded — every one hit connection-refused. ' + + "Harper's HTTP listener does not appear to accept connections until boot (including crash " + + 'recovery + cross-version migration) is fully complete; there is no externally-observable ' + + 'partial-availability window via plain HTTP for this build/config. Reporting plainly per the brief ' + + 'rather than substituting a weaker arm.' + ); + } else { + log( + `(c) ARMED: ${beforeReadyOk} write(s) landed genuinely DURING the boot window (before the ready signal).` + ); + } + + await refreshClient(); + await pollReady('/ScanAll/'); + log(`cross-version upgrade boot (current build over dirty 5.1.x data) complete at ${httpURL}`); + + matrixPostUpgrade = await buildMatrix('POST-CROSSVER-UPGRADE (dirty waveB + boot-boundary waveC)'); + + // The oracle itself must be internally sane: every waveA row (clean, pre-kill) must + // still be present — a scale/shape-triggered defect big enough to lose ORACLE-visible + // rows would be a much larger problem than a read-visibility miss. + let waveACount = 0; + for (const batch of matrixPostUpgrade.oracle.values()) if (batch === 'waveA') waveACount++; + strictEqual( + waveACount, + WAVE_A_COUNT, + `ORACLE: all ${WAVE_A_COUNT} waveA rows must still be scan-visible after the upgrade boot` + ); + } + ); + + test('Q3 POST-UPGRADE: does any surface miss what the oracle sees? (#1865 check, all three waves)', () => { + const m = matrixPostUpgrade; + strictEqual( + m.opsScanIds.size, + m.oracle.size, + `[#1865 CHECK] ops search_by_value union expected ${m.oracle.size}, got ${m.opsScanIds.size}` + ); + strictEqual( + m.sqlByBatchIds.size, + m.oracle.size, + `[#1865 CHECK] SQL-by-batch union expected ${m.oracle.size}, got ${m.sqlByBatchIds.size}` + ); + strictEqual( + m.cachedGetMisses.length, + 0, + `[#1865 CHECK] cached point-GET (bulk) expected 0 misses, got ${m.cachedGetMisses.length}` + ); + strictEqual( + m.opsByIdMisses.length, + 0, + `[#1865 CHECK] ops search_by_id (bulk) expected 0 misses, got ${m.opsByIdMisses.length}` + ); + strictEqual( + m.restMisses.length, + 0, + `[#1865 CHECK] REST (sampled ${m.sampleIds.length}) expected 0 misses, got ${m.restMisses.length}` + ); + strictEqual( + m.sqlInMisses.length, + 0, + `[#1865 CHECK] SQL WHERE id IN (sampled ${m.sampleIds.length}) expected 0 misses, got ${m.sqlInMisses.length}` + ); + }); + + test('Q4 write-then-read heal: does re-putting a missed row repair it?', { timeout: 30_000 }, async (t) => { + const anyMisses = + matrixPostUpgrade.cachedGetMisses.length || + matrixPostUpgrade.restMisses.length || + matrixPostUpgrade.opsByIdMisses.length || + matrixPostUpgrade.sqlInMisses.length; + if (!anyMisses) { + t.skip('no miss observed after the cross-version upgrade boot on any point surface — nothing to heal'); + return; + } + const targetId = + matrixPostUpgrade.cachedGetMisses[0] ?? + matrixPostUpgrade.restMisses[0] ?? + matrixPostUpgrade.opsByIdMisses[0] ?? + matrixPostUpgrade.sqlInMisses[0]; + const batch = matrixPostUpgrade.oracle.get(targetId) ?? 'unknown'; + log(`Q4: healing probe targets ${targetId} (batch=${batch}, missed pre-heal)`); + + const beforeHealRes = await post('/PointGetBulk/', { ids: [targetId] }); + const beforeHeal = ((await beforeHealRes.json()) as { misses: string[] }).misses.length === 0; + const r = await post('/Touch/', { + id: targetId, + sku: `SKU-${batch}-touched`, + batch, + name: `Widget ${batch} touched`, + }); + strictEqual(r.status, 200, 'Touch should return 200'); + const touchBody = (await r.json()) as { foundAfterTouch: boolean }; + const afterHealRes = await post('/PointGetBulk/', { ids: [targetId] }); + const afterHeal = ((await afterHealRes.json()) as { misses: string[] }).misses.length === 0; + log( + `Q4: ${targetId} cachedGET before-write=${beforeHeal}, Touch.foundAfterTouch=${touchBody.foundAfterTouch}, ` + + `cachedGET after-write=${afterHeal} -> ${afterHeal ? 'WRITE HEALS the miss' : 'STILL MISSING after the write'}` + ); + matrixAfterHeal = await buildMatrix('POST-HEAL'); + ok(true, `heal probe recorded (before=${beforeHeal}, after=${afterHeal})`); + }); + + test( + 'Q5 RESTART #2 (current build, no further version change): is any miss stable, cleared, or does it grow?', + { timeout: 120_000 }, + async () => { + await killHarper(ctx); + log('killed Harper cleanly (SIGTERM) for restart #2 (same-build, boot-order check)'); + await startHarper(ctx, { config: BOOT_CONFIG, env: {} }); + await refreshClient(); + await pollReady('/ScanAll/'); + log(`restart #2 (same-build) complete at ${httpURL}`); + + matrixAfterRestart2 = await buildMatrix('POST-RESTART-2 (same-build)'); + strictEqual( + matrixAfterRestart2.oracle.size, + matrixPostUpgrade.oracle.size, + 'ORACLE: population size must be stable across the second (same-build) restart' + ); + + const cmp = (name: string, a: string[], b: string[]) => { + log( + `Q5 boot-order check (${name}): post-crossver-upgrade misses=${b.length}, post-restart#2 misses=${a.length} -> ` + + `${a.length === 0 && b.length > 0 ? 'CLEARED on 2nd (same-build) boot (order-dependent)' : a.length > 0 && b.length === 0 ? 'NEW MISS introduced by 2nd boot alone' : a.length === b.length ? 'UNCHANGED across boots (stable)' : 'CHANGED (different miss count)'}` + ); + }; + cmp('cachedGetMisses', matrixAfterRestart2.cachedGetMisses, matrixPostUpgrade.cachedGetMisses); + cmp('opsByIdMisses', matrixAfterRestart2.opsByIdMisses, matrixPostUpgrade.opsByIdMisses); + cmp('restMisses (sampled)', matrixAfterRestart2.restMisses, matrixPostUpgrade.restMisses); + cmp('sqlInMisses (sampled)', matrixAfterRestart2.sqlInMisses, matrixPostUpgrade.sqlInMisses); + + log( + `FINAL VISIBILITY MATRIX SUMMARY:\n` + + ` PRE-UPGRADE (5.1.x, waveA) : oracle=${matrixPreUpgrade.oracle.size} opsScan=${matrixPreUpgrade.opsScanIds.size} sqlByBatch=${matrixPreUpgrade.sqlByBatchIds.size} cachedGET-misses=${matrixPreUpgrade.cachedGetMisses.length} opsById-misses=${matrixPreUpgrade.opsByIdMisses.length} REST-misses=${matrixPreUpgrade.restMisses.length} SQL-IN-misses=${matrixPreUpgrade.sqlInMisses.length}\n` + + ` POST-CROSSVER-UPGRADE (a+b+c): oracle=${matrixPostUpgrade.oracle.size} opsScan=${matrixPostUpgrade.opsScanIds.size} sqlByBatch=${matrixPostUpgrade.sqlByBatchIds.size} cachedGET-misses=${matrixPostUpgrade.cachedGetMisses.length} opsById-misses=${matrixPostUpgrade.opsByIdMisses.length} REST-misses=${matrixPostUpgrade.restMisses.length} SQL-IN-misses=${matrixPostUpgrade.sqlInMisses.length}\n` + + (matrixAfterHeal + ? ` POST-HEAL : oracle=${matrixAfterHeal.oracle.size} cachedGET-misses=${matrixAfterHeal.cachedGetMisses.length} opsById-misses=${matrixAfterHeal.opsByIdMisses.length} REST-misses=${matrixAfterHeal.restMisses.length} SQL-IN-misses=${matrixAfterHeal.sqlInMisses.length}\n` + : '') + + ` POST-RESTART-2 : oracle=${matrixAfterRestart2.oracle.size} opsScan=${matrixAfterRestart2.opsScanIds.size} sqlByBatch=${matrixAfterRestart2.sqlByBatchIds.size} cachedGET-misses=${matrixAfterRestart2.cachedGetMisses.length} opsById-misses=${matrixAfterRestart2.opsByIdMisses.length} REST-misses=${matrixAfterRestart2.restMisses.length} SQL-IN-misses=${matrixAfterRestart2.sqlInMisses.length}\n` + + ` (b) dirty-shutdown trigger : ${bWaveResult.ackedChunks}/${bWaveResult.numChunks} waveB chunks ACKed before SIGKILL (target ${WAVE_B_TARGET} rows)\n` + + ` (c) boot-boundary writer : ${cWaveAttempts.filter((a) => a.t0 < cWaveReadyAt).length} pre-ready attempts, ${cWaveAttempts.filter((a) => a.t0 < cWaveReadyAt && a.ok).length} succeeded pre-ready` + ); + } + ); + } +); diff --git a/integrationTests/upgrade/cross-version-upgrade-residuals/config.yaml b/integrationTests/upgrade/cross-version-upgrade-residuals/config.yaml new file mode 100644 index 0000000000..efffc0833f --- /dev/null +++ b/integrationTests/upgrade/cross-version-upgrade-residuals/config.yaml @@ -0,0 +1,5 @@ +graphqlSchema: + files: '*.graphql' +jsResource: + files: resources.js +rest: true diff --git a/integrationTests/upgrade/cross-version-upgrade-residuals/resources.js b/integrationTests/upgrade/cross-version-upgrade-residuals/resources.js new file mode 100644 index 0000000000..9ee17d1f27 --- /dev/null +++ b/integrationTests/upgrade/cross-version-upgrade-residuals/resources.js @@ -0,0 +1,162 @@ +// QA-660 (#1865 residual, source QA-658/P-437) — cross-version upgrade-boot read-visibility +// fixture, extended for scale + width-heterogeneity + bulk-capable read surfaces (per-id HTTP +// round trips do not scale to 50k+ rows). +// +// POST /Load/ { count, batch, startIndex, concurrency } — seeds WIDTH-HETEROGENEOUS rows +// concurrently (a worker pool of unawaited table.put() calls, not a serial loop), so a +// SIGKILL sent mid-call has a real externally-timeable window to land mid-write. +// GET /LoadStatus/?batch=X — in-memory progress counter for +// the CURRENTLY RUNNING Load call (polled by the test to fire a SIGKILL genuinely mid-write, +// and to detect the concurrent-writes-across-boot-boundary race). +// GET /ScanAll/ — uncached, INDEX-INDEPENDENT +// full base-store scan. THE ORACLE: proves a row is physically on disk regardless of any +// cache/secondary-index state. Returns {id, batch} pairs so the test can partition by batch +// from a single scan pass. +// POST /PointGetBulk/ { ids: [...] } — cached point-GET surface, +// batched at the TRANSPORT layer only: server-side loop of the same table.get(id) call the +// single-id /PointGet/ used, so the read semantics under test are unchanged. +// GET /PointGet/?id=X — single-id cached point-GET +// (kept for parity/spot checks). +// POST /Touch/ — write-then-read heal probe; +// accepts any row shape (width-heterogeneous rows carry different field sets). +// +// Widget is a component-defined table (schema.graphql) with a FIXED declared field set (id/sku/ +// batch/name) but rows carry additional UNDECLARED fields to build a non-trivial typed-struct +// dictionary (#1865's suspected mechanism) — Harper's document-store tables accept additional +// properties beyond the declared schema. + +function getTable() { + return databases['qa660']['Widget']; +} + +function qp(query, name, def) { + const v = query && (query.get ? query.get(name) : query[name]); + return v == null ? def : v; +} + +/** + * Width-heterogeneous, type-heterogeneous extra-field generator. Six disjoint shapes (narrow, + * arrays, long strings, nested objects, mixed scalars, unicode) cycling by index so the seeded + * dataset is NOT uniform-width — the point of arm (a) in QA-660. + */ +function hetExtra(idx) { + const shape = idx % 6; + switch (shape) { + case 0: + return {}; // narrow: only the declared fields + case 1: + return { tags: [`t${idx % 7}`, `t${idx % 11}`, `t${idx % 13}`], weight: idx * 1.5 }; + case 2: + return { description: 'd'.repeat(64 + (idx % 200)), score: idx % 997 }; + case 3: + return { meta: { a: idx, b: `${idx}`, nested: { deep: idx % 13, flag: idx % 2 === 0 } } }; + case 4: + return { + extra1: idx, + extra2: `v${idx}`, + extra3: [idx, idx + 1, idx + 2], + extra4: idx % 2 === 0, + extra5: 'y'.repeat(40 + (idx % 80)), + extra6: idx * 0.001, + }; + default: + return { label: 'L'.repeat(300), unicode: '标签-' + (idx % 50), gauge: Math.PI * (1 + (idx % 97)) }; + } +} + +function makeRow(batch, idx) { + const id = `w-${batch}-${idx}`; + return { id, sku: `SKU-${batch}-${idx}`, batch, name: `Widget ${batch} ${idx}`, ...hetExtra(idx) }; +} + +export class Load extends Resource { + static loadAsInstance = false; + async post(query, body) { + const b = body || query || {}; + const count = Number(b.count) || 0; + const batch = b.batch || 'default'; + const startIndex = Number(b.startIndex) || 0; + const concurrency = Math.max(1, Number(b.concurrency) || 100); + const table = getTable(); + globalThis.__loadProgress = globalThis.__loadProgress || {}; + globalThis.__loadProgress[batch] = 0; + let idx = startIndex; + const end = startIndex + count; + const ids = []; + let completed = 0; + async function worker() { + for (;;) { + const i = idx++; + if (i >= end) return; + const row = makeRow(batch, i); + await table.put(row); + ids.push(row.id); + completed++; + globalThis.__loadProgress[batch] = completed; + } + } + await Promise.all(Array.from({ length: concurrency }, () => worker())); + return { ok: true, count, batch, completed, ids }; + } +} + +export class LoadStatus extends Resource { + static loadAsInstance = false; + async get(query) { + const batch = qp(query, 'batch', 'default'); + const completed = (globalThis.__loadProgress && globalThis.__loadProgress[batch]) || 0; + return { batch, completed }; + } +} + +export class PointGet extends Resource { + static loadAsInstance = false; + async get(query) { + const id = qp(query, 'id'); + const table = getTable(); + const rec = await table.get(id); + return { id, found: rec != null, record: rec ?? null }; + } +} + +export class PointGetBulk extends Resource { + static loadAsInstance = false; + async post(query, body) { + const b = body || query || {}; + const ids = Array.isArray(b.ids) ? b.ids : []; + const table = getTable(); + const misses = []; + let hits = 0; + for (const id of ids) { + const rec = await table.get(id); + if (rec != null) hits++; + else misses.push(id); + } + return { requested: ids.length, hits, misses }; + } +} + +export class ScanAll extends Resource { + static loadAsInstance = false; + async get() { + const table = getTable(); + const rows = []; + for await (const r of table.search({ conditions: [] })) { + rows.push({ id: r.id, batch: r.batch }); + } + rows.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); + return { count: rows.length, rows }; + } +} + +export class Touch extends Resource { + static loadAsInstance = false; + async post(query, body) { + const row = body || query || {}; + if (!row.id) throw new Error('Touch requires id'); + const table = getTable(); + await table.put({ ...row, touched: Date.now() }); + const after = await table.get(row.id); + return { ok: true, id: row.id, foundAfterTouch: after != null }; + } +} diff --git a/integrationTests/upgrade/cross-version-upgrade-residuals/schema.graphql b/integrationTests/upgrade/cross-version-upgrade-residuals/schema.graphql new file mode 100644 index 0000000000..2a4fc35109 --- /dev/null +++ b/integrationTests/upgrade/cross-version-upgrade-residuals/schema.graphql @@ -0,0 +1,11 @@ +# QA-660 (#1865 residual, source QA-658/P-437) — component-defined table used across a GENUINE +# cross-version upgrade boot (5.1.22 -> current), with a dirty (SIGKILL) shutdown, width- +# heterogeneous rows at scale, and concurrent writes across the boot boundary. Declared fields +# are fixed/narrow; seeded rows also carry additional undeclared fields (see resources.js +# hetExtra()) to build a non-trivial typed-struct dictionary. +type Widget @table(database: "qa660") @export { + id: ID @primaryKey + sku: String @indexed + batch: String @indexed + name: String +} diff --git a/integrationTests/upgrade/cross-version-upgrade-visibility.test.ts b/integrationTests/upgrade/cross-version-upgrade-visibility.test.ts new file mode 100644 index 0000000000..16e1fd297b --- /dev/null +++ b/integrationTests/upgrade/cross-version-upgrade-visibility.test.ts @@ -0,0 +1,454 @@ +/** + * QA-promoted regression anchor — promoted from QA-658 (P-437), a gated qa-explorer finding. + * Originating QA scenario: QA-658 (source: gh#1865). + * Coverage anchored: a REAL 5.1.22 -> current cross-version upgrade boot keeps every + * pre-upgrade component-table row visible on all six read surfaces — the gh#1865 refutation. + * + * GENUINE cross-version upgrade-boot read visibility (5.1.x -> current). + * + * upgrade-boot-schema-change.test.ts (formerly QA-647) ran the *same-build restart* arm of + * #1865's investigation and found ZERO read-surface misses across 6 surfaces. That narrows + * #1865 but cannot refute it: the untested variable is a genuine cross-version on-disk/dictionary + * shape. This suite closes that gap: it seeds component-table rows under a REAL, npm-provisioned + * Harper 5.1.x release, stops it cleanly, then boots the CURRENT build + * (/home/kzyp/dev/harper/dist/bin/harper.js) over the SAME dataRootDir — the actual upgrade-boot + * scenario #1865 reports. + * + * The 6-surface read matrix, the base-store-scan oracle, and the heal/boot-order arms are reused + * verbatim from the same-build-restart arm (same fixture shape, same Resource surface set) so + * results are directly comparable between the two arms: + * - cached point-GET : in-process Resource `table.get(id)` (/PointGet/) + * - uncached scan : index-INDEPENDENT full base-store scan (/ScanAll/) — the ORACLE + * - ops search_by_value : secondary-index bulk scan via the legacy ops API + * - SQL : `operation: 'sql'` + * - REST : `GET /Widget/` + * - ops search_by_id : legacy ops API primary-key point lookup + * + * Also probes: + * - write-then-read heal: does re-putting a missed row's known original fields repair the + * cached point-GET surface? (/Touch/) + * - boot-order dependence: does a SECOND restart (current build, no further version change) + * clear a miss seen after the cross-version restart, or is the miss stable/growing? + * + * Provisioning: 5.1.x is installed via `npm install harper@5.1.22 --no-save` into an isolated + * temp npm prefix (~/dev/tmp/qa658-harper51), NOT via HARPER_LEGACY_VERSION_PATH (that env var is + * reserved for the 4.x-upgrade suite's convention of a pre-existing legacy install directory — + * see integrationTests/upgrade/4.x-upgrade.test.ts). The resolved 5.1.x `dist/bin/harper.js` is + * passed as `harperBinPath` to `setupHarperWithFixture`, exactly as 4.x-upgrade.test.ts passes + * `bin/harperdb.js` for its legacy arm. + * + * Every "restart" reuses setupHarperWithFixture/startHarper's own dataRootDir + hostname + * (ctx.harper is never reset), and re-passes the EXACT same `config` object used on the initial + * boot on every subsequent startHarper call — omitting it on a restart wipes config. + * + * Readiness after a restart is confirmed by polling a real route directly until it stops + * 404-ing — restartHttpWorkers() is NOT used, since it races against a pre-installed fixture and + * flakes on CI. + * + * Harper SHA under test: 07c2bbcb9 (main) — re-run confirmed clean at this SHA on 2026-07-22; + * originally authored at 77b46abf2 (an ancestor of 07c2bbcb9 on main), provisioning reused as-is. + * 5.1.x version installed: harper@5.1.22 (latest 5.1 release on npm as of this run). + * Repro: timeout 420 npm run test:integration -- "integrationTests/upgrade/cross-version-upgrade-visibility.test.ts" + */ +import { suite, test, before, after } from 'node:test'; +import { ok, strictEqual, deepStrictEqual } from 'node:assert'; +import { resolve } from 'node:path'; +import { existsSync } from 'node:fs'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { + startHarper, + setupHarperWithFixture, + killHarper, + teardownHarper, + sendOperation, + type ContextWithHarper, +} from '@harperfast/integration-testing'; +// @ts-expect-error utils/client.mjs has no type declarations; runtime resolves fine +import { createApiClient } from '../apiTests/utils/client.mjs'; + +const FIXTURE_PATH = resolve(import.meta.dirname, 'cross-version-upgrade-visibility'); +const DB = 'qa658'; +const TABLE = 'Widget'; +const ROW_COUNT = 24; +const BATCH = 'waveA'; + +// Provisioned by: cd ~/dev/tmp/qa658-harper51 && npm install harper@5.1.22 --no-save +// Override with HARPER_LEGACY_51_PATH if provisioned elsewhere. +const LEGACY_51_BIN_PATH = + process.env.HARPER_LEGACY_51_PATH ?? + resolve(process.env.HOME ?? '', 'dev/tmp/qa658-harper51/node_modules/harper/dist/bin/harper.js'); +const legacy51Available = existsSync(LEGACY_51_BIN_PATH); + +const skipSuite = process.platform === 'win32' || !legacy51Available; + +// Passed verbatim to setupHarperWithFixture AND to every subsequent startHarper call across every +// restart in this suite — never omitted, so a restart never silently wipes config. +const BOOT_CONFIG = { logging: { console: true, level: 'error' } }; + +interface Row { + id: string; + sku: string; + batch: string; + name: string; +} + +interface SurfaceMatrix { + label: string; + oracleIds: Set; + opsScanIds: Set; + cachedGetHits: string[]; + cachedGetMisses: string[]; + restHits: string[]; + restMisses: string[]; + opsByIdHits: string[]; + opsByIdMisses: string[]; + sqlHits: string[]; + sqlMisses: string[]; +} + +suite( + 'QA-658 cross-version (5.1.x -> current) upgrade-boot read visibility across surfaces (#1865)', + { skip: skipSuite }, + (ctx: ContextWithHarper) => { + let client: ReturnType; + let httpURL: string; + let rows: Row[] = []; + const findings: string[] = []; + function log(line: string) { + findings.push(line); + console.log(`[QA-658] ${line}`); + } + + function get(path: string): Promise { + return fetch(`${httpURL}${path}`, { headers: { Authorization: client.headers.Authorization } }); + } + function post(path: string, body: unknown): Promise { + return fetch(`${httpURL}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'Authorization': client.headers.Authorization }, + body: JSON.stringify(body), + }); + } + + async function pollReady(path: string, maxWaitMs = 60_000): Promise { + const deadline = Date.now() + maxWaitMs; + while (Date.now() < deadline) { + try { + const r = await get(path); + if (r.status !== 404) return; + } catch { + /* not up yet */ + } + await sleep(250); + } + throw new Error(`route ${path} still 404 after ${maxWaitMs}ms`); + } + + async function oracleScan(batch: string): Promise> { + const r = await get(`/ScanAll/?batch=${batch}`); + strictEqual(r.status, 200, `/ScanAll/ should return 200, got ${r.status}`); + const body = (await r.json()) as { ids: string[] }; + return new Set(body.ids); + } + + async function opsScan(batch: string): Promise> { + const res = await sendOperation(ctx.harper, { + operation: 'search_by_value', + schema: DB, + table: TABLE, + search_attribute: 'batch', + search_value: batch, + get_attributes: ['id'], + }); + return new Set((res as Array<{ id: string }>).map((r) => r.id)); + } + + async function cachedGet(id: string): Promise { + const r = await get(`/PointGet/?id=${id}`); + if (r.status !== 200) return false; + const body = (await r.json()) as { found: boolean }; + return body.found; + } + + async function restGet(id: string): Promise { + const r = await get(`/Widget/${id}`); + return r.status === 200; + } + + async function opsById(id: string): Promise { + const res = await sendOperation(ctx.harper, { + operation: 'search_by_id', + schema: DB, + table: TABLE, + ids: [id], + get_attributes: ['id'], + }); + return Array.isArray(res) && res.length === 1; + } + + async function sqlById(id: string): Promise { + const res = await sendOperation(ctx.harper, { + operation: 'sql', + sql: `SELECT id FROM ${DB}.${TABLE} WHERE id = '${id}'`, + }); + return Array.isArray(res) && res.length === 1; + } + + /** Build the full per-surface visibility matrix for `rows` against a live instance. */ + async function buildMatrix(label: string): Promise { + const oracleIds = await oracleScan(BATCH); + const opsScanIds = await opsScan(BATCH); + const m: SurfaceMatrix = { + label, + oracleIds, + opsScanIds, + cachedGetHits: [], + cachedGetMisses: [], + restHits: [], + restMisses: [], + opsByIdHits: [], + opsByIdMisses: [], + sqlHits: [], + sqlMisses: [], + }; + for (const row of rows) { + (await cachedGet(row.id)) ? m.cachedGetHits.push(row.id) : m.cachedGetMisses.push(row.id); + (await restGet(row.id)) ? m.restHits.push(row.id) : m.restMisses.push(row.id); + (await opsById(row.id)) ? m.opsByIdHits.push(row.id) : m.opsByIdMisses.push(row.id); + (await sqlById(row.id)) ? m.sqlHits.push(row.id) : m.sqlMisses.push(row.id); + } + log( + `${label}: oracle(scan)=${oracleIds.size}/${rows.length} opsScan=${opsScanIds.size}/${rows.length} ` + + `cachedGET=${m.cachedGetHits.length}/${rows.length} REST=${m.restHits.length}/${rows.length} ` + + `opsById=${m.opsByIdHits.length}/${rows.length} SQL=${m.sqlHits.length}/${rows.length}` + ); + if (m.cachedGetMisses.length) log(`${label}: cachedGET MISSES: ${m.cachedGetMisses.join(',')}`); + if (m.restMisses.length) log(`${label}: REST MISSES: ${m.restMisses.join(',')}`); + if (m.opsByIdMisses.length) log(`${label}: opsById MISSES: ${m.opsByIdMisses.join(',')}`); + if (m.sqlMisses.length) log(`${label}: SQL MISSES: ${m.sqlMisses.join(',')}`); + return m; + } + + async function refreshClient() { + client = createApiClient(ctx.harper); + httpURL = ctx.harper.httpURL; + } + + let matrixBaseline: SurfaceMatrix; + let matrixAfterCrossVerRestart: SurfaceMatrix; + let matrixAfterHeal: SurfaceMatrix | null = null; + let matrixAfterRestart2: SurfaceMatrix; + + before(async () => { + log(`harper SHA under test: 07c2bbcb9; legacy 5.1.x bin: ${LEGACY_51_BIN_PATH}`); + await setupHarperWithFixture(ctx, FIXTURE_PATH, { + config: BOOT_CONFIG, + env: { TC_AGREEMENT: 'yes' }, + harperBinPath: LEGACY_51_BIN_PATH, + }); + await refreshClient(); + await pollReady('/ScanAll/'); + log(`cold boot (5.1.x) complete at ${httpURL}, dataRootDir=${ctx.harper.dataRootDir}`); + + const loadRes = await post('/Load/', { count: ROW_COUNT, batch: BATCH }); + strictEqual(loadRes.status, 200, 'Load should return 200'); + const loadBody = (await loadRes.json()) as { ids: string[] }; + rows = loadBody.ids.map((id, i) => ({ + id, + sku: `SKU-${BATCH}-${i}`, + batch: BATCH, + name: `Widget ${BATCH} ${i}`, + })); + log(`loaded ${rows.length} rows under 5.1.x (batch=${BATCH})`); + }); + + after(async () => { + await teardownHarper(ctx); + log('--- FINDINGS SUMMARY ---'); + for (const line of findings) console.log(`[QA-658][summary] ${line}`); + }); + + test( + 'Q0 PRE-UPGRADE baseline (on 5.1.x): every surface sees every row (precondition)', + { timeout: 60_000 }, + async () => { + matrixBaseline = await buildMatrix('PRE-UPGRADE (5.1.x)'); + strictEqual( + matrixBaseline.oracleIds.size, + ROW_COUNT, + 'PRECONDITION: oracle scan must see all rows pre-upgrade' + ); + strictEqual( + matrixBaseline.opsScanIds.size, + ROW_COUNT, + 'PRECONDITION: ops search_by_value scan must see all rows pre-upgrade' + ); + strictEqual( + matrixBaseline.cachedGetHits.length, + ROW_COUNT, + 'PRECONDITION: cached point-GET must hit every row pre-upgrade' + ); + strictEqual(matrixBaseline.restHits.length, ROW_COUNT, 'PRECONDITION: REST must hit every row pre-upgrade'); + strictEqual( + matrixBaseline.opsByIdHits.length, + ROW_COUNT, + 'PRECONDITION: ops search_by_id must hit every row pre-upgrade' + ); + strictEqual(matrixBaseline.sqlHits.length, ROW_COUNT, 'PRECONDITION: SQL must hit every row pre-upgrade'); + } + ); + + test( + 'Q1 CROSS-VERSION UPGRADE BOOT: stop 5.1.x cleanly, start CURRENT build over the same dataRootDir', + { timeout: 90_000 }, + async () => { + await killHarper(ctx); + log('killed 5.1.x cleanly (SIGTERM)'); + + // No harperBinPath here -> resolves to the current build (dist/bin/harper.js) via the + // 'harper' package in node_modules, which in this repo checkout is the repo itself. + // Critical: re-pass the SAME config object used on the initial boot. + await startHarper(ctx, { config: BOOT_CONFIG, env: {} }); + await refreshClient(); + await pollReady('/ScanAll/'); + log(`cross-version upgrade boot (current build over 5.1.x data) complete at ${httpURL}`); + + matrixAfterCrossVerRestart = await buildMatrix('POST-CROSSVER-UPGRADE'); + + // Oracle must hold regardless of what any other surface reports — if this fails, rows + // were actually lost (a much larger problem than a read-visibility miss), and everything + // below is moot. + strictEqual( + matrixAfterCrossVerRestart.oracleIds.size, + ROW_COUNT, + `ORACLE: uncached index-independent full scan must still see all ${ROW_COUNT} rows after the cross-version upgrade boot (proves data is on disk)` + ); + deepStrictEqual( + [...matrixAfterCrossVerRestart.oracleIds].sort(), + rows.map((r) => r.id).sort(), + 'ORACLE: post-upgrade scan id set must exactly match the pre-upgrade row set' + ); + } + ); + + test('Q2 POST-CROSSVER-UPGRADE: does the cached point-GET / SQL / REST / ops surface miss what the oracle sees?', () => { + const m = matrixAfterCrossVerRestart; + strictEqual( + m.opsScanIds.size, + ROW_COUNT, + `[#1865 CHECK] ops search_by_value(batch) scan expected ${ROW_COUNT}, got ${m.opsScanIds.size} — secondary-index scan miss` + ); + strictEqual( + m.cachedGetHits.length, + ROW_COUNT, + `[#1865 CHECK] cached point-GET expected ${ROW_COUNT} hits, got ${m.cachedGetHits.length} (misses: ${m.cachedGetMisses.join(',')})` + ); + strictEqual( + m.restHits.length, + ROW_COUNT, + `[#1865 CHECK] REST GET /Widget/ expected ${ROW_COUNT} hits, got ${m.restHits.length} (misses: ${m.restMisses.join(',')})` + ); + strictEqual( + m.opsByIdHits.length, + ROW_COUNT, + `[#1865 CHECK] ops search_by_id expected ${ROW_COUNT} hits, got ${m.opsByIdHits.length} (misses: ${m.opsByIdMisses.join(',')})` + ); + strictEqual( + m.sqlHits.length, + ROW_COUNT, + `[#1865 CHECK] SQL expected ${ROW_COUNT} hits, got ${m.sqlHits.length} (misses: ${m.sqlMisses.join(',')})` + ); + }); + + test('Q3 write-then-read heal: does re-putting a missed row repair it?', { timeout: 30_000 }, async (t) => { + const anyMisses = + matrixAfterCrossVerRestart.cachedGetMisses.length || + matrixAfterCrossVerRestart.restMisses.length || + matrixAfterCrossVerRestart.opsByIdMisses.length || + matrixAfterCrossVerRestart.sqlMisses.length; + if (!anyMisses) { + t.skip('no miss observed after the cross-version upgrade boot on any point surface — nothing to heal'); + return; + } + // Prefer a row missed on cached point-GET (the surface #1865 names); fall back to any miss. + const targetId = + matrixAfterCrossVerRestart.cachedGetMisses[0] ?? + matrixAfterCrossVerRestart.restMisses[0] ?? + matrixAfterCrossVerRestart.opsByIdMisses[0] ?? + matrixAfterCrossVerRestart.sqlMisses[0]; + const row = rows.find((r) => r.id === targetId)!; + log(`Q3: healing probe targets ${targetId} (missed pre-heal)`); + + const beforeHeal = await cachedGet(targetId); + const r = await post('/Touch/', row); + strictEqual(r.status, 200, 'Touch should return 200'); + const touchBody = (await r.json()) as { foundAfterTouch: boolean }; + const afterHeal = await cachedGet(targetId); + log( + `Q3: ${targetId} cachedGET before-write=${beforeHeal}, Touch.foundAfterTouch=${touchBody.foundAfterTouch}, ` + + `cachedGET after-write=${afterHeal} -> ${afterHeal ? 'WRITE HEALS the miss' : 'STILL MISSING after the write'}` + ); + matrixAfterHeal = await buildMatrix('POST-HEAL'); + ok(true, `heal probe recorded (before=${beforeHeal}, after=${afterHeal})`); + }); + + test( + 'Q4 RESTART #2 (current build, no further version change): is the miss stable, cleared, or does it grow?', + { timeout: 90_000 }, + async () => { + await killHarper(ctx); + log('killed Harper cleanly (SIGTERM) for restart #2 (same-build, boot-order check)'); + + // Same config again — no further version change this time (isolates "does a SECOND + // boot on the current build alone change anything" from "does the cross-version boot"). + await startHarper(ctx, { config: BOOT_CONFIG, env: {} }); + await refreshClient(); + await pollReady('/ScanAll/'); + log(`restart #2 (same-build) complete at ${httpURL}`); + + matrixAfterRestart2 = await buildMatrix('POST-RESTART-2 (same-build)'); + + strictEqual( + matrixAfterRestart2.oracleIds.size, + ROW_COUNT, + `ORACLE: uncached full scan must still see all ${ROW_COUNT} rows after the SECOND restart` + ); + + const cmp = (a: string[], b: string[], name: string) => { + log( + `Q4 boot-order check (${name}): after-crossver-upgrade misses=${b.length}, ` + + `after-restart#2 misses=${a.length} -> ${ + a.length === 0 && b.length > 0 + ? 'CLEARED on 2nd (same-build) boot (order-dependent)' + : a.length > 0 && b.length === 0 + ? 'NEW MISS introduced by 2nd boot alone' + : a.length === b.length + ? 'UNCHANGED across boots (stable)' + : 'CHANGED (different miss set/count)' + }` + ); + }; + cmp(matrixAfterRestart2.cachedGetMisses, matrixAfterCrossVerRestart.cachedGetMisses, 'cachedGetMisses'); + cmp(matrixAfterRestart2.restMisses, matrixAfterCrossVerRestart.restMisses, 'restMisses'); + cmp(matrixAfterRestart2.opsByIdMisses, matrixAfterCrossVerRestart.opsByIdMisses, 'opsByIdMisses'); + cmp(matrixAfterRestart2.sqlMisses, matrixAfterCrossVerRestart.sqlMisses, 'sqlMisses'); + + log( + `FINAL VISIBILITY MATRIX SUMMARY:\n` + + ` PRE-UPGRADE (5.1.x) : oracle=${matrixBaseline.oracleIds.size}/${ROW_COUNT} opsScan=${matrixBaseline.opsScanIds.size}/${ROW_COUNT} ` + + `cachedGET=${matrixBaseline.cachedGetHits.length}/${ROW_COUNT} REST=${matrixBaseline.restHits.length}/${ROW_COUNT} ` + + `opsById=${matrixBaseline.opsByIdHits.length}/${ROW_COUNT} SQL=${matrixBaseline.sqlHits.length}/${ROW_COUNT}\n` + + ` POST-CROSSVER-UPGRADE : oracle=${matrixAfterCrossVerRestart.oracleIds.size}/${ROW_COUNT} opsScan=${matrixAfterCrossVerRestart.opsScanIds.size}/${ROW_COUNT} ` + + `cachedGET=${matrixAfterCrossVerRestart.cachedGetHits.length}/${ROW_COUNT} REST=${matrixAfterCrossVerRestart.restHits.length}/${ROW_COUNT} ` + + `opsById=${matrixAfterCrossVerRestart.opsByIdHits.length}/${ROW_COUNT} SQL=${matrixAfterCrossVerRestart.sqlHits.length}/${ROW_COUNT}\n` + + (matrixAfterHeal + ? ` POST-HEAL : oracle=${matrixAfterHeal.oracleIds.size}/${ROW_COUNT} cachedGET=${matrixAfterHeal.cachedGetHits.length}/${ROW_COUNT} REST=${matrixAfterHeal.restHits.length}/${ROW_COUNT} opsById=${matrixAfterHeal.opsByIdHits.length}/${ROW_COUNT} SQL=${matrixAfterHeal.sqlHits.length}/${ROW_COUNT}\n` + : '') + + ` POST-RESTART-2 : oracle=${matrixAfterRestart2.oracleIds.size}/${ROW_COUNT} opsScan=${matrixAfterRestart2.opsScanIds.size}/${ROW_COUNT} ` + + `cachedGET=${matrixAfterRestart2.cachedGetHits.length}/${ROW_COUNT} REST=${matrixAfterRestart2.restHits.length}/${ROW_COUNT} ` + + `opsById=${matrixAfterRestart2.opsByIdHits.length}/${ROW_COUNT} SQL=${matrixAfterRestart2.sqlHits.length}/${ROW_COUNT}` + ); + } + ); + } +); diff --git a/integrationTests/upgrade/cross-version-upgrade-visibility/config.yaml b/integrationTests/upgrade/cross-version-upgrade-visibility/config.yaml new file mode 100644 index 0000000000..efffc0833f --- /dev/null +++ b/integrationTests/upgrade/cross-version-upgrade-visibility/config.yaml @@ -0,0 +1,5 @@ +graphqlSchema: + files: '*.graphql' +jsResource: + files: resources.js +rest: true diff --git a/integrationTests/upgrade/cross-version-upgrade-visibility/resources.js b/integrationTests/upgrade/cross-version-upgrade-visibility/resources.js new file mode 100644 index 0000000000..f339425945 --- /dev/null +++ b/integrationTests/upgrade/cross-version-upgrade-visibility/resources.js @@ -0,0 +1,74 @@ +// QA-658 (#1865) — cross-version upgrade-boot read-visibility fixture. +// +// Same shape as QA-647's fixture (integrationTests/qa-scratch/qa647-upgrade-boot-read/), so the +// read-surface matrix is directly comparable. Widget is a component-defined table +// (schema.graphql), matching #1865's specific claim that the miss is on COMPONENT tables. +// +// POST /Load/ { count, batch } — seed rows via the in-process write path. +// GET /PointGet/?id=X — cached point-GET: in-process table.get(id). +// GET /ScanAll/[?batch=X] — uncached, INDEX-INDEPENDENT full base-store scan. +// The oracle: proves a row is physically on disk regardless of any cache/secondary-index +// state. +// POST /Touch/ { id, sku, batch, name } — write-then-read heal probe. + +function getTable() { + return databases['qa658']['Widget']; +} + +function qp(query, name, def) { + const v = query && (query.get ? query.get(name) : query[name]); + return v == null ? def : v; +} + +export class Load extends Resource { + static loadAsInstance = false; + async post(query, body) { + const b = body || query || {}; + const count = Number(b.count) || 0; + const batch = b.batch || 'default'; + const table = getTable(); + const ids = []; + for (let i = 0; i < count; i++) { + const id = `w-${batch}-${i}`; + await table.put({ id, sku: `SKU-${batch}-${i}`, batch, name: `Widget ${batch} ${i}` }); + ids.push(id); + } + return { ok: true, count, batch, ids }; + } +} + +export class PointGet extends Resource { + static loadAsInstance = false; + async get(query) { + const id = qp(query, 'id'); + const table = getTable(); + const rec = await table.get(id); + return { id, found: rec != null, record: rec ?? null }; + } +} + +export class ScanAll extends Resource { + static loadAsInstance = false; + async get(query) { + const batch = qp(query, 'batch', null); + const table = getTable(); + const ids = []; + for await (const r of table.search({ conditions: [] })) { + if (batch == null || r.batch === batch) ids.push(r.id); + } + return { count: ids.length, ids: ids.sort() }; + } +} + +export class Touch extends Resource { + static loadAsInstance = false; + async post(query, body) { + const b = body || query || {}; + const { id, sku, batch, name } = b; + if (!id) throw new Error('Touch requires id'); + const table = getTable(); + await table.put({ id, sku, batch, name, touched: Date.now() }); + const after = await table.get(id); + return { ok: true, id, foundAfterTouch: after != null }; + } +} diff --git a/integrationTests/upgrade/cross-version-upgrade-visibility/schema.graphql b/integrationTests/upgrade/cross-version-upgrade-visibility/schema.graphql new file mode 100644 index 0000000000..1bdfdb382f --- /dev/null +++ b/integrationTests/upgrade/cross-version-upgrade-visibility/schema.graphql @@ -0,0 +1,10 @@ +# QA-658 (#1865) — component-defined table used across a GENUINE cross-version upgrade +# boot: seeded under a real Harper 5.1.x release, then restarted with the current build +# over the SAME dataRootDir. Shape matches QA-647's fixture (a prior same-build-restart arm +# of the same investigation), so results are directly comparable. +type Widget @table(database: "qa658") @export { + id: ID @primaryKey + sku: String @indexed + batch: String @indexed + name: String +} diff --git a/integrationTests/upgrade/upgrade-boot-schema-change.test.ts b/integrationTests/upgrade/upgrade-boot-schema-change.test.ts new file mode 100644 index 0000000000..6548eab574 --- /dev/null +++ b/integrationTests/upgrade/upgrade-boot-schema-change.test.ts @@ -0,0 +1,433 @@ +/** + * QA-promoted regression anchor — promoted from QA-647 (P-432), a gated qa-explorer finding. + * Originating QA scenario: QA-647 (source: gh#1865). + * Coverage anchored: restart + attribute-adding schema change never makes existing rows + * invisible on any read surface (6-surface matrix, two restarts). + * + * Upgrade-boot read visibility across every read surface. + * + * #1865 reports that pre-restart rows in COMPONENT tables can go missing on some read surfaces + * after an in-place upgrade boot (same dataRootDir, process restarts, config may be backfilled), + * while other surfaces still see them. This probe writes rows into a component table, restarts + * the CURRENT build over the SAME dataRootDir (simulating the in-place upgrade), and re-reads + * those rows through every distinct read surface: + * - cached point-GET : in-process Resource `table.get(id)` (/PointGet/) + * - uncached scan : index-INDEPENDENT full base-store scan (/ScanAll/) — the ORACLE, + * since a cached/indexed surface must never judge itself + * - ops search_by_value : secondary-index bulk scan via the legacy ops API (search_by_value) + * - SQL : `operation: 'sql'` + * - REST : `GET /Widget/` + * - ops search_by_id : legacy ops API primary-key point lookup + * + * Also probes: + * - write-then-read heal: does re-putting a missed row's known original fields repair the + * cached point-GET surface? (/Touch/) + * - boot-order dependence: does a SECOND restart (no further config change) clear a miss seen + * after the first restart, or is the miss stable/growing? + * - config change across the boot: between restart #1's kill and start, the on-disk + * schema.graphql is overwritten to ADD a table attribute (`category`) to the SAME component, + * matching #1865's report that this reproduces on component tables specifically. + * + * Every "restart" reuses setupHarperWithFixture/startHarper's own dataRootDir + hostname + * (ctx.harper is never reset), and re-passes the EXACT same `config` object used on the initial + * boot on every subsequent startHarper call — omitting it on a restart wipes config and would + * invalidate the experiment (per the investigation brief for this suite). + * + * Per eviction-secondary-index.test.ts's pattern: readiness after a restart is confirmed by + * polling a real route directly until it stops 404-ing — restartHttpWorkers() is NOT used here, + * since it races against a pre-installed fixture and flakes on CI. + * + * Harper SHA under test: 1e1edc666 (main). + * Repro: timeout 900 npm run test:integration -- "integrationTests/upgrade/upgrade-boot-schema-change.test.ts" + */ +import { suite, test, before, after } from 'node:test'; +import { ok, strictEqual, deepStrictEqual } from 'node:assert'; +import { resolve, join, basename } from 'node:path'; +import { writeFile } from 'node:fs/promises'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { + startHarper, + setupHarperWithFixture, + killHarper, + teardownHarper, + sendOperation, + type ContextWithHarper, +} from '@harperfast/integration-testing'; +// @ts-expect-error utils/client.mjs has no type declarations; runtime resolves fine +import { createApiClient } from '../apiTests/utils/client.mjs'; + +const FIXTURE_PATH = resolve(import.meta.dirname, 'upgrade-boot-schema-change'); +const COMPONENT_DIR_NAME = basename(FIXTURE_PATH); +const DB = 'qa647'; +const TABLE = 'Widget'; +const ROW_COUNT = 24; +const BATCH = 'waveA'; +const skipSuite = process.platform === 'win32'; + +// Passed verbatim to setupHarperWithFixture AND to every subsequent startHarper call across every +// restart in this suite — never omitted, so a restart never silently wipes config. +const BOOT_CONFIG = { logging: { console: true, level: 'error' } }; + +// v2 schema written to disk between restart #1's kill and start: adds a table attribute +// (`category`) to the SAME component, on the SAME dataRootDir, simulating a config/schema change +// riding along with an in-place upgrade boot. +const SCHEMA_V2 = `# QA-647 v2 — category attribute added across restart #1's boot. +type Widget @table(database: "qa647") @export { + id: ID @primaryKey + sku: String @indexed + batch: String @indexed + name: String + category: String @indexed +} +`; + +interface Row { + id: string; + sku: string; + batch: string; + name: string; +} + +interface SurfaceMatrix { + label: string; + oracleIds: Set; + opsScanIds: Set; + cachedGetHits: string[]; + cachedGetMisses: string[]; + restHits: string[]; + restMisses: string[]; + opsByIdHits: string[]; + opsByIdMisses: string[]; + sqlHits: string[]; + sqlMisses: string[]; +} + +suite('QA-647 upgrade-boot read visibility across surfaces (#1865)', { skip: skipSuite }, (ctx: ContextWithHarper) => { + let client: ReturnType; + let httpURL: string; + let rows: Row[] = []; + const findings: string[] = []; + function log(line: string) { + findings.push(line); + console.log(`[QA-647] ${line}`); + } + + function get(path: string): Promise { + return fetch(`${httpURL}${path}`, { headers: { Authorization: client.headers.Authorization } }); + } + function post(path: string, body: unknown): Promise { + return fetch(`${httpURL}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'Authorization': client.headers.Authorization }, + body: JSON.stringify(body), + }); + } + + async function pollReady(path: string, maxWaitMs = 60_000): Promise { + const deadline = Date.now() + maxWaitMs; + while (Date.now() < deadline) { + try { + const r = await get(path); + if (r.status !== 404) return; + } catch { + /* not up yet */ + } + await sleep(250); + } + throw new Error(`route ${path} still 404 after ${maxWaitMs}ms`); + } + + async function oracleScan(batch: string): Promise> { + const r = await get(`/ScanAll/?batch=${batch}`); + strictEqual(r.status, 200, `/ScanAll/ should return 200, got ${r.status}`); + const body = (await r.json()) as { ids: string[] }; + return new Set(body.ids); + } + + async function opsScan(batch: string): Promise> { + const res = await sendOperation(ctx.harper, { + operation: 'search_by_value', + schema: DB, + table: TABLE, + search_attribute: 'batch', + search_value: batch, + get_attributes: ['id'], + }); + return new Set((res as Array<{ id: string }>).map((r) => r.id)); + } + + async function cachedGet(id: string): Promise { + const r = await get(`/PointGet/?id=${id}`); + if (r.status !== 200) return false; + const body = (await r.json()) as { found: boolean }; + return body.found; + } + + async function restGet(id: string): Promise { + const r = await get(`/Widget/${id}`); + return r.status === 200; + } + + async function opsById(id: string): Promise { + const res = await sendOperation(ctx.harper, { + operation: 'search_by_id', + schema: DB, + table: TABLE, + ids: [id], + get_attributes: ['id'], + }); + return Array.isArray(res) && res.length === 1; + } + + async function sqlById(id: string): Promise { + const res = await sendOperation(ctx.harper, { + operation: 'sql', + sql: `SELECT id FROM ${DB}.${TABLE} WHERE id = '${id}'`, + }); + return Array.isArray(res) && res.length === 1; + } + + /** Build the full per-surface visibility matrix for `rows` against a live instance. */ + async function buildMatrix(label: string): Promise { + const oracleIds = await oracleScan(BATCH); + const opsScanIds = await opsScan(BATCH); + const m: SurfaceMatrix = { + label, + oracleIds, + opsScanIds, + cachedGetHits: [], + cachedGetMisses: [], + restHits: [], + restMisses: [], + opsByIdHits: [], + opsByIdMisses: [], + sqlHits: [], + sqlMisses: [], + }; + for (const row of rows) { + (await cachedGet(row.id)) ? m.cachedGetHits.push(row.id) : m.cachedGetMisses.push(row.id); + (await restGet(row.id)) ? m.restHits.push(row.id) : m.restMisses.push(row.id); + (await opsById(row.id)) ? m.opsByIdHits.push(row.id) : m.opsByIdMisses.push(row.id); + (await sqlById(row.id)) ? m.sqlHits.push(row.id) : m.sqlMisses.push(row.id); + } + log( + `${label}: oracle(scan)=${oracleIds.size}/${rows.length} opsScan=${opsScanIds.size}/${rows.length} ` + + `cachedGET=${m.cachedGetHits.length}/${rows.length} REST=${m.restHits.length}/${rows.length} ` + + `opsById=${m.opsByIdHits.length}/${rows.length} SQL=${m.sqlHits.length}/${rows.length}` + ); + if (m.cachedGetMisses.length) log(`${label}: cachedGET MISSES: ${m.cachedGetMisses.join(',')}`); + if (m.restMisses.length) log(`${label}: REST MISSES: ${m.restMisses.join(',')}`); + if (m.opsByIdMisses.length) log(`${label}: opsById MISSES: ${m.opsByIdMisses.join(',')}`); + if (m.sqlMisses.length) log(`${label}: SQL MISSES: ${m.sqlMisses.join(',')}`); + return m; + } + + async function refreshClient() { + client = createApiClient(ctx.harper); + httpURL = ctx.harper.httpURL; + } + + let matrixBaseline: SurfaceMatrix; + let matrixAfterRestart1: SurfaceMatrix; + let matrixAfterHeal: SurfaceMatrix | null = null; + let matrixAfterRestart2: SurfaceMatrix; + + before(async () => { + await setupHarperWithFixture(ctx, FIXTURE_PATH, { config: BOOT_CONFIG, env: {} }); + await refreshClient(); + await pollReady('/ScanAll/'); + log(`cold boot complete at ${httpURL}, dataRootDir=${ctx.harper.dataRootDir}`); + + const loadRes = await post('/Load/', { count: ROW_COUNT, batch: BATCH }); + strictEqual(loadRes.status, 200, 'Load should return 200'); + const loadBody = (await loadRes.json()) as { ids: string[] }; + rows = loadBody.ids.map((id, i) => ({ id, sku: `SKU-${BATCH}-${i}`, batch: BATCH, name: `Widget ${BATCH} ${i}` })); + log(`loaded ${rows.length} rows (batch=${BATCH})`); + }); + + after(async () => { + await teardownHarper(ctx); + log('--- FINDINGS SUMMARY ---'); + for (const line of findings) console.log(`[QA-647][summary] ${line}`); + }); + + test('Q0 PRE-RESTART baseline: every surface sees every row (precondition)', { timeout: 60_000 }, async () => { + matrixBaseline = await buildMatrix('PRE-RESTART'); + strictEqual(matrixBaseline.oracleIds.size, ROW_COUNT, 'PRECONDITION: oracle scan must see all rows pre-restart'); + strictEqual( + matrixBaseline.opsScanIds.size, + ROW_COUNT, + 'PRECONDITION: ops search_by_value scan must see all rows pre-restart' + ); + strictEqual( + matrixBaseline.cachedGetHits.length, + ROW_COUNT, + 'PRECONDITION: cached point-GET must hit every row pre-restart' + ); + strictEqual(matrixBaseline.restHits.length, ROW_COUNT, 'PRECONDITION: REST must hit every row pre-restart'); + strictEqual( + matrixBaseline.opsByIdHits.length, + ROW_COUNT, + 'PRECONDITION: ops search_by_id must hit every row pre-restart' + ); + strictEqual(matrixBaseline.sqlHits.length, ROW_COUNT, 'PRECONDITION: SQL must hit every row pre-restart'); + }); + + test( + 'Q1 RESTART #1 (+ config change: schema gains a table attribute across the boot)', + { timeout: 90_000 }, + async () => { + // Config change across the boot: mutate the on-disk component schema (adds `category`) + // while Harper is down, same dataRootDir, same component directory. + const schemaPath = join(ctx.harper.dataRootDir, 'components', COMPONENT_DIR_NAME, 'schema.graphql'); + await writeFile(schemaPath, SCHEMA_V2, 'utf8'); + log(`rewrote ${schemaPath} to v2 (adds category:String @indexed) while Harper is down`); + + await killHarper(ctx); + log('killed Harper cleanly (SIGTERM)'); + + // Critical: re-pass the SAME config object used on the initial boot. + await startHarper(ctx, { config: BOOT_CONFIG, env: {} }); + await refreshClient(); + await pollReady('/ScanAll/'); + log(`restart #1 (upgrade-boot simulation) complete at ${httpURL}`); + + matrixAfterRestart1 = await buildMatrix('POST-RESTART-1'); + + // Oracle must hold regardless of what any other surface reports — if this fails, rows + // were actually lost (a much larger problem than a read-visibility miss), and everything + // below is moot. + strictEqual( + matrixAfterRestart1.oracleIds.size, + ROW_COUNT, + `ORACLE: uncached index-independent full scan must still see all ${ROW_COUNT} rows after the upgrade-boot restart (proves data is on disk)` + ); + deepStrictEqual( + [...matrixAfterRestart1.oracleIds].sort(), + rows.map((r) => r.id).sort(), + 'ORACLE: post-restart scan id set must exactly match the pre-restart row set' + ); + } + ); + + test('Q2 POST-RESTART-1: does the cached point-GET / SQL / REST / ops surface miss what the oracle sees?', () => { + const m = matrixAfterRestart1; + strictEqual( + m.opsScanIds.size, + ROW_COUNT, + `[#1865 CHECK] ops search_by_value(batch) scan expected ${ROW_COUNT}, got ${m.opsScanIds.size} — secondary-index scan miss` + ); + strictEqual( + m.cachedGetHits.length, + ROW_COUNT, + `[#1865 CHECK] cached point-GET expected ${ROW_COUNT} hits, got ${m.cachedGetHits.length} (misses: ${m.cachedGetMisses.join(',')})` + ); + strictEqual( + m.restHits.length, + ROW_COUNT, + `[#1865 CHECK] REST GET /Widget/ expected ${ROW_COUNT} hits, got ${m.restHits.length} (misses: ${m.restMisses.join(',')})` + ); + strictEqual( + m.opsByIdHits.length, + ROW_COUNT, + `[#1865 CHECK] ops search_by_id expected ${ROW_COUNT} hits, got ${m.opsByIdHits.length} (misses: ${m.opsByIdMisses.join(',')})` + ); + strictEqual( + m.sqlHits.length, + ROW_COUNT, + `[#1865 CHECK] SQL expected ${ROW_COUNT} hits, got ${m.sqlHits.length} (misses: ${m.sqlMisses.join(',')})` + ); + }); + + test('Q3 write-then-read heal: does re-putting a missed row repair it?', { timeout: 30_000 }, async (t) => { + const anyMisses = + matrixAfterRestart1.cachedGetMisses.length || + matrixAfterRestart1.restMisses.length || + matrixAfterRestart1.opsByIdMisses.length || + matrixAfterRestart1.sqlMisses.length; + if (!anyMisses) { + t.skip('no miss observed after restart #1 on any point surface — nothing to heal'); + return; + } + // Prefer a row missed on cached point-GET (the surface #1865 names); fall back to any miss. + const targetId = + matrixAfterRestart1.cachedGetMisses[0] ?? + matrixAfterRestart1.restMisses[0] ?? + matrixAfterRestart1.opsByIdMisses[0] ?? + matrixAfterRestart1.sqlMisses[0]; + const row = rows.find((r) => r.id === targetId)!; + log(`Q3: healing probe targets ${targetId} (missed pre-heal)`); + + const beforeHeal = await cachedGet(targetId); + const r = await post('/Touch/', row); + strictEqual(r.status, 200, 'Touch should return 200'); + const touchBody = (await r.json()) as { foundAfterTouch: boolean }; + const afterHeal = await cachedGet(targetId); + log( + `Q3: ${targetId} cachedGET before-write=${beforeHeal}, Touch.foundAfterTouch=${touchBody.foundAfterTouch}, ` + + `cachedGET after-write=${afterHeal} -> ${afterHeal ? 'WRITE HEALS the miss' : 'STILL MISSING after the write'}` + ); + matrixAfterHeal = await buildMatrix('POST-HEAL'); + ok(true, `heal probe recorded (before=${beforeHeal}, after=${afterHeal})`); + }); + + test( + 'Q4 RESTART #2 (no further config change): is the miss stable, cleared, or does it grow?', + { timeout: 90_000 }, + async () => { + await killHarper(ctx); + log('killed Harper cleanly (SIGTERM) for restart #2'); + + // Same config again — no further schema mutation this time (isolates "does a SECOND boot + // alone change anything" from "does another config change"). + await startHarper(ctx, { config: BOOT_CONFIG, env: {} }); + await refreshClient(); + await pollReady('/ScanAll/'); + log(`restart #2 complete at ${httpURL}`); + + matrixAfterRestart2 = await buildMatrix('POST-RESTART-2'); + + strictEqual( + matrixAfterRestart2.oracleIds.size, + ROW_COUNT, + `ORACLE: uncached full scan must still see all ${ROW_COUNT} rows after the SECOND restart` + ); + + const cmp = (a: string[], b: string[], name: string) => { + const wasOk = matrixAfterRestart1[name as keyof SurfaceMatrix] as unknown as string[]; + log( + `Q4 boot-order check (${name}): restart#1 misses=${(wasOk as string[]).length}, ` + + `restart#2 misses=${a.length} -> ${ + a.length === 0 && (wasOk as string[]).length > 0 + ? 'CLEARED on 2nd boot (order-dependent)' + : a.length > 0 && (wasOk as string[]).length === 0 + ? 'NEW MISS introduced by 2nd boot alone' + : a.length === (wasOk as string[]).length + ? 'UNCHANGED across boots (stable)' + : 'CHANGED (different miss set/count)' + }` + ); + }; + cmp(matrixAfterRestart2.cachedGetMisses, matrixAfterRestart1.cachedGetMisses, 'cachedGetMisses'); + cmp(matrixAfterRestart2.restMisses, matrixAfterRestart1.restMisses, 'restMisses'); + cmp(matrixAfterRestart2.opsByIdMisses, matrixAfterRestart1.opsByIdMisses, 'opsByIdMisses'); + cmp(matrixAfterRestart2.sqlMisses, matrixAfterRestart1.sqlMisses, 'sqlMisses'); + + log( + `FINAL VISIBILITY MATRIX SUMMARY:\n` + + ` PRE-RESTART : oracle=${matrixBaseline.oracleIds.size}/${ROW_COUNT} opsScan=${matrixBaseline.opsScanIds.size}/${ROW_COUNT} ` + + `cachedGET=${matrixBaseline.cachedGetHits.length}/${ROW_COUNT} REST=${matrixBaseline.restHits.length}/${ROW_COUNT} ` + + `opsById=${matrixBaseline.opsByIdHits.length}/${ROW_COUNT} SQL=${matrixBaseline.sqlHits.length}/${ROW_COUNT}\n` + + ` POST-RESTART-1 : oracle=${matrixAfterRestart1.oracleIds.size}/${ROW_COUNT} opsScan=${matrixAfterRestart1.opsScanIds.size}/${ROW_COUNT} ` + + `cachedGET=${matrixAfterRestart1.cachedGetHits.length}/${ROW_COUNT} REST=${matrixAfterRestart1.restHits.length}/${ROW_COUNT} ` + + `opsById=${matrixAfterRestart1.opsByIdHits.length}/${ROW_COUNT} SQL=${matrixAfterRestart1.sqlHits.length}/${ROW_COUNT}\n` + + (matrixAfterHeal + ? ` POST-HEAL : oracle=${matrixAfterHeal.oracleIds.size}/${ROW_COUNT} cachedGET=${matrixAfterHeal.cachedGetHits.length}/${ROW_COUNT} REST=${matrixAfterHeal.restHits.length}/${ROW_COUNT} opsById=${matrixAfterHeal.opsByIdHits.length}/${ROW_COUNT} SQL=${matrixAfterHeal.sqlHits.length}/${ROW_COUNT}\n` + : '') + + ` POST-RESTART-2 : oracle=${matrixAfterRestart2.oracleIds.size}/${ROW_COUNT} opsScan=${matrixAfterRestart2.opsScanIds.size}/${ROW_COUNT} ` + + `cachedGET=${matrixAfterRestart2.cachedGetHits.length}/${ROW_COUNT} REST=${matrixAfterRestart2.restHits.length}/${ROW_COUNT} ` + + `opsById=${matrixAfterRestart2.opsByIdHits.length}/${ROW_COUNT} SQL=${matrixAfterRestart2.sqlHits.length}/${ROW_COUNT}` + ); + } + ); +}); diff --git a/integrationTests/upgrade/upgrade-boot-schema-change/config.yaml b/integrationTests/upgrade/upgrade-boot-schema-change/config.yaml new file mode 100644 index 0000000000..efffc0833f --- /dev/null +++ b/integrationTests/upgrade/upgrade-boot-schema-change/config.yaml @@ -0,0 +1,5 @@ +graphqlSchema: + files: '*.graphql' +jsResource: + files: resources.js +rest: true diff --git a/integrationTests/upgrade/upgrade-boot-schema-change/resources.js b/integrationTests/upgrade/upgrade-boot-schema-change/resources.js new file mode 100644 index 0000000000..a8eb06e67d --- /dev/null +++ b/integrationTests/upgrade/upgrade-boot-schema-change/resources.js @@ -0,0 +1,79 @@ +// QA-647 (source: gh:1865) — upgrade-boot read-visibility fixture. +// +// Widget is a component-defined table (schema.graphql), matching #1865's specific claim that +// the miss is on COMPONENT tables. Endpoints below exercise the read surfaces the investigation +// needs, in-process, so a restart-induced miss on one surface can be checked against an +// independent oracle rather than judged by itself. +// +// POST /Load/ { count, batch } — seed rows via the in-process write path. +// GET /PointGet/?id=X — cached point-GET: in-process table.get(id), the +// Resource/table-level primary-key lookup — the surface #1865 reports missing rows on. +// GET /ScanAll/[?batch=X] — uncached, INDEX-INDEPENDENT full base-store scan. +// The oracle: proves a row is physically on disk regardless of any cache or secondary-index +// state. Do not let PointGet or the REST/ops surfaces judge themselves — always compare +// against this. +// POST /Touch/ { id, sku, batch, name } — write-then-read heal probe: re-put the row with its +// known original fields (works even if the row currently reads as missing), then re-run +// PointGet, so a single call answers "does a write repair the miss". + +function getTable() { + return databases['qa647']['Widget']; +} + +function qp(query, name, def) { + const v = query && (query.get ? query.get(name) : query[name]); + return v == null ? def : v; +} + +export class Load extends Resource { + static loadAsInstance = false; + async post(query, body) { + const b = body || query || {}; + const count = Number(b.count) || 0; + const batch = b.batch || 'default'; + const table = getTable(); + const ids = []; + for (let i = 0; i < count; i++) { + const id = `w-${batch}-${i}`; + await table.put({ id, sku: `SKU-${batch}-${i}`, batch, name: `Widget ${batch} ${i}` }); + ids.push(id); + } + return { ok: true, count, batch, ids }; + } +} + +export class PointGet extends Resource { + static loadAsInstance = false; + async get(query) { + const id = qp(query, 'id'); + const table = getTable(); + const rec = await table.get(id); + return { id, found: rec != null, record: rec ?? null }; + } +} + +export class ScanAll extends Resource { + static loadAsInstance = false; + async get(query) { + const batch = qp(query, 'batch', null); + const table = getTable(); + const ids = []; + for await (const r of table.search({ conditions: [] })) { + if (batch == null || r.batch === batch) ids.push(r.id); + } + return { count: ids.length, ids: ids.sort() }; + } +} + +export class Touch extends Resource { + static loadAsInstance = false; + async post(query, body) { + const b = body || query || {}; + const { id, sku, batch, name } = b; + if (!id) throw new Error('Touch requires id'); + const table = getTable(); + await table.put({ id, sku, batch, name, touched: Date.now() }); + const after = await table.get(id); + return { ok: true, id, foundAfterTouch: after != null }; + } +} diff --git a/integrationTests/upgrade/upgrade-boot-schema-change/schema.graphql b/integrationTests/upgrade/upgrade-boot-schema-change/schema.graphql new file mode 100644 index 0000000000..3889a2a4eb --- /dev/null +++ b/integrationTests/upgrade/upgrade-boot-schema-change/schema.graphql @@ -0,0 +1,10 @@ +# QA-647 (source: gh:1865) — component-defined table used across an in-place upgrade-boot. +# v1 (pre-restart-1) shape. The test overwrites this file on disk with a v2 shape (adds +# `category`) between restart #1's kill and start, to probe the "config change across the boot" +# angle #1865 also implicates. +type Widget @table(database: "qa647") @export { + id: ID @primaryKey + sku: String @indexed + batch: String @indexed + name: String +}