Skip to content

Commit 5dbe765

Browse files
mvalancyclaude
andauthored
Self-heal the dev DB around heavy test suites (clean systems) (#51)
Large scale-sweep / VLM runs seed real graphs; an interrupted run (timeout, Ctrl-C) skips its per-test cleanup and leaves orphan WorkItems + drifted seed positions behind. That pollution made THE GATE's grow-flow flake (force-click landing on an overlapping node) until a manual re-seed. Now the heavy suites self-heal: - All test-seeded graphs are tagged with a sentinel name prefix ("[E2E]") so they're unmistakably identifiable (never matches a real/seed graph). - tests/helpers/dbHealing.ts sweepTestData() (Cypher over bolt, batched, fully graceful if Neo4j is down) removes: sentinel/legacy test-named graphs + their WorkItems/Edge nodes, orphan WorkItems (no BELONGS_TO), and orphan Edge nodes (missing source/target — the data-integrity incident class). It never touches seed/demo graphs. - scale-sweep.spec.ts and visual-vlm.spec.ts call it in beforeAll (heal leftovers from a prior killed run) and afterAll (clean up this run even if a per-run delete was skipped). Verified: injected a test graph + orphan node + orphan edge → sweep removed exactly those, seed data (44 items) untouched; a scale run leaves 0 test graphs / 0 orphans; THE GATE stays 5/5 green. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 0e2c412 commit 5dbe765

4 files changed

Lines changed: 124 additions & 4 deletions

File tree

tests/e2e/visual-vlm.spec.ts

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import * as fs from 'fs';
33
import * as path from 'path';
44
import { login, TEST_USERS } from '../helpers/auth';
55
import { seedLargeGraph, deleteGraphDeep } from '../helpers/seedGraph';
6+
import { sweepTestData, TEST_GRAPH_PREFIX } from '../helpers/dbHealing';
67
import '../helpers/testEnv';
78
import { isVlmAvailable, evaluateBatch, PERSONAS, personaByKey } from '../helpers/vlm';
89

@@ -31,6 +32,11 @@ async function shot(page: Page, name: string): Promise<string> {
3132
return file;
3233
}
3334

35+
// Self-heal: clear leftover test graphs + orphans before and after, so an
36+
// interrupted run never leaves the dev DB dirty (which can break THE GATE).
37+
test.beforeAll(async () => { await sweepTestData('vlm:before'); });
38+
test.afterAll(async () => { await sweepTestData('vlm:after'); });
39+
3440
test('VLM visual evaluation across personas @vlm', async ({ page }) => {
3541
test.setTimeout(900_000);
3642
const available = await isVlmAvailable();
@@ -44,14 +50,14 @@ test('VLM visual evaluation across personas @vlm', async ({ page }) => {
4450
await page.waitForTimeout(1500);
4551

4652
// 1. Empty graph — first-run invitation (new-user + visual defects).
47-
const empty = await page.evaluate(async () => {
53+
const empty = await page.evaluate(async (pfx) => {
4854
const token = localStorage.getItem('authToken') ?? '';
4955
const post = (query: string, variables?: unknown) =>
5056
fetch('/api/graphql', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${token}` }, body: JSON.stringify({ query, variables }) }).then((r) => r.json());
5157
const me = await post('{ me { id } }');
52-
const g = await post(`mutation($i:[GraphCreateInput!]!){createGraphs(input:$i){graphs{id}}}`, { i: [{ name: `VLM Empty ${Date.now()}`, type: 'PROJECT', status: 'ACTIVE', createdBy: me.data.me.id, isShared: true }] });
58+
const g = await post(`mutation($i:[GraphCreateInput!]!){createGraphs(input:$i){graphs{id}}}`, { i: [{ name: `${pfx} VLM Empty ${Date.now()}`, type: 'PROJECT', status: 'ACTIVE', createdBy: me.data.me.id, isShared: true }] });
5359
return g.data.createGraphs.graphs[0].id as string;
54-
});
60+
}, TEST_GRAPH_PREFIX);
5561
cleanup.push(empty);
5662
await page.setViewportSize({ width: 1440, height: 900 });
5763
await page.evaluate((id) => localStorage.setItem('currentGraphId', id), empty);

tests/helpers/dbHealing.ts

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
import neo4j, { Driver } from 'neo4j-driver';
2+
3+
/**
4+
* Self-healing for the dev Neo4j so test runs never leave the database dirty —
5+
* even when a run is killed mid-flight (timeout, Ctrl-C) and its per-test
6+
* cleanup never executes.
7+
*
8+
* Heavy suites (scale-sweep, visual-vlm) call sweepTestData() in beforeAll
9+
* (heal leftovers from a previous interrupted run) AND afterAll (clean up this
10+
* run). It removes:
11+
* - graphs whose name carries the test sentinel (or a legacy test prefix),
12+
* with their WorkItems and Edge nodes,
13+
* - orphan WorkItems (no BELONGS_TO) — what a half-finished delete leaves,
14+
* - orphan Edge nodes (missing a source or target) — these 500 the edges
15+
* query, the original data-integrity incident.
16+
*
17+
* It NEVER touches seed/demo graphs (Welcome, Cycle 2, Aquarium, …) — only
18+
* sentinel/test-named graphs and true orphans. Fully graceful: if Neo4j is
19+
* unreachable it logs and returns zeros rather than failing the run.
20+
*/
21+
22+
/** Every test-seeded graph name starts with this so the sweep can find them
23+
* unambiguously without ever matching a real graph. */
24+
export const TEST_GRAPH_PREFIX = '[E2E]';
25+
26+
// Legacy/explicit test-name patterns (graphs created before the sentinel, or by
27+
// ad-hoc probes). Anchored so they can't match real graphs.
28+
const LEGACY_TEST_NAME_REGEX =
29+
'^(\\[E2E\\]|Scale |VLM |Clone|Parity|PathP|NodeAttach|Contract|CloneFix|CloneProbe|Pop|TP |Empty Smoke|Living E2E|ParityV|Smoke ).*';
30+
31+
const URI = process.env.NEO4J_URI || 'bolt://localhost:7687';
32+
const USER = process.env.NEO4J_USER || 'neo4j';
33+
const PASS = process.env.NEO4J_PASSWORD || 'graphdone_password';
34+
35+
export interface SweepResult {
36+
testGraphs: number;
37+
testGraphNodes: number;
38+
orphanNodes: number;
39+
orphanEdges: number;
40+
ok: boolean;
41+
}
42+
43+
async function deleteInBatches(session: any, matchDelete: string): Promise<number> {
44+
// matchDelete must be a query of shape: MATCH ... WITH x LIMIT 5000 DETACH DELETE x RETURN count(x) AS c
45+
let total = 0;
46+
for (;;) {
47+
const r = await session.run(matchDelete);
48+
const c = r.records[0]?.get('c')?.toNumber?.() ?? 0;
49+
total += c;
50+
if (c === 0) break;
51+
}
52+
return total;
53+
}
54+
55+
export async function sweepTestData(label = ''): Promise<SweepResult> {
56+
const result: SweepResult = { testGraphs: 0, testGraphNodes: 0, orphanNodes: 0, orphanEdges: 0, ok: false };
57+
let driver: Driver | undefined;
58+
try {
59+
driver = neo4j.driver(URI, neo4j.auth.basic(USER, PASS));
60+
await driver.verifyConnectivity();
61+
const session = driver.session();
62+
try {
63+
// 1) WorkItems + Edge nodes that belong to test-named graphs.
64+
result.testGraphNodes = await deleteInBatches(
65+
session,
66+
`MATCH (g:Graph) WHERE g.name =~ '${LEGACY_TEST_NAME_REGEX}'
67+
MATCH (g)<-[:BELONGS_TO]-(w:WorkItem)
68+
OPTIONAL MATCH (w)<-[:EDGE_SOURCE|EDGE_TARGET]-(e:Edge)
69+
WITH w, e LIMIT 5000 DETACH DELETE e, w RETURN count(w) AS c`
70+
);
71+
// 2) The test-named graphs themselves.
72+
const g = await session.run(
73+
`MATCH (g:Graph) WHERE g.name =~ '${LEGACY_TEST_NAME_REGEX}' DETACH DELETE g RETURN count(g) AS c`
74+
);
75+
result.testGraphs = g.records[0]?.get('c')?.toNumber?.() ?? 0;
76+
// 3) Orphan WorkItems (belong to no graph) — what a killed delete leaves.
77+
result.orphanNodes = await deleteInBatches(
78+
session,
79+
`MATCH (w:WorkItem) WHERE NOT (w)-[:BELONGS_TO]->(:Graph) WITH w LIMIT 5000 DETACH DELETE w RETURN count(w) AS c`
80+
);
81+
// 4) Orphan Edge nodes (missing a source or target) — these break the
82+
// edges query for everyone.
83+
result.orphanEdges = await deleteInBatches(
84+
session,
85+
`MATCH (e:Edge) WHERE NOT (e)-[:EDGE_SOURCE]->(:WorkItem) OR NOT (e)-[:EDGE_TARGET]->(:WorkItem) WITH e LIMIT 5000 DETACH DELETE e RETURN count(e) AS c`
86+
);
87+
result.ok = true;
88+
const touched = result.testGraphs + result.testGraphNodes + result.orphanNodes + result.orphanEdges;
89+
if (touched > 0) {
90+
// eslint-disable-next-line no-console
91+
console.log(
92+
`[db-heal${label ? ' ' + label : ''}] swept ${result.testGraphs} test graphs, ${result.testGraphNodes} their nodes, ${result.orphanNodes} orphan nodes, ${result.orphanEdges} orphan edges`
93+
);
94+
}
95+
} finally {
96+
await session.close();
97+
}
98+
} catch (err) {
99+
// Graceful: never fail the test run because healing couldn't connect.
100+
// eslint-disable-next-line no-console
101+
console.warn(`[db-heal] skipped (${err instanceof Error ? err.message.split('\n')[0] : String(err)})`);
102+
} finally {
103+
await driver?.close();
104+
}
105+
return result;
106+
}

tests/helpers/seedGraph.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { Page } from '@playwright/test';
2+
import { TEST_GRAPH_PREFIX } from './dbHealing';
23

34
/**
45
* Seeds realistically-shaped graphs of arbitrary size through the real GraphQL
@@ -64,7 +65,7 @@ export async function seedLargeGraph(page: Page, opts: SeedOptions): Promise<See
6465
const g = await gql(
6566
page,
6667
`mutation($input: [GraphCreateInput!]!) { createGraphs(input: $input) { graphs { id } } }`,
67-
{ input: [{ name: `${namePrefix} ${size}n ${Date.now()}`, type: 'PROJECT', status: 'ACTIVE', createdBy: userId, isShared: true }] }
68+
{ input: [{ name: `${TEST_GRAPH_PREFIX} ${namePrefix} ${size}n ${Date.now()}`, type: 'PROJECT', status: 'ACTIVE', createdBy: userId, isShared: true }] }
6869
);
6970
const graphId = g.createGraphs.graphs[0].id as string;
7071

tests/perf/scale-sweep.spec.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import * as fs from 'fs';
33
import * as path from 'path';
44
import { login, TEST_USERS } from '../helpers/auth';
55
import { seedLargeGraph, deleteGraphDeep } from '../helpers/seedGraph';
6+
import { sweepTestData } from '../helpers/dbHealing';
67
import '../helpers/testEnv';
78
import { envIntList, envList } from '../helpers/testEnv';
89

@@ -193,6 +194,12 @@ async function measure(page: Page, graphId: string, size: number, quality: strin
193194
test.describe('large-scale graph perf sweep @scale', () => {
194195
test.describe.configure({ mode: 'serial', timeout: 600_000 });
195196

197+
// Self-heal: clear leftover test graphs + orphans from any prior interrupted
198+
// run before starting, and clean up this run afterward even if a per-run
199+
// delete was skipped (e.g. a killed run).
200+
test.beforeAll(async () => { await sweepTestData('scale:before'); });
201+
test.afterAll(async () => { await sweepTestData('scale:after'); });
202+
196203
for (const size of SIZES) {
197204
test(`sweep ${size} nodes`, async ({ page }) => {
198205
await login(page, TEST_USERS.ADMIN);

0 commit comments

Comments
 (0)