diff --git a/docs/flowmine-demo/events.jsonl b/docs/flowmine-demo/events.jsonl new file mode 100644 index 0000000000..860b1d9992 --- /dev/null +++ b/docs/flowmine-demo/events.jsonl @@ -0,0 +1,54 @@ +{"ts":"2026-07-10T14:02:01Z","session":"s-A1","source":"client","type":"page_view","page":"group.index","url_path":"/group"} +{"ts":"2026-07-10T14:02:01Z","session":"s-A1","source":"server","type":"api","route":"GET api/v2/groups/summary","status":200} +{"ts":"2026-07-10T14:02:02Z","session":"s-A1","source":"server","type":"api","route":"GET api/users/{id}/notifications","status":200} +{"ts":"2026-07-10T14:02:09Z","session":"s-A1","source":"client","type":"click","page":"group.index","target":"discovery:a[href=/group/create](text=Start a new group)"} +{"ts":"2026-07-10T14:02:10Z","session":"s-A1","source":"client","type":"page_view","page":"group.create","url_path":"/group/create"} +{"ts":"2026-07-10T14:02:10Z","session":"s-A1","source":"server","type":"api","route":"GET api/timezones","status":200} +{"ts":"2026-07-10T14:02:10Z","session":"s-A1","source":"server","type":"api","route":"GET api/v2/groups/names","status":200} +{"ts":"2026-07-10T14:02:11Z","session":"s-A1","source":"server","type":"api","route":"GET api/users/{id}/notifications","status":200} +{"ts":"2026-07-10T14:02:25Z","session":"s-A1","source":"client","type":"change","page":"group.create","target":"discovery:input#group_name@GroupName"} +{"ts":"2026-07-10T14:03:05Z","session":"s-A1","source":"client","type":"change","page":"group.create","target":"discovery:div.ql-editor@RichTextEditor"} +{"ts":"2026-07-10T14:03:31Z","session":"s-A1","source":"client","type":"change","page":"group.create","target":"discovery:input[placeholder=Enter your address]@GroupLocation"} +{"ts":"2026-07-10T14:03:44Z","session":"s-A1","source":"client","type":"change","page":"group.create","target":"discovery:input.timezone@GroupTimeZone"} +{"ts":"2026-07-10T14:04:02Z","session":"s-A1","source":"client","type":"click","page":"group.create","target":"discovery:button.btn-primary(text=Create group)@GroupAddEdit"} +{"ts":"2026-07-10T14:04:03Z","session":"s-A1","source":"server","type":"api","route":"POST api/v2/groups","status":200,"state_changing":true} +{"ts":"2026-07-10T14:04:04Z","session":"s-A1","source":"client","type":"page_view","page":"group.edit","url_path":"/group/edit/1234","redirect_of":"POST api/v2/groups"} +{"ts":"2026-07-11T09:15:20Z","session":"s-A2","source":"client","type":"page_view","page":"group.create","url_path":"/group/create"} +{"ts":"2026-07-11T09:15:20Z","session":"s-A2","source":"server","type":"api","route":"GET api/timezones","status":200} +{"ts":"2026-07-11T09:15:20Z","session":"s-A2","source":"server","type":"api","route":"GET api/v2/groups/names","status":200} +{"ts":"2026-07-11T09:15:52Z","session":"s-A2","source":"client","type":"change","page":"group.create","target":"discovery:input#group_name@GroupName"} +{"ts":"2026-07-11T09:16:14Z","session":"s-A2","source":"client","type":"change","page":"group.create","target":"discovery:div.ql-editor@RichTextEditor"} +{"ts":"2026-07-11T09:16:33Z","session":"s-A2","source":"client","type":"change","page":"group.create","target":"discovery:input.timezone@GroupTimeZone"} +{"ts":"2026-07-11T09:16:50Z","session":"s-A2","source":"client","type":"change","page":"group.create","target":"discovery:input[placeholder=Enter your address]@GroupLocation"} +{"ts":"2026-07-11T09:17:12Z","session":"s-A2","source":"client","type":"click","page":"group.create","target":"discovery:button.btn-primary(text=Create group)@GroupAddEdit"} +{"ts":"2026-07-11T09:17:13Z","session":"s-A2","source":"server","type":"api","route":"POST api/v2/groups","status":200,"state_changing":true} +{"ts":"2026-07-11T09:17:14Z","session":"s-A2","source":"client","type":"page_view","page":"group.edit","url_path":"/group/edit/1235","redirect_of":"POST api/v2/groups"} +{"ts":"2026-07-09T18:30:11Z","session":"s-B1","source":"client","type":"page_view","page":"user.register","url_path":"/user/register"} +{"ts":"2026-07-09T18:30:40Z","session":"s-B1","source":"client","type":"change","page":"user.register","target":"discovery:input.styled-checkbox[id^=skill-]"} +{"ts":"2026-07-09T18:30:48Z","session":"s-B1","source":"client","type":"click","page":"user.register","target":"discovery:button.btn-next(text=Next step)"} +{"ts":"2026-07-09T18:31:12Z","session":"s-B1","source":"client","type":"change","page":"user.register","target":"discovery:input#registerName"} +{"ts":"2026-07-09T18:31:30Z","session":"s-B1","source":"client","type":"change","page":"user.register","target":"discovery:input#registeremail"} +{"ts":"2026-07-09T18:31:31Z","session":"s-B1","source":"server","type":"api","route":"POST register/check-valid-email","status":200} +{"ts":"2026-07-09T18:31:44Z","session":"s-B1","source":"client","type":"change","page":"user.register","target":"discovery:select#age"} +{"ts":"2026-07-09T18:31:55Z","session":"s-B1","source":"client","type":"change","page":"user.register","target":"discovery:select#country"} +{"ts":"2026-07-09T18:32:10Z","session":"s-B1","source":"client","type":"change","page":"user.register","target":"discovery:input#city"} +{"ts":"2026-07-09T18:32:31Z","session":"s-B1","source":"client","type":"change","page":"user.register","target":"discovery:input#password[type=password]"} +{"ts":"2026-07-09T18:32:45Z","session":"s-B1","source":"client","type":"change","page":"user.register","target":"discovery:input#password-confirm[type=password]"} +{"ts":"2026-07-09T18:32:50Z","session":"s-B1","source":"client","type":"click","page":"user.register","target":"discovery:button.btn-next(text=Next step)"} +{"ts":"2026-07-09T18:33:02Z","session":"s-B1","source":"client","type":"change","page":"user.register","target":"discovery:input#newsletter[type=checkbox]"} +{"ts":"2026-07-09T18:33:10Z","session":"s-B1","source":"client","type":"click","page":"user.register","target":"discovery:button[type=submit](text=Create account)"} +{"ts":"2026-07-09T18:33:11Z","session":"s-B1","source":"server","type":"api","route":"POST user/register","status":302,"state_changing":true} +{"ts":"2026-07-09T18:33:12Z","session":"s-B1","source":"client","type":"page_view","page":"dashboard.index","url_path":"/dashboard","redirect_of":"POST user/register"} +{"ts":"2026-07-12T11:05:00Z","session":"s-B2","source":"client","type":"page_view","page":"user.register","url_path":"/user/register"} +{"ts":"2026-07-12T11:05:30Z","session":"s-B2","source":"client","type":"click","page":"user.register","target":"discovery:button.btn-next(text=Next step)"} +{"ts":"2026-07-12T11:06:00Z","session":"s-B2","source":"client","type":"change","page":"user.register","target":"discovery:input#registerName"} +{"ts":"2026-07-12T11:06:20Z","session":"s-B2","source":"client","type":"change","page":"user.register","target":"discovery:input#registeremail"} +{"ts":"2026-07-12T11:06:21Z","session":"s-B2","source":"server","type":"api","route":"POST register/check-valid-email","status":200} +{"ts":"2026-07-12T11:06:40Z","session":"s-B2","source":"client","type":"change","page":"user.register","target":"discovery:select#age"} +{"ts":"2026-07-12T11:06:55Z","session":"s-B2","source":"client","type":"change","page":"user.register","target":"discovery:select#country"} +{"ts":"2026-07-12T11:07:20Z","session":"s-B2","source":"client","type":"change","page":"user.register","target":"discovery:input#password[type=password]"} +{"ts":"2026-07-12T11:07:35Z","session":"s-B2","source":"client","type":"change","page":"user.register","target":"discovery:input#password-confirm[type=password]"} +{"ts":"2026-07-12T11:07:40Z","session":"s-B2","source":"client","type":"click","page":"user.register","target":"discovery:button.btn-next(text=Next step)"} +{"ts":"2026-07-12T11:07:55Z","session":"s-B2","source":"client","type":"click","page":"user.register","target":"discovery:button[type=submit](text=Create account)"} +{"ts":"2026-07-12T11:07:56Z","session":"s-B2","source":"server","type":"api","route":"POST user/register","status":302,"state_changing":true} +{"ts":"2026-07-12T11:07:57Z","session":"s-B2","source":"client","type":"page_view","page":"dashboard.index","url_path":"/dashboard","redirect_of":"POST user/register"} diff --git a/docs/flowmine-demo/flowmine_demo.py b/docs/flowmine-demo/flowmine_demo.py new file mode 100644 index 0000000000..aced3f07f6 --- /dev/null +++ b/docs/flowmine-demo/flowmine_demo.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +""" +flowmine fitness demo (prototype of the PHP artisan pipeline). + +Input : events.jsonl (simulated capture, derived from real UI code) +Reads : the REAL Playwright specs in tests/Integration/ to build the manifest +Output: per-flow verdict — MATCHED(spec, evidence) or UNCOVERED(generated spec) +""" +import json, re, sys, glob, collections, os + +REPO = __import__('os').path.join(__import__('os').path.dirname(__file__), '..', '..') +EVENTS = os.path.join(os.path.dirname(__file__), 'events.jsonl') + +# ── mining-time noise controls (from the design) ──────────────────────────── +AMBIENT_ROUTES = { + # seeded from the mounted()-hook audit + 'GET api/timezones', 'GET api/v2/groups/names', + 'GET api/users/{id}/notifications', 'GET api/v2/groups/summary', + 'GET api/v2/networks', 'GET api/v2/groups/tags', +} + +def sessionize(): + sessions = collections.defaultdict(list) + for line in open(EVENTS): + e = json.loads(line) + sessions[e['session']].append(e) + for s in sessions.values(): + s.sort(key=lambda e: e['ts']) + return sessions + +def flow_signature(events): + """Canonical steps: page views, non-ambient action routes, interaction targets. + A flow ends at the first state-changing API response; the redirect page_view + that follows it is included as the terminal step (segmentation rule).""" + steps, pages, routes, targets = [], [], set(), set() + ended = False + for e in events: + if ended and not e.get('redirect_of'): + break + if e['type'] == 'page_view': + pages.append(e['page']) + steps.append(('page', e['page'])) + if e.get('redirect_of'): + break + elif e['type'] in ('click', 'change'): + t = normalize_target(e['target']) + targets.add(t) + steps.append((e['type'], t)) + elif e['type'] == 'api': + if e['route'] in AMBIENT_ROUTES: + continue # ambient-route denylist + routes.add(e['route']) + steps.append(('api', e['route'])) + if e.get('state_changing'): + ended = True + return {'pages': pages, 'routes': routes, 'targets': targets, 'steps': steps} + +def normalize_target(t): + """Variant normalization: strip the discovery: prefix, drop per-instance noise.""" + t = t.replace('discovery:', '') + t = re.sub(r'\[id\^=.*?\]', '', t) + return t + +def variant_key(sig): + """Design's field-group abstraction: a variant is defined by its dominant + flow page and its action-route set; optional-field presence/order is an + annotation on exemplars, not a new variant.""" + flow_page = sig['pages'][0] + for p in sig['pages']: + if p not in ('dashboard.index',): + flow_page = p # last non-terminal page wins as the flow home + core = [p for p in sig['pages'] if not p.endswith('.index')] + flow_page = core[0] if core else sig['pages'][0] + return (flow_page, frozenset(sig['routes'])) + +# ── manifest side: read the REAL specs ────────────────────────────────────── +def extract_spec_manifest(): + manifest = {} + helper_effects = { + # utils.js helpers resolved once (the real extractor parses utils.js too) + 'login': {'pages': ['/login'], 'selectors': {'#fp_email', '#password', 'button[type=submit]'}, 'routes': set()}, + 'createGroup': {'pages': ['/group', '/group/create'], + 'selectors': {'#group_name', '.ql-editor', '.timezone', + '[placeholder=Enter your address]', 'button(text=Create group)'}, + 'routes': {'POST api/v2/groups'}}, + 'unfollowGroup': {'pages': ['/group/view/{id}'], 'selectors': {'#groupactions', '.dropdown-item'}, 'routes': set()}, + } + for spec in glob.glob(f'{REPO}/tests/Integration/*.test.js'): + src = open(spec).read() + name = os.path.basename(spec) + pages = re.findall(r"page\.goto\(['\"`]([^'\"`]+)", src) + pages = [re.sub(r"\$\{?\w+\}?|'\s*\+.*", '{id}', p.replace("' + baseURL + '", '')) for p in pages] + selectors = set(re.findall(r"page\.(?:locator|click|fill|waitForSelector)\(['\"`]([^'\"`]+)['\"`]", src)) + selectors |= {f'(text={t})' for t in re.findall(r"hasText:\s*['\"`]([^'\"`]+)", src)} + routes = set() + for rx, method in re.findall(r"waitForResponse\(\s*resp\s*=>\s*/(.*?)/\.test\(resp\.url\(\)\)[^)]*?method\(\)\s*===\s*['\"](\w+)['\"]", src, re.S): + routes.add(f"{method} {rx.replace(chr(92)+'/', '/').replace(chr(92)+'d+', '{id}').strip('/')}") + titles = re.findall(r"test\(['\"`]([^'\"`]+)", src) + used_helpers = [h for h in helper_effects if re.search(rf'\b{h}\(', src)] + helper_pages, helper_selectors, helper_routes = [], set(), set() + for h in used_helpers: + helper_pages += helper_effects[h]['pages'] + helper_selectors |= helper_effects[h]['selectors'] + helper_routes |= helper_effects[h]['routes'] + # evaluate() blocks containing .click(): flag as unextractable but note text hints + evals = re.findall(r"page\.evaluate\((.*?)\)\s*\n", src, re.S) + eval_click_texts = set() + for ev in evals: + if '.click()' in ev: + eval_click_texts |= {f'(text={t})' for t in re.findall(r"includes\(['\"]([^'\"]+)['\"]\)", ev)} + selectors |= eval_click_texts + manifest[name] = {'pages': set(pages), 'selectors': selectors, 'routes': routes, + 'helper_pages': set(helper_pages), 'helper_selectors': helper_selectors, + 'helper_routes': helper_routes, 'titles': titles, + 'unextractable_eval_clicks': bool(eval_click_texts)} + return manifest + +# ── matching ──────────────────────────────────────────────────────────────── +def sel_tokens(s): + return set(re.findall(r'[#.]?[\w-]+|\(text=[^)]+\)', s)) + +def match(sig, manifest): + PAGE_URL = {'group.index': '/group', 'group.create': '/group/create', + 'group.edit': '/group/edit/{id}', 'user.register': '/user/register', + 'dashboard.index': '/dashboard'} + urlset = {PAGE_URL.get(p, p) for p in sig['pages']} + flow_tokens = set() + for p in sig['pages']: + flow_tokens |= set(p.split('.')) + best, best_score, best_ev = None, 0.0, None + for spec, m in manifest.items(): + # Scaffolding rule (design: "domain noun" disambiguation): helper-derived + # evidence counts fully ONLY if a test title in this spec shares the + # flow's domain tokens; otherwise it is scaffolding, weighted 0.25. + title_tokens = set(re.findall(r'[a-z]+', ' '.join(m['titles']).lower())) + subject = bool(flow_tokens & title_tokens & {'group', 'register', 'event', + 'device', 'user', 'create', 'edit'} + and flow_tokens & title_tokens - {'create', 'edit', 'index'}) + w = 1.0 if subject else 0.25 + pages = m['pages'] | m['helper_pages'] + eff_routes = set(m['routes']) | {r for r in m['helper_routes']} + route_hits = sum((1.0 if r in m['routes'] else w) for r in (sig['routes'] & eff_routes)) + route_overlap = route_hits / max(len(sig['routes']), 1) + page_hits = sum((1.0 if u in m['pages'] else w) for u in (urlset & pages)) + page_overlap = page_hits / max(len(urlset - {'/dashboard'}), 1) + flat_targets = set().union(*(sel_tokens(t) for t in sig['targets'])) if sig['targets'] else set() + own_sel = set().union(*(sel_tokens(s) for s in m['selectors'])) if m['selectors'] else set() + helper_sel = set().union(*(sel_tokens(s) for s in m['helper_selectors'])) if m['helper_selectors'] else set() + sel_hits = sum((1.0 if t in own_sel else w) for t in (flat_targets & (own_sel | helper_sel))) + sel_overlap = sel_hits / max(len(flat_targets), 1) + score = min(1.0, 0.4 * route_overlap + 0.35 * min(page_overlap, 1.0) + 0.25 * min(sel_overlap, 1.0)) + if score > best_score: + best, best_score = spec, score + best_ev = {'subject_match': subject, + 'page_overlap': round(min(page_overlap, 1.0), 2), + 'route_overlap': round(route_overlap, 2), + 'selector_overlap': round(min(sel_overlap, 1.0), 2), + 'shared_routes': sorted(sig['routes'] & eff_routes), + 'titles': m['titles'][:3]} + return (best, best_score, best_ev) + +THRESHOLD = 0.60 + +def main(): + sessions = sessionize() + manifest = extract_spec_manifest() + variants = collections.defaultdict(list) + sigs = {} + for sid, events in sessions.items(): + sig = flow_signature(events) + sigs[sid] = sig + variants[variant_key(sig)].append(sid) + + print(f"Sessions: {len(sessions)} → flow variants after normalization: {len(variants)}\n") + for i, (vk, sids) in enumerate(sorted(variants.items(), key=lambda kv: -len(kv[1])), 1): + sig = sigs[sids[0]] + name_guess = f"{sig['pages'][0]}" if sig['pages'] else 'unknown' + print(f"── Variant {i}: entry page '{name_guess}', {len(sids)} session(s): {sids}") + print(f" action routes: {sorted(sig['routes'])}") + spec, score, ev = match(sig, manifest) + if score >= THRESHOLD: + print(f" VERDICT: MATCHED → {spec} (score {score:.2f} ≥ {THRESHOLD})") + print(f" evidence: {json.dumps(ev)}") + print(f" ACTION: no test generated — duplicate avoided.\n") + else: + print(f" VERDICT: UNCOVERED (best candidate {spec} scored {score:.2f} < {THRESHOLD})") + print(f" nearest-miss evidence: {json.dumps(ev)}") + print(f" ACTION: generate Playwright spec from steps:") + for kind, val in sig['steps']: + print(f" {kind:6} {val}") + print() + +if __name__ == '__main__': + main() diff --git a/docs/usage-instrumentation-design.md b/docs/usage-instrumentation-design.md new file mode 100644 index 0000000000..cd7638807b --- /dev/null +++ b/docs/usage-instrumentation-design.md @@ -0,0 +1,260 @@ +# Usage instrumentation → flow mining → AI test-coverage analysis + +Status: **approved design v2** (2026-07-14). Implementation follows in phased PRs. +Decisions by Edward marked ✔. + +An executed matching dry-run accompanies this design: see +`usage-instrumentation-worked-example.md` and the runnable prototype in +`flowmine-demo/` (mined group-create correctly matched to group.test.js with +duplicate avoided; uncovered registration flow detected and a candidate spec +generated). + +Two adversarial review rounds shaped this document. Round 1 (privacy/GDPR, +signal quality, operations, frontend feasibility — 27 findings) hardened the +original bespoke pipeline. Round 2 attacked the build-vs-buy premise and the +flow→test matching feasibility with a worked example from the real codebase; +it split the design: the client capture layer now rides the already-deployed +Matomo tracker (✔), while the server-side stream, mining, and coverage +matching remain custom because no off-the-shelf product addresses them +(verified against Matomo, PostHog, OpenReplay, Countly, Snowplow, Jitsu, +RudderStack, Faro, Umami, Plausible, Highlight.io; "ClickTail" does not exist +as a maintained 2026 project). + +## Goal + +Capture the flows real users actually perform (client actions + API calls), +mine them into canonical flows, compare against the existing test estate +(Playwright / PHPUnit API / Jest), and have an AI produce the missing +golden-flow tests — ranked by real usage, validated by a round-trip gate +before any generated test is trusted. + +## Ground truth (corrected in round 2) + +- **Matomo Cloud — not self-hosted — is already wired into every page** + (`restartproject.matomo.cloud`, AWS Frankfurt, InnoCraft DPA). Google Tag + Manager's noscript iframe is also present. The original "no third-party + SaaS" framing was therefore already breached by the status quo; the + operative non-goal is **no NEW SaaS dependencies** (✔ continued Matomo + Cloud use accepted; migrating to self-hosted Matomo remains a separately + costed option that would not change this design's shape). +- The site is not an SPA: 221 blade views, Vue 2.7 islands, no vue-router; + whole flows are blade-only (user/profile ×14, registration, password + reset). The main JS bundle IS loaded on those pages (verified), so one + client capture layer covers both worlds. The 4 iframe/stat embeds don't + load it — out of scope. +- The consent flag (`analyticsCookieEnabled`) is read once at page parse and + **does not currently gate Matomo** (only Sentry's beforeSend consumes it); + nothing reacts to the banner's `gdprCookiesEnabled` event. Fixed in + phase 1 via Matomo's native consent API. +- `api_token` travels in query strings → server-side logging strips query + strings everywhere. +- Laravel's method override (`_method=PATCH` in POSTed forms) is applied + before any middleware runs, so wire method ≠ effective method for exactly + the save calls that matter (verified: Kernel.php `enableHttpMethodParameterOverride`, + and the Vuex store's edit action posts `_method: PATCH`). + +## Architecture + +``` +Browser (all pages) + flow-delegator module (small; init at DOMContentLoaded, NOT inside the + jQuery/Leaflet polling gate) + - document capture-phase listeners: click, submit, change + - TAGGED events: nearest [data-flow="area.step"] → + _paq.push(['trackEvent', 'flow', '', ]) + - DISCOVERY events (✔ always-on): untagged interactive elements → + sanitized elements_chain descriptor (strips data-v-* hashes, + __BVID__*/__BV_* generated ids, noise classes; supplementary + el.__vue__ walk skipping library internals) → + _paq.push(['trackEvent', 'discovery', '', ]) + Volume note: discovery events count as Matomo Cloud hits — phase 1 + checks plan headroom and includes a sampling knob (default 100%). + - form fields: identity + emptiness only, never values; password fields + suppressed entirely. No mouse paths, scroll, or keystrokes. + - session_id: sessionStorage, mirrored into a SameSite=Lax cookie AND + pushed to Matomo as a custom dimension — the join key across streams. + (Native blade form POSTs and Dropzone uploads correlate via the cookie.) + - page identity: added to layouts (does not exist yet + — phase 1 deliverable; URL prefixes are NOT a usable discriminator: + /group/create, /group/edit/{id}, /group/view/{id} share a prefix but + are different flows) + - consent: Matomo native requireConsent()/setConsentGiven()/ + forgetConsentGiven(), wired to the existing gdprCookiesEnabled event — + withdrawal stops tracking within the page view. This REPLACES the + currently-broken arrangement where Matomo fires regardless of consent. + - transport: Matomo tracker (alwaysUseSendBeacon). There is NO bespoke + ingestion endpoint — /api/session-events and all its hardening are + deleted from the design. + ▼ +Matomo Cloud (existing) Laravel RecordApiUsage middleware + events queryable via (web+api): route PATTERN, status, + Live.getLastVisitsDetails duration_ms, BOTH wire method + (per-visit action sequences, (getRealMethod) and effective + paginated, 200 req/min — method, user id, session id + ample at this volume) (header, cookie fallback); + │ NDJSON → /var/log/usage/*.log + │ (logrotate 3–7d) → Alloy tail → + │ Loki (restarters-loki Fly app, + │ private 6PN, labels {app,env, + │ source,event_type} only, 90d) + │ Consent tier: runs for everyone + │ (legitimate interest, route-level + │ only ✔); session/user correlation + │ fields ONLY with analytics consent + └──────────────┬─────────────────────┘ + ▼ +flowmine — PHP artisan commands (✔) + flowmine:export pulls BOTH streams (Matomo Live API + Loki HTTP API, + ≤30-day windows ✔) into local SQLite, joined on + session_id custom dimension + time + flowmine:mine sessionize → directly-follows graph → variant ranking, + with the noise controls listed below + flowmine:manifest test-coverage manifest (below) + flowmine:report ranked gaps (frequency × missing coverage), k≥5 distinct + sessions per exported variant; LLM brief is aggregate-only + flowmine:lint data-flow registry CI check (below) + ▼ +Claude writes missing Playwright / API / Jest tests → human-reviewed PRs +``` + +## Mining correctness (round-2 fitness findings, all binding) + +The paper simulation of the group-create flow showed naive matching fails +(route-set Jaccard ≈ 0.17 vs its own covering test). These rules are part of +the design, not optional tuning: + +1. **Ambient-route denylist**: routes fired unconditionally from mounted()/ + created() hooks (audit found e.g. `api/timezones`, `api/v2/groups/names`, + `api/users/{id}/notifications` on the create page alone) are excluded from + flow identity. Seeded by a one-off audit of Vue mounted() hooks; + maintained in a checked-in list. +2. **Dual HTTP method fields**: server events record wire + effective method; + matching uses effective (aligns with route patterns), wire retained for + client-side reconciliation. +3. **Flow segmentation rule**: a flow ends at the first state-changing API + response after a page_view; a navigation that is the direct redirect + target of that response (e.g. create → /group/edit/{id}) continues the + same flow, otherwise a new candidate begins. Auth is segmented out as its + own sub-flow prefix (sessionStorage session ids make login a near-always + prefix in test-driven sessions but not production ones). +4. **Variant normalization**: optional-field interactions collapse into + field-group abstractions before variant counting; role-gated UI + (canApprove/canNetwork etc.) is a labelled dimension, not separate flows. +5. **Known-unmatchable classes**: bare positional selectors + (`nth-child`-only) and third-party-widget-generated classes are scored as + unmatchable in the manifest, not silently counted against similarity. + The Google Places autocomplete step is a documented capture blind spot + (Google-injected DOM + custom events) — the gap report must not claim + coverage judgement over it. + +## Test-coverage manifest + +- **PHPUnit**: a listener records route patterns per test. Simple stack + inspection is NOT sufficient — 58 call sites across 23 files share the + identical `TestCase::createGroup()` helper, including the one real + create-group test. The disambiguation rule (first state-changing call in + the test body of a test whose class/method name matches the route's domain + noun, plus explicit override annotations where needed) is specified and + unit-tested against that exact corpus before its output is trusted. +- **Playwright**: extractor reads `[data-flow]` selectors + page.goto + targets, resolving shared helpers (utils.js) into their page/selector/route + effects. Helper-derived evidence is weighted as SCAFFOLDING (0.25) unless a + test title in the spec shares the flow's domain tokens — the worked example + (docs/usage-instrumentation-worked-example.md) showed flat scoring + mis-attributes group-create to device.test.js, which uses the same + createGroup() helper as setup. Precondition (phase 3): audit-and-rewrite + pass converting `page.evaluate()`-driven interactions (the existing group + spec's submit is one) to locator-based calls; a lint flags evaluate() + blocks containing `.click()` as unextractable. +- **Jest**: component-name map. +- Output: `tests/coverage-manifest.json`, regenerated in CI. + +## The data-flow convention (✔, with decay protection) + +- Phase 1 adds `data-flow="area.step"` to the ~30–40 highest-traffic + interactive elements; the same attributes become the Playwright selector + convention (zero `data-flow` exists today anywhere, including the 6 specs — + the retrofit is a hard precondition of the round-trip gate). +- `tests/flow-registry.json` + `flowmine:lint` in CI: removing/renaming a + registered attribute fails; malformed values fail; Playwright references + to unregistered flows fail; new unregistered template attributes warn. + Scans .vue AND blade templates (plain text scan). + +## Round-trip validation gate (phase 3 exit criterion) + +Run the 6 existing Playwright spec files (51 test blocks; each test = one +isolated session) against a locally instrumented build with capture on; +flowmine mines ONLY those events against a hand-authored, checked-in +`(spec, test title) → expected flow` ground-truth map. Pass requires: +≥90% of test-driven sessions attributed to the correct flow name, zero +cross-matches between different flows, and the create/edit ambiguity +(group.test.js deliberately straddles it) resolved by the segmentation rule. +Production mining output is not trusted until this gate passes. + +## Privacy posture + +- Client capture (tagged + discovery): consent-gated via Matomo's native + consent API, reactive to withdrawal (✔ two-tier model). +- Server stream: route-level operational logging for everyone under + legitimate interest (no behavioural detail; strictly less than existing + nginx access logs); session/user correlation fields only with consent. +- No raw IPs in the server stream; numeric user ids only; 64-char + truncation; descriptor depth caps. Matomo Cloud's own IP handling follows + the existing DPA (anonymization settings reviewed in phase 1). +- Erasure: Matomo Cloud provides GDPR deletion tooling for its side; the + server-side raw NDJSON window is short (3–7d, user-id filterable, + documented script); Loki entries age out ≤90d; flowmine exports are k≥5 + aggregates with no identifiers, and the LLM-facing brief contains nothing + Sentry-joinable. +- Privacy policy page updated in phase 1. + +## Deployment & cost + +- New infra: `restarters-loki` only (~$3–4/month, private, version pinned, + volume explicitly unbacked-up — losing it restarts collection). Alloy on + prod + restarters-dev; per-PR previews excluded (no volume, no-swap + suspend contract, synthetic traffic) — FEATURE__USAGE_TELEMETRY forced off. +- Matomo Cloud: phase 1 verifies plan hit-volume headroom for always-on + discovery events; sampling knob available if needed. +- Volume gate: a week of nginx access-log counts sizes the server stream + before Loki sizing is committed. +- Local dev: file-only server stream + `flowmine:tail`; Matomo events to a + dev site id or logged locally via a stub. + +## Phasing (each phase its own PR) + +1. **Capture**: data-flow attributes + registry + flowmine:lint; the + flow-delegator module (tagged + discovery via Matomo); Matomo consent fix + (native API, reactive); `` in layouts; session-id + custom dimension + cookie; RecordApiUsage (dual methods, query + stripping); NDJSON + Alloy + restarters-loki + logrotate; mounted()-hook + ambient-route audit; Matomo plan headroom check; privacy-policy copy. +2. **flowmine export/mine**: Matomo Live API + Loki export → SQLite; DFG + mining with denylist/segmentation/normalization; first usage report over + ≥2 weeks of production data. +3. **Manifest + matching + round-trip gate**: PHPUnit listener (tested + against the 58-call-site corpus), Playwright spec rewrite + extractor, + Jest map, matcher, k≥5 floor, the round-trip gate. Gate passes before any + production gap report is issued. +4. **Golden-flow tests**: Claude consumes the gap brief; top uncovered flows + land as reviewed tests; the loop documented as repeatable. + +## Decisions taken (Edward) + +| Decision | Choice | +|---|---| +| Consent model | Two-tier: LI server route logs; consent-gated client capture | +| Client transport | Matomo (existing tracker; native consent API; Live API export) | +| Discovery capture | Always-on via Matomo discovery events (plan-volume checked) | +| Store of record | Loki (server stream) + Matomo (client stream), joined in flowmine's SQLite; 30-day windows | +| Element identity | data-flow primary + registry + CI lint against decay | +| flowmine language | PHP artisan | +| Grafana UI | Not initially | +| Matomo Cloud | Continued use accepted; self-hosting it = separate future decision | + +## Non-goals + +Session replay / DOM recording; keystroke or mouse-path capture; a new +ingestion endpoint or any NEW SaaS dependency; replacing Sentry; real-time +dashboards; instrumenting iframe stat embeds; capture on per-PR preview +apps. diff --git a/docs/usage-instrumentation-worked-example.md b/docs/usage-instrumentation-worked-example.md new file mode 100644 index 0000000000..bbd2c1165f --- /dev/null +++ b/docs/usage-instrumentation-worked-example.md @@ -0,0 +1,157 @@ +# Worked example: mined events → duplicate-avoidance → generated Playwright test + +Companion to `usage-instrumentation-design.md`. This is an executed dry run of +the flowmine matching pipeline, not a thought experiment: a prototype matcher +was run against simulated capture data, with the manifest side extracted from +the **real** spec files in `tests/Integration/`. Two flows were mined — one the +suite already covers (group create), one it does not (account registration) — +to demonstrate both halves of the claim: *recognise existing coverage to avoid +duplication* and *generate a new test for a real gap*. + +Honesty statement: the input events are hand-derived from the actual UI code +(GroupAddEdit.vue and children, auth/register-new.blade.php, the Vuex store's +API calls, the mounted()-hook ambient audit) because capture is not built yet — +that is exactly what the phase-3 round-trip gate replaces with real recorded +events. The manifest extraction and matching, however, ran against the real +spec files. + +## Input + +4 sessions, 55 events (`events.jsonl`): two users creating a group (one via +/group → create button, one landing directly on /group/create; different +optional-field orders), and two users registering an account (one filling all +optional fields + newsletter, one minimal). Server-side events include the +ambient mount-time calls (`GET api/timezones`, `GET api/v2/groups/names`, +`GET api/users/{id}/notifications`) precisely so the denylist has something +to prove. + +## What the first run got wrong — and why that matters + +The naive first run produced **exactly the failure the fitness review +predicted**: + +1. Group-create matched to `device.test.js` (score 0.75), not + `group.test.js`. Cause: `device.test.js` (and `event.test.js`) call the + same `createGroup()` helper as *scaffolding*, so their manifests contain + the full group-create footprint. Flat overlap scoring cannot tell a spec + that tests a flow from a spec that merely performs it on the way to + something else — the Playwright twin of the PHPUnit 58-call-site + scaffolding problem. +2. The two group-create sessions fragmented into two variants (different + entry pages), as did the two registrations (optional-field differences). + +Fixes applied to the matcher (and now binding in the design): + +- **Scaffolding weighting with a subject rule**: helper-derived manifest + evidence counts at 0.25 weight unless a test title in the spec shares the + flow's domain tokens ("Can create group" ↔ flow pages `group.*`), in which + case the helper is the subject and counts fully. +- **Variant key = (flow page, action-route set)** after the ambient denylist — + optional-field presence/order becomes an annotation on exemplars, not a new + variant. + +## Corrected run output (verbatim) + +``` +Sessions: 4 → flow variants after normalization: 2 + +── Variant 1: entry page 'group.index', 2 session(s): ['s-A1', 's-A2'] + action routes: ['POST api/v2/groups'] + VERDICT: MATCHED → group.test.js (score 0.74 ≥ 0.6) + evidence: {"subject_match": true, "page_overlap": 0.67, "route_overlap": 1.0, + "selector_overlap": 0.41, "shared_routes": ["POST api/v2/groups"], + "titles": ["Can create group", "Can unfollow group", + "Group image upload persists on view page"]} + ACTION: no test generated — duplicate avoided. + +── Variant 2: entry page 'user.register', 2 session(s): ['s-B1', 's-B2'] + action routes: ['POST register/check-valid-email', 'POST user/register'] + VERDICT: UNCOVERED (best candidate grouptags.test.js scored 0.06 < 0.6) + ACTION: generate Playwright spec from steps: [16 steps listed] +``` + +Registration is genuinely uncovered: no spec under tests/Integration touches +`/user/register`, `POST user/register`, or any `register*` selector. + +## Generated test (from variant 2's steps + verification against the view) + +The generation step is not blind templating: the mined steps give the skeleton +(pages, fields touched, API calls to await, terminal redirect), and the +generating LLM then verifies selectors against the actual view code. That +verification mattered here: the mined click descriptor for the final submit +(`button[type=submit]`) resolves in `register-new.blade.php` to +`#register-form-submit`, and the view revealed a **step-4 GDPR consent +checkbox (`#consent_gdpr`)** that the simulated sessions never recorded +(consent-checkbox interactions are plausible fast-path omissions in real data +too). Generated spec, flagged for human review: + +```js +// tests/Integration/registration.test.js +// GENERATED from mined flow 'user.register' +// (2/2 observed sessions: multi-step wizard → POST user/register → /dashboard) +// Verified against resources/views/auth/register-new.blade.php before proposal. +const { test } = require('./fixtures') +const { expect } = require('@playwright/test') +const faker = require('faker') + +test('Can register a new account', async ({page, baseURL}) => { + test.slow() + + await page.goto(baseURL + '/user/register') + + // Step 1: skills are optional (1 of 2 observed sessions skipped them). + await page.locator('#step-1 button.btn-next').click() + + // Step 2: identity + password. The email field triggers an async + // validity check observed in both sessions — wait for it so the wizard + // doesn't advance on a stale state. + await page.fill('#registerName', faker.name.findName()) + const [emailCheck] = await Promise.all([ + page.waitForResponse(resp => resp.url().includes('/register/check-valid-email')), + page.fill('#registeremail', faker.internet.email().toLowerCase()), + ]) + expect(emailCheck.status()).toBe(200) + await page.selectOption('#age', { index: 1 }) + await page.selectOption('#country', 'GB') + await page.fill('#password', 'passw0rd!Reg') + await page.fill('#password-confirm', 'passw0rd!Reg') + await page.locator('#step-2 button.btn-next').click() + + // Step 3: preferences (both observed sessions proceeded without changes + // being required; newsletter was ticked in 1 of 2). + await page.locator('#step-3 button.btn-next').click() + + // Step 4: GDPR consent — NOT present in the mined events (see note above); + // required by the form, added from the view code. + await page.check('#consent_gdpr') + + const [registerResponse] = await Promise.all([ + page.waitForResponse(resp => resp.url().includes('/user/register') + && resp.request().method() === 'POST'), + page.locator('#register-form-submit').click(), + ]) + expect([200, 302]).toContain(registerResponse.status()) + + // Both observed sessions ended on the dashboard. + await page.waitForSelector('section.dashboard') +}) +``` + +Status: **proposed, not committed to tests/Integration** — per the design, +generated tests land only via a reviewed PR after running against a live dev +environment (and this one needs step-transition waits validated against the +wizard's JS). + +## What this demonstrates, and what it does not + +Demonstrated end-to-end: sessionization → ambient filtering → variant +normalization → manifest extraction from real specs → duplicate avoidance +with evidence → gap detection → generation grounded in both the mined steps +and the actual view code — including the pipeline catching its own +mis-attribution defect and the fix generalising (scaffolding weighting). + +Not demonstrated (this is what phase 3's round-trip gate exists for): real +captured events (these were hand-derived), matching at production variant +diversity, the PHPUnit/Jest manifest sides, and thresholds tuned on more than +two flows. The prototype matcher (~150 lines of Python) is the specification +for the flowmine:match implementation, not the implementation itself.