diff --git a/CHANGELOG.md b/CHANGELOG.md index 17f3071e..058487a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ ### Changes +- Reporter: internal page-inspection calls Explorbot makes on its own — `I.grabBrowserLogs()`, `I.grabSource()`, + `I.saveScreenshot()` and the like — no longer appear among a test's steps or in its log on Testomat.io. Reported + tests now list only the actions the test actually performed. +- Reporter: browser console errors and failed network requests are no longer reported as failed steps. They are + collected in the test log instead, so a page that logs errors in the background no longer makes a passing test + read as broken. Pilot and the session report still receive them as before. +- Reporter: a console or network error arriving at the very end of a test no longer replaces the test's summary + message in the report. +- [Rerunner] `explorbot rerun` no longer prints or records Explorbot's own page-inspection calls between the steps + of the test being re-run. +- Observability: each step in a Langfuse trace is now named after the command it ran (`I.click`, `I.fillField`) + instead of a generic `I.step`. - [Navigator] A site that redirects to another host under the same domain — a bare domain sending the browser to its `www.` host, for example — no longer fails every navigation. Explorbot used to compare the landing host to the configured one exactly, so it rejected the page it had just loaded and reported `expected /, got /`. A host now diff --git a/src/action.ts b/src/action.ts index 6ee8e5ea..a16ed7fc 100644 --- a/src/action.ts +++ b/src/action.ts @@ -16,6 +16,7 @@ import { createDebug, setStepSpanParent, tag } from './utils/logger.js'; import { Overlay, OverlayPage } from './utils/overlay.js'; import { sleep, waitForPageReadiness } from './utils/page-readiness.ts'; import type { Region } from './utils/region.js'; +import { isInternalStep } from './utils/step-analyzer.ts'; import { safeFilename } from './utils/strings.ts'; import { isSameHostFamily } from './utils/url-matcher.js'; import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts'; @@ -564,7 +565,7 @@ export const attachStepLogger = (target: ExecutedStep[], assertionsTarget?: Arra let batchFailed = false; const listener: StepListener = (step, error) => { if (!step?.toCode) return; - if (step.name?.startsWith('grab')) return; + if (isInternalStep(step)) return; const existing = recorded.get(step); if (existing) { diff --git a/src/ai/rerunner.ts b/src/ai/rerunner.ts index 7cd6d49d..a2970bd3 100644 --- a/src/ai/rerunner.ts +++ b/src/ai/rerunner.ts @@ -18,6 +18,7 @@ import { formatHeadings } from '../utils/context-formatter.ts'; import { createDebug, tag } from '../utils/logger.ts'; import { loop } from '../utils/loop.ts'; import { RulesLoader } from '../utils/rules-loader.ts'; +import { isInternalStep } from '../utils/step-analyzer.ts'; import type { Agent, AgentDeps } from './agent.ts'; import { toolExecutionLabel } from './conversation.ts'; import type { Navigator } from './navigator.ts'; @@ -85,6 +86,7 @@ export class Rerunner extends TaskAgent implements Agent { const onStepStarted = (step: any) => { if (!step.toCode) return; + if (isInternalStep(step)) return; const code = highlight(step.toCode(), { language: 'javascript' }); console.log(chalk.dim(` ${code}`)); }; @@ -92,12 +94,14 @@ export class Rerunner extends TaskAgent implements Agent { const onStepPassed = (step: any) => { const task = this.getCurrentTask(testMap); if (!task || !step.toCode) return; + if (isInternalStep(step)) return; task.addStep(step.toCode(), step.duration, 'passed'); }; const onStepFailed = (step: any, error: any) => { const task = this.getCurrentTask(testMap); if (!task || !step.toCode) return; + if (isInternalStep(step)) return; task.addStep(step.toCode(), step.duration, 'failed', error?.message); console.log(chalk.red(` ${figureSet.cross} ${step.toCode()} — ${error?.message || 'failed'}`)); }; diff --git a/src/explorer.ts b/src/explorer.ts index 5f7ec626..4aaee026 100644 --- a/src/explorer.ts +++ b/src/explorer.ts @@ -21,6 +21,7 @@ import { Test, TestResult } from './test-plan.ts'; import { BrowserRecoveryError, browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from './utils/browser-errors.ts'; import { createDebug, log, tag } from './utils/logger.js'; import { sleep, waitForPageReadiness } from './utils/page-readiness.ts'; +import { isInternalStep } from './utils/step-analyzer.ts'; declare global { namespace NodeJS { @@ -206,8 +207,7 @@ class Explorer { const stepHandler = (step: any, status?: string, error?: string, log?: string) => { if (!step.toCode) return; - if (step?.name?.startsWith('grab')) return; - if (step?.name?.startsWith('save')) return; + if (isInternalStep(step)) return; test.addStep(step.toCode(), step.duration, status, error, log); diff --git a/src/reporter.ts b/src/reporter.ts index 4d0dd4c6..4aabfa12 100644 --- a/src/reporter.ts +++ b/src/reporter.ts @@ -138,6 +138,7 @@ export class Reporter { protected combineStepsAndNotes(test: Test, lastScreenshotFile?: string): Step[] { const noteEntries = Object.entries(test.notes) + .filter(([, note]) => !note.observation) .map(([timestampKey, note]) => ({ startTime: note.startTime, endTime: note.endTime, @@ -272,9 +273,12 @@ export class Reporter { description: test.description, code: test.generatedCode || '', steps, - logs: Object.values(test.steps) - .map((stepData) => stepData.text) - .join('\n'), + logs: [ + ...Object.values(test.steps).map((stepData) => stepData.text), + ...Object.values(test.notes) + .filter((note) => note.observation) + .map((note) => note.message), + ].join('\n'), files: Object.values(test.artifacts) || [], message: test.summary || this.extractLastNoteMessage(test) || '', meta, @@ -341,7 +345,7 @@ export class Reporter { } private extractLastNoteMessage(test: Test): string { - const notes = Object.values(test.notes); + const notes = Object.values(test.notes).filter((note) => !note.observation); if (notes.length === 0) return ''; return notes[notes.length - 1].message; } diff --git a/src/utils/logger.ts b/src/utils/logger.ts index b44796e4..5d4ecb3c 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -215,7 +215,7 @@ class SpanDestination implements LogDestination { if (!step?.toCode) { return; } - const stepName = step?.name ? `I.${step.name}` : 'I.step'; + const stepName = step?.title ? `I.${step.title}` : 'I.step'; const stepInput = typeof step?.toCode === 'function' ? step.toCode() : entry.content; const errorFromStep = step?.error; const errorMessage = diff --git a/src/utils/step-analyzer.ts b/src/utils/step-analyzer.ts index aea50661..86b01c0e 100644 --- a/src/utils/step-analyzer.ts +++ b/src/utils/step-analyzer.ts @@ -5,6 +5,8 @@ import { isDynamicId } from './xpath.ts'; export const CODECEPT_TOOLS = ['click', 'hover', 'pressKey', 'form'] as const; export type CodeceptToolName = (typeof CODECEPT_TOOLS)[number]; +const INTERNAL_STEP_PREFIXES = ['grab', 'save']; + const CODECEPT_FORM_COMMANDS: readonly string[] = ['I.fillField', 'I.type', 'I.selectOption', 'I.attachFile', 'I.checkOption', 'I.uncheckOption']; export function isCodeceptToolName(toolName: string): toolName is CodeceptToolName { @@ -71,3 +73,9 @@ export function mergeUniqueStepsByCode(primary: SessionStep[], secondary: Sessio } return merged; } + +export function isInternalStep(step: { title?: string }): boolean { + const title = step?.title; + if (!title) return false; + return INTERNAL_STEP_PREFIXES.some((prefix) => title.startsWith(prefix)); +} diff --git a/tests/unit/explorer-step-listeners.test.ts b/tests/unit/explorer-step-listeners.test.ts index d6e402d1..807c6b2c 100644 --- a/tests/unit/explorer-step-listeners.test.ts +++ b/tests/unit/explorer-step-listeners.test.ts @@ -15,11 +15,11 @@ function buildExplorer(dispatcher: EventEmitter) { return explorer as Explorer; } -function buildTest() { +function buildTest(addStep: (code: string) => void = () => {}) { return { scenario: 'listener leak regression', start: () => {}, - addStep: () => {}, + addStep, setActiveNoteScreenshot: () => {}, getPrintableNotes: () => '', } as any; @@ -39,6 +39,18 @@ describe('Explorer step listener cleanup', () => { expect(dispatcher.listenerCount('test.after')).toBe(0); }); + it('does not record grabbers and savers as test steps', async () => { + const dispatcher = new EventEmitter(); + const recorded: string[] = []; + await buildExplorer(dispatcher).beginTest(buildTest((code) => recorded.push(code))); + + dispatcher.emit('step.passed', { title: 'grabBrowserLogs', toCode: () => 'I.grabBrowserLogs()' }); + dispatcher.emit('step.passed', { title: 'saveScreenshot', toCode: () => 'I.saveScreenshot("x.png")' }); + dispatcher.emit('step.passed', { title: 'click', toCode: () => 'I.click("Login")' }); + + expect(recorded).toEqual(['I.click("Login")']); + }); + it('does not accumulate listeners across repeated startTest cycles', async () => { const dispatcher = new EventEmitter(); for (let i = 0; i < 5; i++) { diff --git a/tests/unit/reporter.test.ts b/tests/unit/reporter.test.ts index 34ebe2e4..376d90d6 100644 --- a/tests/unit/reporter.test.ts +++ b/tests/unit/reporter.test.ts @@ -202,6 +202,24 @@ describe('Reporter', () => { expect(steps[0].steps).toBeUndefined(); }); + test('should keep console and network observations out of report steps', async () => { + const reporter = new TestableReporter(); + const test = new Test('Test Scenario', 'high', ['Expected outcome'], 'https://example.com'); + + const note = test.startNote('Open dashboard'); + await new Promise((resolve) => setTimeout(resolve, 10)); + test.addStep('I.click("Dashboard")', 40, 'passed'); + note.commit(TestResult.PASSED); + + test.addObservation('Console error: Uncaught TypeError'); + test.addObservation('Network error: GET /api/items → 500'); + + const steps = reporter.combineStepsAndNotes(test); + + expect(steps.length).toBe(1); + expect(steps[0].title).toBe('Open dashboard'); + }); + test('should handle empty test', () => { const reporter = new TestableReporter(); const test = new Test('Test Scenario', 'high', ['Expected outcome'], 'https://example.com'); diff --git a/tests/unit/step-analyzer.test.ts b/tests/unit/step-analyzer.test.ts index 000612d7..6f560a62 100644 --- a/tests/unit/step-analyzer.test.ts +++ b/tests/unit/step-analyzer.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'bun:test'; import type { SessionStep } from '../../src/experience-tracker.ts'; -import { getCodeceptToolNameFromCode, isCodeceptToolName, isNonReusableCode, mergeUniqueStepsByCode, stripComments, toReusableSessionStep } from '../../src/utils/step-analyzer.ts'; +import { getCodeceptToolNameFromCode, isCodeceptToolName, isInternalStep, isNonReusableCode, mergeUniqueStepsByCode, stripComments, toReusableSessionStep } from '../../src/utils/step-analyzer.ts'; describe('step-analyzer', () => { it('maps CodeceptJS commands to agent tool names', () => { @@ -12,6 +12,15 @@ describe('step-analyzer', () => { expect(getCodeceptToolNameFromCode('I.see("Done")')).toBe(null); }); + it('treats grabbers and savers as internal instrumentation', () => { + expect(isInternalStep({ title: 'grabBrowserLogs' })).toBe(true); + expect(isInternalStep({ title: 'grabSource' })).toBe(true); + expect(isInternalStep({ title: 'saveScreenshot' })).toBe(true); + expect(isInternalStep({ title: 'click' })).toBe(false); + expect(isInternalStep({ title: 'see' })).toBe(false); + expect(isInternalStep({})).toBe(false); + }); + it('recognizes CodeceptJS agent tools by name', () => { expect(isCodeceptToolName('click')).toBe(true); expect(isCodeceptToolName('form')).toBe(true);