diff --git a/CHANGELOG.md b/CHANGELOG.md index 054ca15e..e7f01ae6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 2026-09-15 + +### Fixes + +- [Pilot] A scenario whose goal is that an invalid action must not succeed is no longer failed because the + app blocks it differently than the plan predicted. The plan names one guard — a validation message, say — + but the app may block by disabling the control or refusing the submit, and any of those met the goal. + Pilot told Tester to record such a run as failed, so tests that found the app working correctly were + reported as failures. +- [Navigator] `verify()` can now assert which page is open. There was no URL check among the assertions + Navigator is allowed to write, so every claim about the current URL failed even when the page was on + exactly that address, and Tester was left without evidence that a navigation had worked. + ## 2026-09-14 ### Changes diff --git a/rules/navigator/verification-actions.md b/rules/navigator/verification-actions.md index 91c59847..0ab208aa 100644 --- a/rules/navigator/verification-actions.md +++ b/rules/navigator/verification-actions.md @@ -50,6 +50,18 @@ Checks that page title contains expected text. I.seeInTitle('Dashboard'); +### I.seeInCurrentUrl + +I.seeInCurrentUrl() + +Checks that the current URL contains the expected path. Substring match — pass the path, not the full URL. +This is the ONLY way to assert the URL — page text and source are not evidence of it. + + + I.seeInCurrentUrl('/dashboard'); + I.seeInCurrentUrl('/users/42/edit'); + + ### I.seeInSource I.seeInSource() @@ -95,6 +107,16 @@ Checks that an input field does NOT contain the specified value. I.dontSeeInField('Email', ''); +### I.dontSeeInCurrentUrl + +I.dontSeeInCurrentUrl() + +Checks that the current URL does NOT contain the given path. + + + I.dontSeeInCurrentUrl('/login'); + + ### I.dontSeeInSource I.dontSeeInSource() diff --git a/src/ai/pilot.ts b/src/ai/pilot.ts index 2421e77f..b1df1d13 100644 --- a/src/ai/pilot.ts +++ b/src/ai/pilot.ts @@ -1116,6 +1116,7 @@ export class Pilot implements Agent { Already-achieved detection: if the scenario goal is met in the current state (page_summary, ariaDiff, state), instruct Tester to verify() and finish(). If goal was already true at the start, propose different input data so the test is meaningful. If Tester repeats the same successful action, STOP. + A goal of "action must not succeed" is met by any guard that blocks it — the predicted guard is a route, not a requirement. If needed you should pick the exact item the scenario should act on (from the page, or precondition() one) and pass it to tester diff --git a/src/ai/tester.ts b/src/ai/tester.ts index 3eecab02..5d974fa9 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -1126,14 +1126,9 @@ export class Tester extends TaskAgent implements Agent { } } - if (input.status !== null && task.isComplete()) { - if (this.pilot) { - const currentState = this.getCurrentState(); - await this.pilot.reviewCompletion(task, currentState, conversation, this.navigator); - } else { - const hasPassed = task.hasAchievedAny(); - task.finish(hasPassed ? TestResult.PASSED : TestResult.FAILED); - } + if (input.status !== null && task.isComplete() && !this.pilot) { + const hasPassed = task.hasAchievedAny(); + task.finish(hasPassed ? TestResult.PASSED : TestResult.FAILED); } const remainingExpectations = task.getRemainingExpectations();