From 387b3e62581e1d95f311d102918ea894b119dd8c Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 15 Sep 2026 11:17:14 +0300 Subject: [PATCH] Judge a blocked action on the block, not on the guard the plan named MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A plan for a negative scenario names the guard it expects — a validation message beside the field. An app may block the same action by disabling the control instead, which meets the goal just as well. Pilot read the missing message as a failure and told Tester to record the run failed, so a test that found the app behaving correctly was reported as a failure. Navigator had no URL assertion available either. Its verification vocabulary is closed, and no command in it could express "the page is at this address", so every URL claim failed while the page was on exactly that URL and Tester lost the cheapest evidence a navigation worked. seeInCurrentUrl and dontSeeInCurrentUrl were already accepted by Action and the recorder; only the vocabulary omitted them. Pilot's completion review was unreachable: Tester called it under task.isComplete(), and reviewDecision returns early on task.hasFinished, which is itself status === DONE || isComplete(). The trigger was also the bail-out, so the call never did anything. Removed; the no-Pilot branch keeps its behavior. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 13 +++++++++++++ rules/navigator/verification-actions.md | 22 ++++++++++++++++++++++ src/ai/pilot.ts | 1 + src/ai/tester.ts | 11 +++-------- 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 054ca15e..e7f01ae6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 2026-09-15 + +### Fixes + +- [Pilot] A scenario whose goal is that an invalid action must not succeed is no longer failed because the + app blocks it differently than the plan predicted. The plan names one guard — a validation message, say — + but the app may block by disabling the control or refusing the submit, and any of those met the goal. + Pilot told Tester to record such a run as failed, so tests that found the app working correctly were + reported as failures. +- [Navigator] `verify()` can now assert which page is open. There was no URL check among the assertions + Navigator is allowed to write, so every claim about the current URL failed even when the page was on + exactly that address, and Tester was left without evidence that a navigation had worked. + ## 2026-09-14 ### Changes diff --git a/rules/navigator/verification-actions.md b/rules/navigator/verification-actions.md index 91c59847..0ab208aa 100644 --- a/rules/navigator/verification-actions.md +++ b/rules/navigator/verification-actions.md @@ -50,6 +50,18 @@ Checks that page title contains expected text. I.seeInTitle('Dashboard'); +### I.seeInCurrentUrl + +I.seeInCurrentUrl() + +Checks that the current URL contains the expected path. Substring match — pass the path, not the full URL. +This is the ONLY way to assert the URL — page text and source are not evidence of it. + + + I.seeInCurrentUrl('/dashboard'); + I.seeInCurrentUrl('/users/42/edit'); + + ### I.seeInSource I.seeInSource() @@ -95,6 +107,16 @@ Checks that an input field does NOT contain the specified value. I.dontSeeInField('Email', ''); +### I.dontSeeInCurrentUrl + +I.dontSeeInCurrentUrl() + +Checks that the current URL does NOT contain the given path. + + + I.dontSeeInCurrentUrl('/login'); + + ### I.dontSeeInSource I.dontSeeInSource() diff --git a/src/ai/pilot.ts b/src/ai/pilot.ts index 2421e77f..b1df1d13 100644 --- a/src/ai/pilot.ts +++ b/src/ai/pilot.ts @@ -1116,6 +1116,7 @@ export class Pilot implements Agent { Already-achieved detection: if the scenario goal is met in the current state (page_summary, ariaDiff, state), instruct Tester to verify() and finish(). If goal was already true at the start, propose different input data so the test is meaningful. If Tester repeats the same successful action, STOP. + A goal of "action must not succeed" is met by any guard that blocks it — the predicted guard is a route, not a requirement. If needed you should pick the exact item the scenario should act on (from the page, or precondition() one) and pass it to tester diff --git a/src/ai/tester.ts b/src/ai/tester.ts index 3eecab02..5d974fa9 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -1126,14 +1126,9 @@ export class Tester extends TaskAgent implements Agent { } } - if (input.status !== null && task.isComplete()) { - if (this.pilot) { - const currentState = this.getCurrentState(); - await this.pilot.reviewCompletion(task, currentState, conversation, this.navigator); - } else { - const hasPassed = task.hasAchievedAny(); - task.finish(hasPassed ? TestResult.PASSED : TestResult.FAILED); - } + if (input.status !== null && task.isComplete() && !this.pilot) { + const hasPassed = task.hasAchievedAny(); + task.finish(hasPassed ? TestResult.PASSED : TestResult.FAILED); } const remainingExpectations = task.getRemainingExpectations();