From ea6824456bcfd8a672fc632c2f09db3b79987602 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Mon, 5 Oct 2026 20:34:58 -0700 Subject: [PATCH 1/2] fix(check): say when rules ran unverified, and name what would verify them Every unverified path (--anonymous, no token, no usable GitHub remote, a reconcile that cannot complete) now prints one single-line notice naming the cause and its fix, whether or not the project has runtime rules. The remote failure keeps the specific problem resolveRepositoryUrl identified, and organization_not_found shares its remedy with rule create through orgNotFoundRemedy(). --- .changeset/check-unverified-remedy.md | 5 + .../proposal.md | 61 ++++++ .../specs/cli-check/spec.md | 117 ++++++++++++ .../tasks.md | 23 +++ openspec/specs/cli-check/spec.md | 64 ++++++- packages/cli/src/agent/check.md | 12 +- packages/cli/src/rules/generate.ts | 15 +- packages/cli/src/rules/plan-check.ts | 173 ++++++++++++++---- packages/cli/test/demo-command.test.ts | 4 +- packages/cli/test/mixed-engine-check.test.ts | 4 +- packages/cli/test/runtime-check.test.ts | 110 ++++++++++- packages/cli/test/vale-orchestration.test.ts | 8 +- 12 files changed, 539 insertions(+), 57 deletions(-) create mode 100644 .changeset/check-unverified-remedy.md create mode 100644 openspec/changes/archive/2026-10-05-check-unverified-remedy/proposal.md create mode 100644 openspec/changes/archive/2026-10-05-check-unverified-remedy/specs/cli-check/spec.md create mode 100644 openspec/changes/archive/2026-10-05-check-unverified-remedy/tasks.md diff --git a/.changeset/check-unverified-remedy.md b/.changeset/check-unverified-remedy.md new file mode 100644 index 00000000..c5ac3935 --- /dev/null +++ b/.changeset/check-unverified-remedy.md @@ -0,0 +1,5 @@ +--- +"@taskless/cli": patch +--- + +`taskless check` now says when it ran rules without verifying them, and what would let it verify them. Logged out, `--anonymous`, no usable GitHub `origin`, or a reconcile that cannot complete each print one notice naming the fix: `auth login` or a `TASKLESS_TOKEN` in CI, dropping `--anonymous`, the specific remote problem, or the GitHub App installation steps `rule create` already gives. The notice prints whether or not the project has runtime rules, so an unauthenticated CI job no longer runs its ast-grep and Vale rules unverified without a word. Under `--json` it is an entry in `notices`; the exit code is unchanged. diff --git a/openspec/changes/archive/2026-10-05-check-unverified-remedy/proposal.md b/openspec/changes/archive/2026-10-05-check-unverified-remedy/proposal.md new file mode 100644 index 00000000..6fb306df --- /dev/null +++ b/openspec/changes/archive/2026-10-05-check-unverified-remedy/proposal.md @@ -0,0 +1,61 @@ +## Why + +When `check` cannot have the rule service verify a project's rules, it says +that runtime rules did not run but never says what would let them run. The +three early paths (`--anonymous`, no token, no resolvable GitHub remote) +attached a reason to each skipped runtime rule and printed no notice, so a +project with no runtime rules got no word at all that its ast-grep and Vale +rules ran unverified. The remote failure discarded the specific problem +`resolveRepositoryUrl` had already identified, and the `organization_not_found` +cause named the condition without the remedy `rule create` gives for it. + +Server verification of issued rules is new in 0.12.0. A CI job without a token +is the likeliest place to hit the unauthenticated path, and that path was +silent whenever there were no runtime rules. + +The standing spec required the opposite for the logged-out case: "SHALL NOT +emit a warning about missing authentication", from a design that kept routine +offline use quiet. That design also allowed "an informational line", and this +change uses that allowance: a `Notice:`, not a warning, with the exit code +unchanged. + +## What Changes + +- Every path that leaves rules unverified (`--anonymous`, no token, no + resolvable GitHub remote, a reconcile that cannot complete) prints ONE + notice saying the rules were not verified and naming the fix. It prints + whether or not the project has runtime rules. +- The fix named per cause: `--anonymous` names running without it; no token + names `auth login` and `TASKLESS_TOKEN`; a remote failure names which of + not-a-repository, no `origin`, or a non-GitHub `origin` it is; a rejected + token names `auth login`; `organization_not_found` carries the same two + steps as `orgNotFoundMessage()`, now shared through `orgNotFoundRemedy()`. +- Each skipped runtime rule's reason becomes the short cause, since the + notice carries the remedy. +- The `check` recipe goes to topic v6 and describes the notice. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `cli-check`: one ADDED requirement for the notice. Two MODIFIED + requirements, restated in full: "Check selects what it runs from auth state" + drops "SHALL NOT emit a warning about missing authentication", and "Check + accepts --anonymous as a no-op" lets the notice name `--anonymous` as the + cause instead of matching the unauthenticated output byte for byte. + +## Impact + +Additive stderr output on unverified runs, and an entry in the `--json` +`notices` array where there was none. `success`, `results`, `skipped` and the +exit code are unchanged; `skipped[].reason` is reworded. `patch`: the package +is pre-1.0. + +## Delivery shape + +**Single PR.** Spec, implementation, recipe and tests are one small diff, and +the change is archived in it. diff --git a/openspec/changes/archive/2026-10-05-check-unverified-remedy/specs/cli-check/spec.md b/openspec/changes/archive/2026-10-05-check-unverified-remedy/specs/cli-check/spec.md new file mode 100644 index 00000000..239e7be9 --- /dev/null +++ b/openspec/changes/archive/2026-10-05-check-unverified-remedy/specs/cli-check/spec.md @@ -0,0 +1,117 @@ +## ADDED Requirements + +### Requirement: Check names the remedy when rules run unverified + +Whenever `taskless check` runs rules without verifying them, because `--anonymous` is set, +no token is available, no GitHub repository URL resolves, or reconciliation cannot complete, +it SHALL emit exactly one notice that says the rules were not verified, states the cause, and +names what would let verification happen. The notice SHALL be emitted whether or not the +project has runtime rules. It SHALL NOT change the exit code. It SHALL be a single line, cause +and remedy together, so that under `--json` it is one entry in the `notices` array; in human +output it SHALL be written to stderr marked as a notice. `--dangerously-run-scripts` is excluded: it carries its own warning. + +The remedy SHALL be specific to the cause: + +- `--anonymous`: run `check` without `--anonymous`. +- No token: `taskless auth login`, or a `TASKLESS_TOKEN` where `check` runs in CI. +- No repository URL: which of the three problems applies (not a git repository, no `origin` + remote, or an `origin` that is not GitHub) and the step for that problem. +- A rejected token: re-authenticate with `taskless auth login`, or replace the token. +- `404 organization_not_found`: the same steps `rule create` gives for that code, confirming + the Taskless GitHub App installation covers the repository and re-authenticating with + `taskless auth login`. + +#### Scenario: Logged out with no runtime rules still says so + +- **WHEN** a user runs `taskless check` with no available token in a project with no runtime rules +- **THEN** the CLI SHALL emit one notice that the rules were not verified +- **AND** the notice SHALL name `auth login` +- **AND** the exit code SHALL NOT change because of the notice + +#### Scenario: Anonymous names the flag, not authentication + +- **WHEN** a user runs `taskless check --anonymous` +- **THEN** the notice SHALL name `--anonymous` as the cause and running without it as the remedy + +#### Scenario: A missing origin is named as such + +- **WHEN** an authenticated `check` runs in a git repository with no `origin` remote +- **THEN** the notice SHALL say the repository has no `origin` remote and how to add one + +#### Scenario: Organization not found carries the installation remedy + +- **WHEN** reconciliation returns `404 organization_not_found` +- **THEN** the notice SHALL name confirming the Taskless GitHub App installation and re-authenticating with `auth login` + +#### Scenario: The notice is a notices entry under --json + +- **WHEN** `taskless check --json` runs unverified +- **THEN** the `notices` array SHALL contain the notice +- **AND** `success`, `results`, and `skipped` SHALL keep their existing meaning + +## MODIFIED Requirements + +### Requirement: Check accepts --anonymous as a no-op + +The `taskless check` command SHALL accept the global `--anonymous` flag (per the `cli` +capability). Because `check` reconciles against the Taskless API when authenticated, +`--anonymous` SHALL force the logged-out path: it SHALL suppress the reconcile network call +and run all local static rules. Aside from forcing the logged-out path, `--anonymous` SHALL NOT +change scan behavior, output shape, or exit codes relative to an unauthenticated `check`. The +not-verified notice SHALL name `--anonymous` as its cause rather than authentication. + +#### Scenario: check --anonymous skips reconciliation + +- **WHEN** a user runs `taskless check --anonymous` +- **THEN** the CLI SHALL NOT call `POST /cli/api/v2/reconcile` +- **AND** SHALL scan all local static rules + +#### Scenario: check --anonymous matches an unauthenticated check + +- **WHEN** a user runs `taskless check --anonymous` +- **THEN** its scan behavior, output shape, and exit code SHALL match `taskless check` run with + no available token +- **AND** only the cause and remedy named in the not-verified notice SHALL differ + +### Requirement: Check selects what it runs from auth state + +`taskless check` SHALL NOT require authentication, and it SHALL choose what it verifies from +the current auth state. When a token is available and `--anonymous` is not set, the CLI SHALL +reconcile **every** rule (ast-grep, Vale, and runtime) and apply the verdict policy of the +`cli-rule-reconciliation` capability. When no token is available, when `--anonymous` is set, or +when reconciliation cannot complete, the CLI SHALL run every static rule (ast-grep and Vale) +unverified and SHALL skip runtime execution unless `--dangerously-run-scripts` is set. The +unauthenticated path SHALL succeed with no network access. It SHALL report that the rules were +not verified as an informational notice ("Check names the remedy when rules run unverified"), +and SHALL NOT fail or change the exit code because no token is available. + +#### Scenario: Unauthenticated check runs static rules and skips runtime rules + +- **WHEN** a user runs `taskless check` with no available token +- **THEN** the CLI SHALL scan all static rules +- **AND** SHALL NOT call `POST /cli/api/v2/reconcile` +- **AND** SHALL skip runtime rules +- **AND** SHALL emit the not-verified notice naming `auth login`, without changing the exit code + +#### Scenario: Authenticated check reconciles runtime rules + +- **WHEN** a user runs `taskless check` with an available token and without `--anonymous` +- **THEN** the CLI SHALL reconcile its ast-grep, Vale, and runtime rules in one request +- **AND** SHALL run or execute each rule according to its verdict + +#### Scenario: Anonymous forces the logged-out path + +- **WHEN** a user runs `taskless check --anonymous` while a token is available +- **THEN** the CLI SHALL behave exactly as an unauthenticated `check` (static rules run, runtime rules skipped, no reconcile call) + +#### Scenario: Logged out, an edited static rule still runs + +- **WHEN** a user runs `taskless check` with no available token and an issued sg or vale rule has been edited +- **THEN** the edited rule SHALL run with no signature enforcement +- **AND** runtime rules SHALL be skipped +- **AND** the exit code SHALL NOT change because the rule was edited + +#### Scenario: Logged in, an edited rule of any engine does not run + +- **WHEN** an authenticated `check` reconciles and a rule of any engine is `unsafe` +- **THEN** that rule SHALL NOT run or execute diff --git a/openspec/changes/archive/2026-10-05-check-unverified-remedy/tasks.md b/openspec/changes/archive/2026-10-05-check-unverified-remedy/tasks.md new file mode 100644 index 00000000..330a21eb --- /dev/null +++ b/openspec/changes/archive/2026-10-05-check-unverified-remedy/tasks.md @@ -0,0 +1,23 @@ +## 1. Spec + +- [x] 1.1 ADDED requirement for the not-verified notice; MODIFIED "Check + selects what it runs from auth state" and "Check accepts --anonymous as + a no-op", each restated in full. +- [x] 1.2 Dry-run `openspec archive` and compare the scenario count in + `cli-check` before and after. + +## 2. Implementation + +- [x] 2.1 `plan-check.ts`: one notice per unverified plan, with cause and + remedy, regardless of runtime rule count. +- [x] 2.2 Name the specific remote problem from the `CLIError` code. +- [x] 2.3 Extract `orgNotFoundRemedy()` from `orgNotFoundMessage()` and use it + for `organization_not_found` in `check`. +- [x] 2.4 `check` recipe topic v6. + +## 3. Tests + +- [x] 3.1 Logged out with and without runtime rules, human and `--json`. +- [x] 3.2 `--anonymous`, the three remote problems, `organization_not_found`. +- [x] 3.3 Tests that asserted `notices` absent on a clean logged-out `--json` + run now assert the not-verified notice is the only one. diff --git a/openspec/specs/cli-check/spec.md b/openspec/specs/cli-check/spec.md index c57e574b..e07deb1a 100644 --- a/openspec/specs/cli-check/spec.md +++ b/openspec/specs/cli-check/spec.md @@ -236,7 +236,8 @@ The `taskless check` command SHALL accept the global `--anonymous` flag (per the capability). Because `check` reconciles against the Taskless API when authenticated, `--anonymous` SHALL force the logged-out path: it SHALL suppress the reconcile network call and run all local static rules. Aside from forcing the logged-out path, `--anonymous` SHALL NOT -change scan behavior, output shape, or exit codes relative to an unauthenticated `check`. +change scan behavior, output shape, or exit codes relative to an unauthenticated `check`. The +not-verified notice SHALL name `--anonymous` as its cause rather than authentication. #### Scenario: check --anonymous skips reconciliation @@ -247,8 +248,9 @@ change scan behavior, output shape, or exit codes relative to an unauthenticated #### Scenario: check --anonymous matches an unauthenticated check - **WHEN** a user runs `taskless check --anonymous` -- **THEN** its scan behavior, output, and exit code SHALL match `taskless check` run with no - available token +- **THEN** its scan behavior, output shape, and exit code SHALL match `taskless check` run with + no available token +- **AND** only the cause and remedy named in the not-verified notice SHALL differ ### Requirement: Check error output uses standardized error envelope @@ -268,8 +270,9 @@ reconcile **every** rule (ast-grep, Vale, and runtime) and apply the verdict pol `cli-rule-reconciliation` capability. When no token is available, when `--anonymous` is set, or when reconciliation cannot complete, the CLI SHALL run every static rule (ast-grep and Vale) unverified and SHALL skip runtime execution unless `--dangerously-run-scripts` is set. The -unauthenticated path SHALL succeed with no network access and SHALL NOT emit a warning about -missing authentication. +unauthenticated path SHALL succeed with no network access. It SHALL report that the rules were +not verified as an informational notice ("Check names the remedy when rules run unverified"), +and SHALL NOT fail or change the exit code because no token is available. #### Scenario: Unauthenticated check runs static rules and skips runtime rules @@ -277,7 +280,7 @@ missing authentication. - **THEN** the CLI SHALL scan all static rules - **AND** SHALL NOT call `POST /cli/api/v2/reconcile` - **AND** SHALL skip runtime rules -- **AND** SHALL NOT emit a warning about missing authentication +- **AND** SHALL emit the not-verified notice naming `auth login`, without changing the exit code #### Scenario: Authenticated check reconciles runtime rules @@ -734,3 +737,52 @@ nothing to report. - **WHEN** reconciliation returns vale rule `bar-2` in `unknown` with `copyOf: { ruleId: "foo-1", revisionId: "r1", files: [{ path: ".vale.ini", expected, got }] }` and returns `foo-1` as `missing` with `revisionId` `r2` - **THEN** `integrity` SHALL include `{ ruleId: "bar-2", engine: "vale", verdict: "unknown", files: [{ path: ".vale.ini", expected, got }], copyOf: { ruleId: "foo-1", revisionId: "r1", sourceMissing: true } }` - **AND** SHALL include `{ ruleId: "foo-1", engine: "vale", verdict: "missing", revisionId: "r2" }` + +### Requirement: Check names the remedy when rules run unverified + +Whenever `taskless check` runs rules without verifying them, because `--anonymous` is set, +no token is available, no GitHub repository URL resolves, or reconciliation cannot complete, +it SHALL emit exactly one notice that says the rules were not verified, states the cause, and +names what would let verification happen. The notice SHALL be emitted whether or not the +project has runtime rules. It SHALL NOT change the exit code. It SHALL be a single line, cause +and remedy together, so that under `--json` it is one entry in the `notices` array; in human +output it SHALL be written to stderr marked as a notice. `--dangerously-run-scripts` is excluded: it carries its own warning. + +The remedy SHALL be specific to the cause: + +- `--anonymous`: run `check` without `--anonymous`. +- No token: `taskless auth login`, or a `TASKLESS_TOKEN` where `check` runs in CI. +- No repository URL: which of the three problems applies (not a git repository, no `origin` + remote, or an `origin` that is not GitHub) and the step for that problem. +- A rejected token: re-authenticate with `taskless auth login`, or replace the token. +- `404 organization_not_found`: the same steps `rule create` gives for that code, confirming + the Taskless GitHub App installation covers the repository and re-authenticating with + `taskless auth login`. + +#### Scenario: Logged out with no runtime rules still says so + +- **WHEN** a user runs `taskless check` with no available token in a project with no runtime rules +- **THEN** the CLI SHALL emit one notice that the rules were not verified +- **AND** the notice SHALL name `auth login` +- **AND** the exit code SHALL NOT change because of the notice + +#### Scenario: Anonymous names the flag, not authentication + +- **WHEN** a user runs `taskless check --anonymous` +- **THEN** the notice SHALL name `--anonymous` as the cause and running without it as the remedy + +#### Scenario: A missing origin is named as such + +- **WHEN** an authenticated `check` runs in a git repository with no `origin` remote +- **THEN** the notice SHALL say the repository has no `origin` remote and how to add one + +#### Scenario: Organization not found carries the installation remedy + +- **WHEN** reconciliation returns `404 organization_not_found` +- **THEN** the notice SHALL name confirming the Taskless GitHub App installation and re-authenticating with `auth login` + +#### Scenario: The notice is a notices entry under --json + +- **WHEN** `taskless check --json` runs unverified +- **THEN** the `notices` array SHALL contain the notice +- **AND** `success`, `results`, and `skipped` SHALL keep their existing meaning diff --git a/packages/cli/src/agent/check.md b/packages/cli/src/agent/check.md index 480c216e..ef5f7246 100644 --- a/packages/cli/src/agent/check.md +++ b/packages/cli/src/agent/check.md @@ -1,4 +1,4 @@ -# Topic: check (CLI v%(CLI_VERSION)s / topic v5) +# Topic: check (CLI v%(CLI_VERSION)s / topic v6) ## Goal Run the applicable rules against the codebase and report matches. Two @@ -40,7 +40,12 @@ logged in: - **Logged out, `--anonymous`, no GitHub remote, or service unavailable**: nothing is verified. ast-grep and Vale rules run as they are on disk, runtime rules are **skipped** (reported, never run), and - the exit code is unaffected. + the exit code is unaffected. `check` prints ONE notice saying the rules + were not verified and naming the fix: `%(TASKLESS_CLI)s auth login` (or + a `TASKLESS_TOKEN` secret in CI), dropping `--anonymous`, the specific + remote problem, or the GitHub App installation. It prints whether or not + the project has runtime rules, because the static rules ran unverified + either way. Pass the fix on to the user rather than ignoring it. - **`--dangerously-run-scripts`**: nothing is verified and no network call is made, logged in or not. Every rule of every engine runs, runtime included, behind a prominent warning. This is the only way to @@ -55,7 +60,8 @@ plan does not include restoring rules, the git steps that do instead Notices about skipped runtime rules are human-readable stderr only. Under `--json` they do NOT appear as warnings; instead an additive optional `skipped: [{ rule, reason }]` array is included alongside the -unchanged `{ success, results }`, and the CI backstop +unchanged `{ success, results }`, the not-verified notice is an entry +in `notices`, and the CI backstop (`%(TASKLESS_CLI)s agent ci`) is the enforcement point. ## An edited rule diff --git a/packages/cli/src/rules/generate.ts b/packages/cli/src/rules/generate.ts index 7a97c3cb..e892b9b5 100644 --- a/packages/cli/src/rules/generate.ts +++ b/packages/cli/src/rules/generate.ts @@ -50,11 +50,22 @@ export function orgNotFoundMessage(): string { "", "Most often the organization's Taskless GitHub App installation does not cover this repository. It can also mean your login no longer has access to the organization.", "", - "- Confirm the Taskless app is installed on this repository's owner and includes this repository.", - `- If access recently changed, re-authenticate with \`${getCliPrefix()} auth login\`.`, + ...orgNotFoundRemedy().map((step) => `- ${step}`), ].join("\n"); } +/** + * The steps that resolve `organization_not_found`, one sentence each. Shared + * with `check`, which reports the same condition and must name the same + * remedy. + */ +export function orgNotFoundRemedy(): string[] { + return [ + "Confirm the Taskless app is installed on this repository's owner and includes this repository.", + `If access recently changed, re-authenticate with \`${getCliPrefix()} auth login\`.`, + ]; +} + /** A request's terminal status. */ export type FinishedRequest = RequestStatus; diff --git a/packages/cli/src/rules/plan-check.ts b/packages/cli/src/rules/plan-check.ts index e77a557a..e558a74a 100644 --- a/packages/cli/src/rules/plan-check.ts +++ b/packages/cli/src/rules/plan-check.ts @@ -1,8 +1,13 @@ import { getToken } from "../auth/token"; import { resolveActingOrg } from "../auth/org"; import { reconcileRules, retryAdvice } from "../api/v2"; -import { resolveRepositoryUrl } from "../util/git-remote"; +import { CLIError } from "../util/cli-error"; +import { + resolveRepositoryPath, + resolveRepositoryUrl, +} from "../util/git-remote"; import { getCliPrefix } from "../util/package-manager"; +import { orgNotFoundRemedy } from "./generate"; import { recoveryAdvice } from "./recovery-advice"; import { reportRules } from "./report"; import type { RunDirectory } from "./run-directory"; @@ -48,6 +53,17 @@ export interface SkippedRuntimeRule { reason: string; } +/** Why rules went unverified, and what would let them be verified. */ +interface Unverified { + /** A short clause, also used as each skipped runtime rule's reason. */ + cause: string; + /** + * Sentences naming the fix, empty when there is none. They join the notice + * on one line: a `--json` notice never spans lines. + */ + remedy: string[]; +} + /** The plan outcome for an organization whose plan withholds runtime rules. */ export interface PlanEntitlement { runtimeSignatures: false; @@ -96,6 +112,24 @@ export async function planCheck( integrity: [], }; + /** + * A plan whose rules were not verified. It carries ONE notice, whether or + * not the project has runtime rules, because the static rules ran without + * the integrity check either way; the notice names what would make + * verification happen. Each skipped runtime rule carries the short cause. + */ + const unverified = (rules: RuntimeRule[], why: Unverified): CheckPlan => { + log.write(`unverified run: ${why.cause}`); + return { + ...empty, + skipped: rules.map((rule) => ({ + rule: rule.name, + reason: `${why.cause}, so it was not verified`, + })), + notices: [unverifiedNotice(why, rules.length)], + }; + }; + if (options.dangerouslyRunScripts) { log.write( "--dangerously-run-scripts: no reconcile; every rule runs unverified" @@ -103,33 +137,26 @@ export async function planCheck( return { ...empty, execute: discovered, notices: [RUN_SCRIPTS_WARNING] }; } - const unverified = (reason: string, notice?: string): CheckPlan => { - log.write(`unverified run: ${reason}`); - return { - ...empty, - skipped: discovered.map((rule) => ({ rule: rule.name, reason })), - notices: notice === undefined ? [] : [notice], - }; - }; - if (options.anonymous) { - return unverified( - "anonymous mode — runtime rules were not verified and did not run" - ); + return unverified(discovered, { + cause: "`--anonymous` was set", + remedy: ["Run `check` without `--anonymous` to verify them."], + }); } const token = await getToken(cwd, { silent: true }); if (!token) { - return unverified( - "not authenticated — runtime rules were not verified and did not run" - ); + return unverified(discovered, { + cause: "not authenticated", + remedy: [ + `Run \`${getCliPrefix()} auth login\`, or set \`TASKLESS_TOKEN\` where \`check\` runs in CI.`, + ], + }); } let repositoryUrl: string; try { repositoryUrl = await resolveRepositoryUrl(cwd); - } catch { - return unverified( - "no GitHub remote — runtime rules could not be verified and did not run" - ); + } catch (error) { + return unverified(discovered, await remoteProblem(cwd, error)); } const report = await reportRules(snapshot); @@ -212,16 +239,11 @@ export async function planCheck( }` ); if (outcome.status !== "ok") { - const cause = reconcileFailureCause(outcome); - const remaining = await discoverRuntimeRulesIn(runtimeRoot); return { - ...empty, - skipped: remaining.map((rule) => ({ rule: rule.name, reason: cause })), - notices: [ - `Rule verification could not be performed: ${cause}. Static rules ran unverified and runtime rules did not run.${ - outcome.status === "unavailable" ? retryAdvice(outcome) : "" - }`, - ], + ...unverified( + await discoverRuntimeRulesIn(runtimeRoot), + reconcileFailure(outcome) + ), failures, integrity, }; @@ -333,33 +355,104 @@ function withheldNotice(entitlement: PlanEntitlement): string { } /** - * Why reconcile gave no verdicts, as a clause for the notice and each skipped - * rule. + * Why reconcile gave no verdicts, and what would let it, for the notice and + * each skipped rule. * * "Unavailable" is kept for the service not answering. A documented code is an * answer, and calling it an outage sends the user to retry something that * will be rejected identically every time. */ -function reconcileFailureCause( +function reconcileFailure( outcome: Exclude>, { status: "ok" }> -): string { +): Unverified { switch (outcome.status) { case "unauthorized": { - return `authentication was rejected — run \`${getCliPrefix()} auth login\` to re-authenticate`; + return { + cause: "authentication was rejected", + remedy: [ + `Re-authenticate with \`${getCliPrefix()} auth login\`, or replace an expired \`TASKLESS_TOKEN\`.`, + ], + }; } case "unavailable": { - return `the rule service was unavailable (${outcome.reason})`; + const retry = retryAdvice(outcome).trim(); + return { + cause: `the rule service was unavailable (${outcome.reason})`, + remedy: retry ? [retry] : [], + }; } case "refused": { - return "the rule service answered with an unexpected refusal"; + return { + cause: "the rule service answered with an unexpected refusal", + remedy: [], + }; } case "error": { if (outcome.code === "organization_not_found") { - return "the Taskless GitHub App installation does not cover this repository, or your login lost access to the organization"; + return { + cause: + "the Taskless GitHub App installation does not cover this repository, or your login lost access to the organization", + remedy: orgNotFoundRemedy(), + }; } - return `the rule service rejected the verification request (${outcome.code}${ - outcome.details?.length ? `: ${outcome.details.join(", ")}` : "" - })`; + return { + cause: `the rule service rejected the verification request (${outcome.code}${ + outcome.details?.length ? `: ${outcome.details.join(", ")}` : "" + })`, + remedy: [], + }; + } + } +} + +function unverifiedNotice(why: Unverified, runtimeRules: number): string { + const ran = + runtimeRules === 0 + ? "Static rules ran without verification." + : `Static rules ran without verification, and ${String(runtimeRules)} runtime rule(s) did not run.`; + return [`Rules were not verified: ${why.cause}.`, ran, ...why.remedy].join( + " " + ); +} + +/** + * Name the specific reason no repository URL resolved, and its fix. The three + * codes are the populations `resolveRepositoryUrl` already tells apart; their + * own messages are about remote rule generation, so `check` words its own. + */ +async function remoteProblem(cwd: string, error: unknown): Promise { + const code = error instanceof CLIError ? error.code : undefined; + switch (code) { + case "NOT_A_GIT_REPOSITORY": { + return { + cause: "this directory is not a git repository", + remedy: [ + "Verification identifies the project by its GitHub `origin` remote, so run `check` in a clone of the repository.", + ], + }; + } + case "NO_ORIGIN_REMOTE": { + return { + cause: "this repository has no `origin` remote", + remedy: [ + "Add the GitHub repository as `origin` with `git remote add origin https://github.com//`.", + ], + }; + } + case "UNSUPPORTED_REMOTE_HOST": { + const path = await resolveRepositoryPath(cwd); + return { + cause: `\`origin\` is not a GitHub remote${path ? ` (${path})` : ""}`, + remedy: [ + "Verification supports GitHub repositories only. Point `origin` at the repository's GitHub URL.", + ], + }; + } + default: { + return { + cause: "no GitHub `origin` remote could be resolved", + remedy: [], + }; } } } diff --git a/packages/cli/test/demo-command.test.ts b/packages/cli/test/demo-command.test.ts index 46daaf93..7e5642fd 100644 --- a/packages/cli/test/demo-command.test.ts +++ b/packages/cli/test/demo-command.test.ts @@ -168,7 +168,7 @@ describe("the runtime sample does not weaken the execution gate", () => { await run(["demo", "runtime", "-d", project]); }); - it("is skipped by an unauthenticated check, with the existing reason", async () => { + it("is skipped by an unauthenticated check, with the unauthenticated reason", async () => { const { stdout, stderr, exitCode } = await run(["check", "-d", project]); expect(exitCode).toBe(0); // The skip is a NOTICE on stderr, not a finding on stdout: `check` found no @@ -176,7 +176,7 @@ describe("the runtime sample does not weaken the execution gate", () => { // opposite of what the gate means. expect(stdout).toContain("No issues found."); expect(stderr).toContain("env-keys-declared"); - expect(stderr).toContain("runtime rules were not verified and did not run"); + expect(stderr).toContain("not authenticated, so it was not verified"); }); it("does not execute its fixtures without the documented flag", async () => { diff --git a/packages/cli/test/mixed-engine-check.test.ts b/packages/cli/test/mixed-engine-check.test.ts index 26dd4932..dee5eb62 100644 --- a/packages/cli/test/mixed-engine-check.test.ts +++ b/packages/cli/test/mixed-engine-check.test.ts @@ -341,7 +341,9 @@ describe("check over a project with both engines", () => { expect(output.failures?.[0]).toContain("Vale did not run"); expect(output.failures?.[0]).toContain("no-simply/.vale.ini line 1:"); expect(output.failures?.[0]).toContain("no-simply.no-simply"); - expect(output.notices).toBeUndefined(); + expect(output.notices).toEqual([ + expect.stringMatching(/^Rules were not verified: not authenticated\./), + ]); // The other engine is unaffected: ast-grep still reports, and Vale, which // was refused, reports nothing rather than something partial. diff --git a/packages/cli/test/runtime-check.test.ts b/packages/cli/test/runtime-check.test.ts index 19b14211..d6ede31e 100644 --- a/packages/cli/test/runtime-check.test.ts +++ b/packages/cli/test/runtime-check.test.ts @@ -330,6 +330,97 @@ describe("check: static vs runtime dispatch", () => { expect(stdout).toContain("no-console"); }); + it("logged out: one notice says rules were not verified and names auth login", async () => { + const { stderr } = await runCli(["check", "-d", directory]); + expect(stderr.split("Rules were not verified")).toHaveLength(2); + expect(stderr).toContain( + "Notice: Rules were not verified: not authenticated." + ); + expect(stderr).toContain("1 runtime rule(s) did not run"); + expect(stderr).toMatch( + /did not run\. Run `.+ auth login`, or set `TASKLESS_TOKEN`/ + ); + }); + + it("logged out with no runtime rules: the notice still prints, and is in --json notices", async () => { + await rm(join(directory, ".taskless", "runtime"), { + recursive: true, + force: true, + }); + const human = await runCli(["check", "-d", directory]); + expect(human.stderr).toContain( + "Notice: Rules were not verified: not authenticated. Static rules ran without verification." + ); + expect(human.stderr).not.toContain("runtime rule"); + expect(human.stderr).toContain("auth login"); + + const { stdout, exitCode } = await runCli([ + "check", + "-d", + directory, + "--json", + ]); + const output = parseJson(stdout); + expect(exitCode).toBe(0); + expect(output.skipped).toBeUndefined(); + expect(output.notices?.join("\n")).toMatch( + /Rules were not verified: not authenticated/ + ); + }); + + it("--anonymous: the notice names dropping --anonymous, not auth login", async () => { + const { stderr } = await runCli(["check", "-d", directory, "--anonymous"]); + expect(stderr).toContain("Rules were not verified: `--anonymous` was set."); + expect(stderr).toContain("without `--anonymous`"); + expect(stderr).not.toContain("auth login"); + }); + + it.each([ + { + name: "no origin remote", + arrange: (cwd: string) => + execFileAsync("git", ["remote", "remove", "origin"], { cwd }), + cause: "this repository has no `origin` remote", + remedy: "git remote add origin", + }, + { + name: "a non-GitHub origin", + arrange: (cwd: string) => + execFileAsync( + "git", + [ + "remote", + "set-url", + "origin", + "https://gitlab.com/acme/widgets.git", + ], + { cwd } + ), + cause: "`origin` is not a GitHub remote (gitlab.com/acme/widgets)", + remedy: "supports GitHub repositories only", + }, + { + name: "not a git repository", + arrange: (cwd: string) => + rm(join(cwd, ".git"), { recursive: true, force: true }), + cause: "this directory is not a git repository", + remedy: "so run `check` in a clone of the repository", + }, + ])( + "authenticated with $name: the notice names that remote problem", + async ({ arrange, cause, remedy }) => { + await arrange(directory); + const { stderr, exitCode } = await runCli(["check", "-d", directory], { + TASKLESS_TOKEN: "fake.token", + // Never reached: the remote is resolved before any network call. + TASKLESS_API_URL: "http://127.0.0.1:9/cli", + }); + expect(exitCode).toBe(0); + expect(stderr).toContain(`Rules were not verified: ${cause}.`); + expect(stderr).toContain(remedy); + } + ); + /** Run \`check\` authenticated against a mock that answers with \`responder\`. */ async function authedCheck( responder: Responder, @@ -676,11 +767,28 @@ describe("check: static vs runtime dispatch", () => { expect(output.results.some((r) => r.ruleId === "no-console")).toBe(true); expect(output.skipped?.some((s) => s.rule === "demo")).toBe(true); expect(output.notices?.join("\n")).toMatch( - /verification could not be performed/ + /Rules were not verified: the rule service was unavailable/ ); expect(output.notices?.join("\n")).toMatch(/try again/); }); + it("organization not found: the notice names the app installation and auth login", async () => { + const { stderr, exitCode } = await authedCheck( + () => ({ statusCode: 404, body: { error: "organization_not_found" } }), + [] + ); + expect(exitCode).toBe(0); + expect(stderr).toContain( + "Rules were not verified: the Taskless GitHub App installation does not cover this repository" + ); + expect(stderr).toContain( + "Confirm the Taskless app is installed on this repository's owner" + ); + expect(stderr).toMatch( + /If access recently changed, re-authenticate with `.+ auth login`/ + ); + }); + it("reconcile validation_error: reported as a rejection, never as an outage", async () => { const { stdout, exitCode } = await authedCheck(() => ({ statusCode: 400, diff --git a/packages/cli/test/vale-orchestration.test.ts b/packages/cli/test/vale-orchestration.test.ts index 6b37ddf9..71e0de01 100644 --- a/packages/cli/test/vale-orchestration.test.ts +++ b/packages/cli/test/vale-orchestration.test.ts @@ -530,14 +530,18 @@ describe("an engine failure under --json", () => { expect(output.failures?.[0]).toContain("sg engine failed"); }); - it("omits both fields when nothing failed, as `skipped` does", async () => { + it("omits failures, and carries no engine notice, when nothing failed", async () => { const cwd = makeMixedProject({ valeRules: false }); const { stdout } = await runCli(["check", "-d", cwd, "doc.md", "--json"]); const output = parseJson(stdout); expect(output.success).toBe(true); expect(output.failures).toBeUndefined(); - expect(output.notices).toBeUndefined(); + // The not-verified notice is the only one: a logged-out run always + // carries it, and nothing else had anything to say. + expect(output.notices).toEqual([ + expect.stringMatching(/^Rules were not verified: not authenticated\./), + ]); }); }); From 30408e357ce1539a3f460f3f452bfd4f572762c8 Mon Sep 17 00:00:00 2001 From: Jakob Heuser Date: Mon, 5 Oct 2026 21:26:01 -0700 Subject: [PATCH 2/2] test(check): cover the rejected-token notice --- packages/cli/test/runtime-check.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/cli/test/runtime-check.test.ts b/packages/cli/test/runtime-check.test.ts index d6ede31e..63f36fc2 100644 --- a/packages/cli/test/runtime-check.test.ts +++ b/packages/cli/test/runtime-check.test.ts @@ -802,6 +802,24 @@ describe("check: static vs runtime dispatch", () => { expect(notices).not.toMatch(/unavailable|try again/); }); + it("token rejected: the notice names auth login and replacing TASKLESS_TOKEN", async () => { + const { stderr, exitCode } = await authedCheck( + () => ({ statusCode: 401 }), + [] + ); + expect(exitCode).toBe(0); + expect(stderr.split("Rules were not verified")).toHaveLength(2); + expect(stderr).toContain( + "Rules were not verified: authentication was rejected." + ); + expect(stderr).toMatch( + /Re-authenticate with `.+ auth login`, or replace an expired `TASKLESS_TOKEN`\./ + ); + expect(stderr).toContain( + "runtime rule demo was not run — authentication was rejected, so it was not verified." + ); + }); + it("--anonymous with a token: skips runtime and never calls reconcile", async () => { const { stdout, server } = await authedCheck( (request) => ({ statusCode: 200, body: answer(request) }),