diff --git a/app/src/ai/skills/bundled_tests.rs b/app/src/ai/skills/bundled_tests.rs index 53c5612da3b..718a5e708dd 100644 --- a/app/src/ai/skills/bundled_tests.rs +++ b/app/src/ai/skills/bundled_tests.rs @@ -86,9 +86,7 @@ fn factory_files_bundled_skill_is_always_active_and_scoped_to_authoring() { )); for reference in [ - "references/schema.md", "references/scorers.md", - "references/triggers.md", "references/examples.md", "references/validation.md", "scripts/validate_factory_files.py", @@ -104,104 +102,52 @@ fn factory_files_bundled_skill_is_always_active_and_scoped_to_authoring() { } } -/// The schemas are the contract the skill tells agents to author against, so -/// they have to stay parseable and keep the Factory document's shape. +/// The skill must not carry a copy of the Factory file format. +/// +/// A bundled copy ships inside a Warp release and goes stale against the +/// warp-server it is used against. A stale copy does not fail quietly: it +/// reports fields the server accepts as unknown, and an agent clearing that +/// diagnostic deletes working configuration. An earlier revision did exactly +/// that to the Linear and Slack trigger aliases. The format is fetched from +/// the server now, so nothing here should describe it. #[test] -fn factory_files_schemas_are_parseable_and_keep_the_factory_contract() { - let schemas_dir = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../resources/bundled/skills/factory-files/schemas"); - fn assert_refs_resolve(value: &serde_json::Value, current_name: &str, schemas_dir: &Path) { - match value { - serde_json::Value::Object(object) => { - if let Some(reference) = object.get("$ref").and_then(serde_json::Value::as_str) { - let (name, fragment) = reference.split_once('#').unwrap_or((reference, "")); - let target_name = if name.is_empty() { current_name } else { name }; - let raw = std::fs::read_to_string(schemas_dir.join(target_name)) - .unwrap_or_else(|error| panic!("read $ref target {target_name}: {error}")); - let target: serde_json::Value = serde_json::from_str(&raw) - .unwrap_or_else(|error| panic!("parse $ref target {target_name}: {error}")); - assert!( - fragment.is_empty() || target.pointer(fragment).is_some(), - "{current_name} contains unresolved $ref {reference}" - ); - } - for child in object.values() { - assert_refs_resolve(child, current_name, schemas_dir); - } - } - serde_json::Value::Array(values) => { - for child in values { - assert_refs_resolve(child, current_name, schemas_dir); - } +fn factory_files_skill_carries_no_copy_of_the_format() { + let skill_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../resources/bundled/skills/factory-files") + .canonicalize() + .expect("factory-files skill directory"); + + let mut schemas = Vec::new(); + let mut pending = vec![skill_dir.clone()]; + while let Some(directory) = pending.pop() { + for entry in std::fs::read_dir(&directory).expect("read skill directory") { + let path = entry.expect("read skill entry").path(); + if path.is_dir() { + pending.push(path); + } else if path.to_string_lossy().ends_with(".schema.json") { + schemas.push(path); } - _ => {} } } - - for name in [ - "common.schema.json", - "factory.schema.json", - "agent.schema.json", - "automation.schema.json", - "runner.schema.json", - "scorer.schema.json", - ] { - let raw = std::fs::read_to_string(schemas_dir.join(name)) - .unwrap_or_else(|error| panic!("read {name}: {error}")); - let schema: serde_json::Value = serde_json::from_str(&raw) - .unwrap_or_else(|error| panic!("{name} should be valid JSON: {error}")); - assert_eq!( - schema.get("$id").and_then(serde_json::Value::as_str), - Some(name), - "{name} should use a relative $id so sibling $refs resolve locally" - ); - assert_refs_resolve(&schema, name, &schemas_dir); - } - - let factory: serde_json::Value = serde_json::from_str( - &std::fs::read_to_string(schemas_dir.join("factory.schema.json")).unwrap(), - ) - .unwrap(); - let required: Vec<&str> = factory["required"] - .as_array() - .expect("factory schema declares required fields") - .iter() - .map(|value| value.as_str().expect("required entries are strings")) - .collect(); - assert_eq!( - required, - ["schemaVersion", "name", "repositories", "agentDefaults"] - ); - // These schemas ship inside a Warp release and are routinely older than the - // warp-server they validate against, so they stay open on purpose: a closed - // schema would reject configuration a newer server accepts. Flipping either - // assertion to `false` is a regression, not a tightening. See - // specs/REMOTE-2727/TECH.md. - assert_eq!( - factory["additionalProperties"], - serde_json::Value::Bool(true) - ); - // Both the current key and the legacy alias stay accepted; the server reads - // cloudProviders first and falls back to providers. - assert!(factory["properties"].get("cloudProviders").is_some()); - assert!(factory["properties"].get("providers").is_some()); - - let scorer: serde_json::Value = serde_json::from_str( - &std::fs::read_to_string(schemas_dir.join("scorer.schema.json")).unwrap(), - ) - .unwrap(); - assert_eq!( - scorer["additionalProperties"], - serde_json::Value::Bool(true) + assert!( + schemas.is_empty(), + "the skill has regrown bundled schemas, which go stale against the server \ + and produce false rejections; fetch the format instead: {schemas:?}" ); - for required in ["agents", "labels", "passingScore", "model"] { + + let validator = std::fs::read_to_string(skill_dir.join("scripts/validate_factory_files.py")) + .expect("read the validator"); + for banned in ["import yaml", "def load_yaml", "jsonschema"] { assert!( - scorer["required"].as_array().is_some_and(|fields| { - fields.iter().any(|field| field.as_str() == Some(required)) - }), - "scorer schema should require {required}" + !validator.contains(banned), + "the validator parses the format again ({banned}); it should send bytes \ + to the server and relay the verdict" ); } + assert!( + validator.contains("/api/v1/factory-files/validate"), + "the validator should reach the server's validation endpoint" + ); } #[test] diff --git a/resources/bundled/skills/factory-files/SKILL.md b/resources/bundled/skills/factory-files/SKILL.md index 5535f9d191c..abe549d19a1 100644 --- a/resources/bundled/skills/factory-files/SKILL.md +++ b/resources/bundled/skills/factory-files/SKILL.md @@ -9,6 +9,12 @@ A software factory can be defined by files in a repository. This skill covers authoring and editing those files, and validating them before you open a pull request. +warp-server owns the format. It publishes the schema for each version it +supports and validates a tree with the same parser the apply path uses. This +skill carries no copy of the format: a copy ships inside a Warp release, goes +stale against the server, and then reports confident, wrong diagnostics. When +the server cannot be reached, the answer is that the tree was not checked. + Use this skill for repository files. It is not the skill for operating a live factory: use `factory-mcp` to send work to a factory, inspect task status, or pull a task down locally. Playbooks under a factory's own `skills/` directories @@ -19,7 +25,9 @@ not a schema change, so this skill's rules do not apply to their contents. Every Factory tree is rooted at the directory containing `factory.yaml`. All paths below are relative to that root. A repository may register a subdirectory as the root, so find `factory.yaml` rather than assuming the -repository root. +repository root. Do not follow symlinks while looking: the server parses the +repository tree, where a symlink is stored as its target path rather than its +target's content. If there is no `factory.yaml`, this is not a Factory tree and nothing here applies. `agents//agent.md` and similar paths are also used by other @@ -50,66 +58,78 @@ unless the user asks you to normalize the tree. Scorer's body is its rubric. Never fold either into frontmatter. 3. Prefer the smallest edit that satisfies the request. -## Author against the schema -The current server parser rejects unknown fields, but bundled schemas can be -older than the server. They therefore validate known fields while preserving -unknown properties and newer catalogue values. Do not invent a field when a -documented one exists, and do not delete an existing unknown field. Duplicate -keys, YAML anchors, aliases, explicit tags, and multiple documents remain -invalid. - -The bundled JSON Schemas are the machine-readable contract: +## Author against the server's schema +Read the tree's `schemaVersion` from `factory.yaml`; a tree that omits it is +`v1alpha1`. Then fetch the schema for that version: -``` -schemas/factory.schema.json factory.yaml -schemas/agent.schema.json agents//agent.md frontmatter -schemas/automation.schema.json automations//automation.md frontmatter -schemas/runner.schema.json runners/.yaml -schemas/scorer.schema.json scorers//scorer.md frontmatter -schemas/common.schema.json shared definitions referenced by the above +```bash +curl -s https://app.warp.dev/api/v1/factory-files/schemas +curl -s https://app.warp.dev/api/v1/factory-files/schemas/ ``` -Read `references/schema.md` for the field-by-field reference, defaults, and -inheritance rules. Read `references/triggers.md` before writing or changing an -automation trigger: filter keys are specific to each provider and event, and -the parser does not catch a wrong one. Read `references/scorers.md` before -writing or changing a Scorer. Read `references/examples.md` for worked -examples of each resource. +The registry lists the versions the server supports. The version endpoint +returns every document describing one version, keyed by file name: +`factory.schema.json` for `factory.yaml`, `agent.schema.json`, +`automation.schema.json`, `runner.schema.json` and `scorer.schema.json` for the +corresponding resources, and `common.schema.json` for the definitions they +share. Both endpoints are unauthenticated. They are exact for the version they +describe: an unknown field is an error, and each enumerated value is one the +server accepts today. + +If the server does not publish the declared version, stop. Do not measure the +tree against a version it does not claim to be, and never lower +`schemaVersion` to make a check pass. + +Read `references/examples.md` for worked examples of each resource, and +`references/scorers.md` before writing or changing a Scorer. The field-by-field +catalogue is not duplicated here any more; the fetched schema carries it, with +a description on each field. ## Validate before opening a pull request -Run the bundled validator with Python 3.8 or newer, using the host's command (`python3`, -`python`, or `py -3`). Quote both paths because an app-bundle path can contain -spaces. +Run the bundled validator with Python 3.8 or newer, using the host's command +(`python3`, `python`, or `py -3`). Quote both paths because an app-bundle path +can contain spaces. ```bash python3 "{{skill_dir}}/scripts/validate_factory_files.py" "" ``` -Add `--json` for machine-readable output. A non-zero exit means at least one -problem; fix every reported problem and re-run until it is clean. +It selects the tree's resource files and submits them to the server, which runs +the real parser. Add `--json` for machine-readable output and `--server-root +`, or `WARP_SERVER_ROOT`, to point at a local, staging, or self-hosted +server. No credential is required; `WARP_API_KEY` is forwarded when the +environment already carries one, as an agent sandbox does. -If no Python 3 interpreter is available, do not install one or claim the tree -was validated without the user's approval. Check the changed document against -the corresponding JSON Schema manually and report that automated validation -was unavailable. +The exit code distinguishes three outcomes, and so must you: -The validator checks known field structure, mutual exclusions, trigger and -Scorer semantics, cron syntax, runner platform rules, and tree-level rules -(exactly one MAIN agent, Agent references, duplicate resource names). Unknown -properties and newer catalogue values pass through for version skew. It does -not resolve server state: model IDs, environment IDs, secret names, runner -names, Scorer model IDs, MCP server IDs, and integration availability are all -validated when the plan is applied. Report that distinction rather than -claiming a tree is fully verified. +- `0` the server checked the tree and found no problem. +- `1` the server checked the tree and reported diagnostics. Fix every one and + re-run until it is clean. +- `2` the tree was **not** checked. This is not a pass and not a failure; it + says nothing about the files at all. -If a `warp-server` checkout is available, its parser tests are the authority. -Run them from that checkout, not from the Factory repository: +### Never imply a check that did not happen +On exit `2`, say plainly that validation did not run and why. Do not describe +the files as valid, correct, or ready, and do not substitute your own reading +of the schema for a verdict. If you cannot reach a server and the change +matters, say so and let the user decide. -```bash -go test ./logic/factoryfile -``` +On exit `0`, repeat the sentence the validator prints rather than paraphrasing +it into something stronger. A pass means the parser and the state-independent +checks agreed; it does not mean the tree will apply. -When the Factory is already registered, a server plan is the strongest +Validation resolves no server state. Model IDs, environment IDs, secret names, +runner names, Scorer model IDs, MCP server IDs, integration availability, and +the values of Linear and Slack name aliases are all checked when the plan is +applied. The response lists what it did not check, including any deferred name +aliases; report that distinction rather than claiming a tree is fully verified. + +If no Python 3 interpreter is available, do not install one or claim the tree +was validated without the user's approval. Check the changed document against +the fetched schema by hand and report that automated validation was +unavailable. + +When the Factory is already registered, a server plan remains the strongest available check. See `references/validation.md` for diagnostic codes and how to read them. @@ -128,31 +148,22 @@ read them. non-empty `filter.schedule_ids`, and never both. - Linux runners require `platform.linux.dockerImage`. A runner with no `platform` section defaults to Linux and will fail for that reason. - -## Schema drift and version skew -The format is `v1alpha1` and still changing. `logic/factoryfile` in -`warp-server` is the authority; these schemas only mirror it. - -They also ship inside your Warp version rather than coming from the server, so -they can be older than the server the Factory syncs against. A field the server -added after your version was built will be reported here as unknown. - -Because of that: - -- Never delete, rename, or rewrite a field only because the validator calls it - unknown. On a file you did not author, that is at least as likely to be a - newer field as a mistake. Leave it, and say the schemas may be behind. -- Treat unknown-field reports on your own new edits as real. You are the one - who just introduced the field. -- If the server and these schemas disagree, the server is right. Say the - bundled schemas look stale rather than working around the validator by - skipping it. -- If the validator reports that it does not describe the tree's - `schemaVersion`, it stopped instead of applying `v1alpha1` rules to a format - it does not know. Validate with the server; never downgrade `schemaVersion` - to make the local run pass. - -If you are editing the bundled schemas themselves rather than a Factory tree, -their openness is deliberate and load-bearing. Read the "If you are changing -these schemas" section of `references/validation.md` before tightening -anything. +- Trigger filter keys depend on the `(provider, event)` pair. Some fields have + a friendlier authoring spelling that the server rewrites for you: GitHub + `baseBranches` and `prNumbers`, Linear `teams`, `projects`, `states` and + `issues`, and Slack `channels`, `users` and `itemUsers`. Each stands in for + its canonical key, and declaring both is an error. The Linear and Slack ones + name objects the server looks up at apply time, so they take a plain list of + names rather than an `in`/`not_in` matcher. + +## Do not add a local copy of the format +It is tempting to bundle the schema, or to reimplement a few checks here so +authoring works offline. Both have been tried and removed. A copy inside a Warp +release is routinely older than the server it is used against, and a stale copy +does not fail quietly: it reports a valid field as unknown, and an agent trying +to get to a clean run deletes working configuration to satisfy it. That has +already happened once, to Linear and Slack trigger aliases the server accepts. + +Reporting that a tree was not checked costs a little. Reporting the wrong +answer costs correct configuration. Fetch the format when you need it; say +nothing when you cannot. diff --git a/resources/bundled/skills/factory-files/references/schema.md b/resources/bundled/skills/factory-files/references/schema.md deleted file mode 100644 index 789306d2880..00000000000 --- a/resources/bundled/skills/factory-files/references/schema.md +++ /dev/null @@ -1,142 +0,0 @@ -# v1alpha1 field reference -Mirrors `logic/factoryfile` in `warp-server`. The JSON Schemas under -`schemas/` are the machine-readable form of everything here. - -Two layers enforce these rules. The **parser** reads the tree and produces -`FF_*` diagnostics; it validates shape, field names, and enums. **Resolution -and apply** validate everything that needs server state, plus a few rules the -parser leaves alone (integration provider slugs, runner platform, instance -shapes, harness model catalogues). A file can parse cleanly and still be -rejected when the plan is applied. - -## factory.yaml -Required: `schemaVersion`, `name`, `repositories`, `agentDefaults`. - -- `schemaVersion` — `v1alpha1`, the only version these schemas describe. A tree - declaring a different version is reported as unvalidatable rather than - checked against v1alpha1 rules; never downgrade the value to silence that. -- `name` — non-empty string. -- `description` — free text. -- `alias` — display handle, used as the factory's @-mention name on integrated - platforms. Letters, digits, spaces, `-`, `_`, `.`; at most 60 characters. - The server trims surrounding whitespace before counting and storing it. - Case is preserved; uniqueness is compared case-insensitively across the - workspace. There is no lowercase or hyphenation requirement. -- `credentialStrategy` — `EXECUTOR` or `CREATOR`. Omitting it leaves the value - already stored on the server untouched; it does not reset to a default. - Explicit null has the same undeclared meaning. -- `repositories` — at least one `{owner, name}` pair, no other keys, no - duplicates. -- `secrets` — list of managed secret names. Duplicates are rejected here. -- `mcpServers` — map of server name to `{warpId}`. `warpId` is the only key an - entry may carry. -- `cloudProviders.gcp` — `projectNumber`, `workloadIdentityFederationPoolId`, and - `workloadIdentityFederationProviderId` are all required; - `serviceAccountEmail` is optional. Quote `projectNumber` so YAML keeps it a - string. -- `cloudProviders.aws` — `roleArn` required. -- `providers` — legacy read-only alias for `cloudProviders`. New files should - use `cloudProviders`; when both exist, the server uses `cloudProviders`. -- `integrations` — list of `{type}`. Current known types are `jira`, `linear`, - and `slack`; preserve newer provider slugs. - The current server rejects `github` because repository access comes from - `repositories`; an older bundled schema leaves the final catalogue decision - to a server plan. An empty list explicitly detaches every provider; omitting - the section leaves the server-owned set alone. -- `agentDefaults` — execution defaults every agent inherits. Accepts `model` - XOR `harness` (one is required), plus `runner`, `environmentId`, `secrets`, - `mcpServers`, and `workerHost`. When `harness` is used here, both - `harness.type` and `harness.model` are required. - -## `agents//agent.md` -The directory name is the agent name. All frontmatter fields are optional; a -file with empty frontmatter is valid. The Markdown body is the agent's prompt. - -- `description` -- `agentType` — `CUSTOM`, `MAIN`, `FOREMAN`, `TRIAGE`, `SPEC`, `IMPLEMENT`, - `REVIEW`, `VERIFY`. `MAIN` is an authoring alias for `FOREMAN`. Omitting it - resolves to `CUSTOM`. Exactly one agent in the tree must be `MAIN`/`FOREMAN`. -- `credentialStrategy` — as above. -- `model` / `harness` — mutually exclusive override. Null `model` inherits. -- `runner` — a runner name. It may name a runner declared under `runners/`, or - an existing team runner that the tree does not declare. Null inherits. -- `environmentId` — null inherits. -- `secrets` — replaces the inherited list. -- `mcpServers` — replaces the inherited map. -- `workerHost` — self-hosted worker host. Null or empty clears an inherited - host and defers to the workspace default. - -## `automations//automation.md` -The directory name is the automation name. `triggers` is required and must -have at least one entry. The Markdown body is the run prompt. - -- `enabled` — boolean, defaults to true. -- `agent` — name of a declared agent. Defaults to the MAIN agent. -- `model` / `harness`, `runner`, `environmentId`, `secrets`, `mcpServers`, - `workerHost` — same semantics as on an agent. -- `triggers` — see `triggers.md`. - -## `runners/.yaml` -The file name is the runner name. All fields are optional to the parser, but -apply-time platform validation makes some effectively required. - -- `description` -- `setupCommands` — list of shell commands. -- `instanceShape` — when present, both `vcpus` and `memoryGb` are required. - Linux requires both to be positive powers of two, with no format-level upper - bound. macOS accepts only `4/7`, `6/14`, `8/14`, `12/28`, and `12/56`. -- `platform.os` — `linux` or `macos`, defaulting to `linux`. -- `platform.arch` — `x86_64` or `aarch64`, defaulting to `x86_64` on Linux and - `aarch64` on macOS. Supported pairs are `linux/x86_64`, `linux/aarch64`, and - `macos/aarch64`. -- `platform.linux.dockerImage` — required for every Linux runner, including one - that only defaults to Linux by omitting `platform.os`. -- `platform.mac.version` — `14`, `15`, `26`, or `27`. The whole `mac` section - may be omitted, which defaults the version to `26`; if the section is - present, `version` is required. Quote it so YAML keeps it a string. Only - valid on macOS. - -## `scorers//scorer.md` -The directory name is the Scorer name. The Markdown body is its required -rubric. Read `scorers.md` for the full contract and a worked example. - -## The harness block -`model: ` is shorthand for `harness: {type: oz, model: }`. Declaring -both `model` and `harness` is an error at every level. - -- `harness.type` — `oz`, `claude` (`claude-code` is also accepted), `codex`, - or `gemini`. -- `harness.model` — a model ID valid for that harness. On an override, null - inherits; an empty string is invalid. -- `harness.reasoningLevel` — rejected by the current server on the `oz` - harness. Per-harness capabilities change, so the bundled validator leaves - that call to the server. -- `harness.auth` — rejected by the current server on the `oz` harness, on the - same server-owned basis as `reasoningLevel`. Null explicitly clears inherited - auth. - - `auth.source: managedSecret` requires `auth.secretName`. - - `auth.source: workerEnvironment` forbids `auth.secretName` and requires a - self-hosted `workerHost`. - -As an override on an agent or automation, `harness` must declare at least one -of `type`, `model`, `reasoningLevel`, or `auth`; an empty block is an error. - -## Inheritance -Values flow `factory.agentDefaults` → agent → automation. Each layer overrides -the one above it for the fields it declares. - -`secrets` and `mcpServers` **replace** rather than merge: an agent that -declares `secrets: []` gets no factory secrets. Additional secrets and MCP -servers the server requires (for example those backing a declared integration) -are merged in on top during resolution. - -`workerHost` and `harness.reasoningLevel` are three-state: omitted inherits, -null or empty clears, and a non-empty value overrides. `harness.model`, -top-level `model`, `runner`, and `environmentId` inherit when omitted or null; -their empty-string form is invalid. - -## YAML restrictions -Each file is a single YAML document. The parser rejects anchors (`&name`), -aliases (`*name`), explicit tags (`!!str`), merge keys (`<<`), duplicate -mapping keys, and non-scalar mapping keys. Markdown resources must open with a -`---` fence and close it before the body. diff --git a/resources/bundled/skills/factory-files/references/scorers.md b/resources/bundled/skills/factory-files/references/scorers.md index 7a6ae3848ca..2e47b8f4120 100644 --- a/resources/bundled/skills/factory-files/references/scorers.md +++ b/resources/bundled/skills/factory-files/references/scorers.md @@ -33,8 +33,7 @@ finishing. Return exactly one declared label. Names are trimmed and must be unique. - `enabled` — optional boolean, default true. Use `enabled: false` rather than a zero sampling rate to pause scoring. -- `output` — optional output form. `classification` is the current known form; - preserve newer values for forward compatibility. +- `output` — optional output form. `classification` is the current known form. - `labels` — required non-empty list of classifications; the current server accepts at most 20. Each label requires a non-empty `value` and a numeric `score` from 0 through 1; `description` is optional. Label values are @@ -46,6 +45,8 @@ finishing. Return exactly one declared label. - `model` — required model ID. The server validates availability. - `selfImprovement` — optional boolean, default false. -The Markdown body must not be empty. Scorer fields are forward-compatible: -preserve unknown fields rather than deleting them to satisfy an older bundled -schema. +The Markdown body must not be empty. + +The field list above is a summary for authoring; the server's schema is the +contract. If a Scorer already contains a field this page does not mention, +leave it alone and validate against the server rather than deleting it. diff --git a/resources/bundled/skills/factory-files/references/triggers.md b/resources/bundled/skills/factory-files/references/triggers.md deleted file mode 100644 index 28bad223c08..00000000000 --- a/resources/bundled/skills/factory-files/references/triggers.md +++ /dev/null @@ -1,144 +0,0 @@ -# Automation triggers -A trigger names a `provider` and an `event`, and may narrow which deliveries -match with a `filter`. - -```yaml -triggers: - - provider: github - event: pull_request_opened - filter: - repos: [warpdotdev/warp-server] - base_branches: [main] -``` - -## Why filter keys need care -The Factory file parser accepts any mapping as a `filter`. Filter keys are -validated later, when the plan is applied. A wrong key therefore parses fine -and fails at apply time, so treat the catalogue below as the contract. Keys are -`snake_case`; a camelCase spelling such as `baseBranches` is rejected. - -Fields combine with AND. An absent field is a wildcard. - -## Filter values -Each filter field takes a matcher. A bare array is sugar for `{in: [...]}`. - -```yaml -labels: [ready] # ANY-of -labels: - in: [ready, urgent] # ANY-of - not_in: [wip] # excludes ANY-of -``` - -A value present in both `in` and `not_in` is rejected: the filter could never -match. Some fields compare canonical forms too — for example, case-insensitive -usernames or emoji names — so equivalent values can conflict even when their -source spelling differs. `schedule_ids` supports only `in`, because excluding -one schedule would match every other schedule in scope. - -All values are strings except `pr_numbers`, which takes integers. - -## github -- `push` — `repos`, `branches`, `paths` -- `issue_created`, `issue_labeled` — `repos`, `labels`, `assignees`, `authors` -- `pull_request_opened`, `pull_request_ready`, `pull_request_closed`, - `pull_request_merged`, `pull_request_labeled`, `pull_request_synchronized`, - `pull_request_reopened` — `repos`, `base_branches`, `pr_numbers`, `paths`, - `assignees`, `authors`, `labels` -- `issue_mentioned`, `pull_request_mentioned` — `repos`, `mentioned`, `labels` -- `issue_assigned`, `pull_request_assigned` — `repos`, `assignees`, `labels` -- `pull_request_review_requested` — `repos`, `reviewers`, `reviewer_teams` -- `pull_request_review_submitted` — `repos`, `mentioned`, `labels`, - `review_states` -- `check_suite_completed` — `repos`, `conclusions`, `branches`, `labels`, - `authors` -- `workflow_run_completed` — `repos`, `conclusions`, `branches`, `workflows`, - `labels`, `authors` -- `check_run_rerequested`, `check_suite_rerequested` — `repos` - -`repos` values are `owner/name`. Branch values may be written with or without a -`refs/heads/` prefix. - -## gitlab -- `merge_request` — `repos`, `actions`, `base_branches` -- `bot_mentioned` — `repos` - -`mentioned` is accepted on `bot_mentioned` but the server seeds it, so declaring -it has no effect. - -## factory -- `work_item_stage_changed` — `stages` - -A Factory delivery is already scoped to one factory, so the stage the work item -moved into is the only dimension worth constraining. - -## linear -- `issue_created`, `issue_labeled`, `issue_state_changed`, `issue_assigned` — - `team_ids`, `project_ids`, `labels`, `state_ids`, `assignee_ids`, - `mentioned_user_ids`, `creator_ids` -- `comment_created` — `team_ids`, `project_ids`, `labels`, `state_ids`, - `issue_ids`, `mentioned_user_ids`, `creator_ids` -- `agent_session_created` — `team_ids`, `creator_ids`, `keywords` - -Linear `*_ids` fields take durable Linear UUIDs, not display names or keys. -`labels` matches by name, case-insensitively. - -## jira -- `issue_created`, `issue_labeled` — `project_keys`, `labels` -- `status_changed` — `project_keys`, `status_ids` -- `agent_session_created` — `project_keys`, `labels`, `keywords` - -Jira `labels` match case-sensitively, unlike the other providers. On -`agent_session_created` the session payload does not carry labels, so a -constrained subscription resolves them through a best-effort issue fetch and -fails closed when that returns none. - -## slack -- `app_mention`, `message_dm`, `message_im`, `message_mpim`, `message_posted` — - `channel_ids`, `user_ids`, `keywords` -- `reaction_added` — `channel_ids`, `user_ids`, `emojis`, `keywords`, - `item_user_ids` -- `member_joined_channel` — `channel_ids`, `user_ids` - -`channel_ids` and `user_ids` take Slack IDs (`C…`, `U…`), not `#channel` or -`@user` names. `emojis` are emoji names; colons and skin-tone suffixes are -ignored. `keywords` match message text case-insensitively as substrings. - -A channel message that mentions the app produces both a `message_posted` and an -`app_mention` delivery, so subscribe to one kind or the other, not both. - -## schedule -- `cron_fired` — `schedule_ids` - -A `schedule.cron_fired` trigger must name exactly one source of schedule: - -```yaml -# Declare the schedule inline. -triggers: - - provider: schedule - event: cron_fired - schedule: - name: nightly-sweep - cron: 0 3 * * * - -# Or watch schedules that already exist. -triggers: - - provider: schedule - event: cron_fired - filter: - schedule_ids: [sched_abc123] -``` - -Declaring both, or neither, is an error: a trigger with neither would subscribe -to every schedule delivery for the team. - -`schedule` is only valid on a `schedule.cron_fired` trigger. - -### Inline schedules -- `cron` — a standard five-field expression or a descriptor (`@daily`, - `@hourly`, `@every 1h`). Always interpreted in UTC, so a `CRON_TZ=` or `TZ=` - prefix is rejected, as is the six-field form carrying seconds. Field ranges, - lists, steps, and month/day names follow the server's robfig/cron grammar. -- `name` — the declaration's stable identity within its automation. Editing - `cron` under an unchanged `name` updates the running schedule in place; - changing `name` replaces it. At most one inline schedule per automation may - omit `name`, and names must be unique within the automation. diff --git a/resources/bundled/skills/factory-files/references/validation.md b/resources/bundled/skills/factory-files/references/validation.md index 69c0403ef18..07ce4f42aac 100644 --- a/resources/bundled/skills/factory-files/references/validation.md +++ b/resources/bundled/skills/factory-files/references/validation.md @@ -1,63 +1,87 @@ # Validating and reading diagnostics ## Layers of validation -1. **The bundled validator** (`scripts/validate_factory_files.py`) checks the - files themselves against the JSON Schemas plus the tree-level rules. It runs - offline and needs nothing but Python 3. Run it before every pull request. -2. **The parser** (`logic/factoryfile` in `warp-server`) is the authority for - everything the validator checks. Its diagnostics carry `FF_*` codes and a - file path and line. -3. **Resolution and apply** validate everything that needs server state: model +1. **The server's parser** (`logic/factoryfile` in `warp-server`) is the only + authority. `POST /api/v1/factory-files/validate` runs it over a tree you + submit as paths and content, and adds the state-independent rules the apply + path enforces next: runner platforms and instance shapes, and trigger filter + keys and matchers. Its diagnostics carry `FF_*` codes with a path, line, and + column. +2. **Resolution and apply** validate everything that needs server state: model IDs, environment IDs, secret names, runner names, MCP server IDs, integration providers, harness model catalogues, worker-host entitlement, - and runner platform and instance-shape rules. + and the values of Linear and Slack name aliases. -A clean validator run means the files pass the bundled structural and -state-independent semantic checks. It does not mean the plan will apply. Say -so rather than overstating what was checked. +There is no third layer, and deliberately no local one. A clean result from the +endpoint means the files pass the structural and state-independent semantic +checks. It does not mean the plan will apply. The response lists the checks it +did not run; say so rather than overstating what was checked. ## Running the validator The script lives at `scripts/validate_factory_files.py` inside this skill's -directory; `SKILL.md` shows its resolved path. +directory; `SKILL.md` shows its resolved path. It does not parse the format. It +selects the tree's resource files by path, refuses symlinks, submits the bytes, +and relays what comes back. ```bash python3 "/scripts/validate_factory_files.py" "" python3 "/scripts/validate_factory_files.py" "" --json ``` -Use Python 3.8 or newer via the host's command (`python3`, `python`, or `py -3`). If none is -available, do not install an interpreter or claim automated validation without -the user's approval; inspect the changed document against its JSON Schema and -report the validation gap. - -Exit code 0 means no problems. Each problem reports the file, the field path, -and what is wrong. Fix them all and re-run; do not stop at the first one, since -one wrong field often produces several messages. -The bundled reader handles the canonical YAML forms this skill emits, not every -piece of YAML syntax accepted by `gopkg.in/yaml.v3`. If it cannot read an -existing file that the server accepts, do not normalize or rewrite the file -merely for the reader; report that local validation was unavailable and use a -server plan when possible. - -A resource file that is a symlink is reported and not read. The server parses -the repository tree, where a symlink is stored as its target path rather than -its target's content, so it never follows one either; a Factory resource has to -be a real file. Reading the target locally would also let a repository aim a -resource at any readable path on the machine. - -The schemas are ordinary JSON Schema 2020-12 documents, so any standard -validator works too if the tree is already converted to JSON. `x-warp-*` -annotations carry constraints JSON Schema cannot express portably, such as -trimmed Unicode alias rules; only the bundled validator enforces those -annotations. +`--server-root ` or `WARP_SERVER_ROOT` selects a local, staging, or +self-hosted server. The endpoint needs no credential; `WARP_API_KEY` is +forwarded when the environment already has one, which makes the request +attributable inside an agent sandbox. + +Use Python 3.8 or newer via the host's command (`python3`, `python`, or +`py -3`). If none is available, do not install an interpreter or claim +automated validation without the user's approval; inspect the changed document +against the fetched schema and report the gap. + +## The three outcomes +- `0` — the server checked the tree and found no problem. +- `1` — the server checked the tree and reported diagnostics. +- `2` — the tree was **not** checked. + +Exit `2` is not a pass and not a failure. It happens when the server is +unreachable, answers with an error or a malformed body, the directory is not a +Factory root, or the tree is larger than the endpoint accepts. In every case +the correct report is that validation did not run, with the reason. Saying +anything about whether the files are correct would be inventing a verdict. + +With `--json`, `validated` distinguishes the cases: a run that reached no +verdict carries `validated: false` and no `valid` key at all, so there is +nothing to misread. + +Each problem reports the file, the field path, and what is wrong. Fix them all +and re-run; do not stop at the first one, since one wrong field often produces +several messages. + +A resource file that is a symlink is reported and never uploaded. The server +parses the repository tree, where a symlink is stored as its target path rather +than its target's content, so it never follows one either; a Factory resource +has to be a real file. Reading the target locally would also let a repository +aim a resource at any readable path on the machine. + +## Deferred resolutions +A response can carry `deferred_resolutions` alongside its diagnostics. A +deferred entry is not a problem: it names an authored value the endpoint +deliberately did not prove, because proving it needs provider state. Linear and +Slack name aliases are the current case — the endpoint checks that `teams` or +`channels` is a list of non-empty names applicable to that event, and leaves +whether those names exist to apply time. + +Report deferred entries. They are the difference between "this parses" and +"this will work". ## Diagnostic codes -The server reports these when a plan is run against a registered Factory. +The server reports these from the validation endpoint and when a plan is run +against a registered Factory. - `FF_MISSING_FACTORY` — no `factory.yaml` at the Factory root. - `FF_UNSUPPORTED_VERSION` — `schemaVersion` names no registered tree adapter. - The bundled validator reports an unrecognized version and stops rather than - applying v1alpha1 rules to a tree it does not describe. + The server stops rather than applying another version's rules. Correct the + version; never lower it to make a check pass. - `FF_UNSUPPORTED_PATH` — a file that resembles an Agent, Automation, Runner, or Scorer resource is at a non-canonical path. Other unrelated files under those directories are intentionally ignored. @@ -72,7 +96,7 @@ The server reports these when a plan is run against a registered Factory. - `FF_ANCHOR`, `FF_ALIAS`, `FF_TAG` — YAML anchors, aliases, and explicit tags are not permitted. - `FF_UNKNOWN_FIELD` — a field the schema does not define. Check spelling and - the field reference; do not add the field to the schema to make it pass. + the fetched schema. - `FF_MISSING_REQUIRED` — a required field is absent or empty. - `FF_TYPE_MISMATCH` — a value has the wrong YAML type. - `FF_INVALID_VALUE` — a value violates a format or exclusivity rule, such as @@ -85,59 +109,46 @@ The server reports these when a plan is run against a registered Factory. schedule on a non-schedule trigger, or a `schedule.cron_fired` trigger that declares both or neither of `schedule.cron` and `filter.schedule_ids`. - `FF_INVALID_EVENT`, `FF_INVALID_FILTER` — the event is unknown, or a filter - value is outside its valid domain. - -The bundled schemas do not reproduce every catalogue rejection above. Unknown -properties, agent types, credential strategies, harnesses and their per-harness -capabilities, integration types, trigger providers and events, runner platform -values, Scorer output forms, and server-tunable limits such as the Scorer label -cap are all preserved so an older client does not reject source accepted by a -newer server. A server plan is authoritative. - -Filter keys are the one catalogue still checked, because a misspelled key is a -common mistake that otherwise survives until apply. The check applies only when -both the provider and the event are ones these schemas know; a newer provider, -or a newer event on a known provider, leaves its filter unconstrained. - -What the bundled validator still refuses is what stays wrong under any of those -changes: malformed YAML and frontmatter, missing required fields, values of the -wrong type, references to Agents the tree does not declare, more or fewer than -one `MAIN`/`FOREMAN` Agent, duplicate resource names and labels, an empty -Scorer rubric, a label set that cannot both pass and fail, and filters that can -never match. - -## If you are changing these schemas -The permissiveness above is load-bearing, not an unfinished edge. These files -ship inside a Warp release and are routinely older than the `warp-server` they -run against, so closing them back up would reject configuration a newer server -accepts and push agents to delete working fields. - -When the format gains a value, add it to the relevant `x-warp-known-values` or -`x-warp-known-max-items` annotation. Do not turn an annotation back into -`enum`, `const`, `maxItems`, or `additionalProperties: false`. The regression -corpus in `script/test_factory_files_skill.py` asserts several of these -tolerances on purpose; if one starts failing, a schema was tightened. + key or value is outside its valid domain. -## Fixing a diagnostic -Change the file the diagnostic names, at the field it names. Do not silence a -diagnostic by deleting the resource, loosening the schema, or moving a file to -a path the parser ignores. +## Fetching the schema +The schema endpoints are unauthenticated and cacheable: + +```bash +curl -s https://app.warp.dev/api/v1/factory-files/schemas +curl -s https://app.warp.dev/api/v1/factory-files/schemas/v1alpha1 +``` -If a diagnostic contradicts these references, the server is right. Say that the -bundled schemas look stale and, where you can, point at what changed in -`logic/factoryfile`. +They are ordinary JSON Schema 2020-12 documents, exact for the version they +describe, so any standard validator works if a tree is already converted to +JSON. `x-warp-*` annotations carry constraints JSON Schema cannot express +portably, such as trimmed Unicode alias rules and the power-of-two Linux +compute sizes; only a Warp validator enforces those. -## Version skew -The schemas ship inside the Warp version running them, not from the server, so -they can lag the server that a Factory actually syncs against. Cloud agent runs -track releases closely; an installed desktop client can be much older. +Fetching a schema is not validation. A successful fetch says the server is +reachable, nothing more. -The asymmetry matters when reading an `unknown field` report: +## Why there is no offline mode +This skill used to bundle the schemas and a local validator so authoring worked +without a server. That was removed, and should not be reintroduced. -- On a field you just wrote, it is almost certainly a mistake. Fix it. -- On a field that was already in the file, it may be a newer field your copy of - the schemas does not know. Leave it alone and report the possibility. Removing - it would silently drop working configuration. +The copy shipped inside a Warp release, so it was routinely older than the +server a Factory syncs against. A stale copy does not degrade gracefully: it +reports a field the server accepts as unknown, and an agent trying to reach a +clean run resolves that by deleting working configuration. It happened — an +earlier revision rejected the Linear and Slack trigger aliases (`teams`, +`projects`, `states`, `issues`, `channels`, `users`, `itemUsers`) that the +apply path rewrites and accepts, on a tree taken from the server's own +`testdata/valid`. + +The trade is deliberate: never checking is recoverable, and the report says so. +Checking wrongly costs correct configuration and is not obviously wrong to the +agent acting on it. + +## Fixing a diagnostic +Change the file the diagnostic names, at the field it names. Do not silence a +diagnostic by deleting the resource or moving a file to a path the parser +ignores. ## Checking against the parser directly When `warp-server` is checked out locally, its parser tests are the closest @@ -145,7 +156,7 @@ thing to ground truth. Run them from that checkout, not from the Factory repository: ```bash -go test ./logic/factoryfile +go test ./logic/factoryfile/... ``` Fixtures under `logic/factoryfile/testdata` show accepted and rejected trees. diff --git a/resources/bundled/skills/factory-files/schemas/agent.schema.json b/resources/bundled/skills/factory-files/schemas/agent.schema.json deleted file mode 100644 index 913fbc269a2..00000000000 --- a/resources/bundled/skills/factory-files/schemas/agent.schema.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "agent.schema.json", - "$comment": "FORWARD COMPATIBILITY - DO NOT TIGHTEN. additionalProperties is true, and catalogue values are recorded as x-warp-known-values / x-warp-known-max-items annotations rather than enum, const, or maxItems. That is deliberate, not an oversight. This file ships inside a Warp release and is routinely OLDER than the warp-server it validates against, so a closed schema rejects configuration a newer server accepts and pushes agents to delete working fields. When the format gains a value, add it to the annotation; do not convert the annotation back into a rejecting keyword. The two deliberate exceptions are documented in specs/REMOTE-2727/TECH.md: trigger filter keys, which only apply when both provider and event are known, and schemaVersion, which stops validation rather than misapplying v1alpha1 rules. See also references/validation.md.", - "title": "agents//agent.md frontmatter (v1alpha1)", - "description": "YAML frontmatter of an Agent file. The Agent's name comes from its directory, never from a field. The Markdown body after the closing fence is the Agent's prompt.", - "type": "object", - "additionalProperties": true, - "properties": { - "description": { - "type": ["string", "null"] - }, - "agentType": { "$ref": "common.schema.json#/$defs/agentType" }, - "credentialStrategy": { "$ref": "common.schema.json#/$defs/credentialStrategy" }, - "model": { "$ref": "common.schema.json#/$defs/ozModelOverride" }, - "harness": { "$ref": "common.schema.json#/$defs/harnessOverride" }, - "runner": { "$ref": "common.schema.json#/$defs/runnerRef" }, - "environmentId": { "$ref": "common.schema.json#/$defs/environmentId" }, - "secrets": { "$ref": "common.schema.json#/$defs/secrets" }, - "mcpServers": { "$ref": "common.schema.json#/$defs/mcpServers" }, - "workerHost": { "$ref": "common.schema.json#/$defs/workerHost" } - }, - "allOf": [{ "$ref": "common.schema.json#/$defs/modelXorHarnessOptional" }] -} diff --git a/resources/bundled/skills/factory-files/schemas/automation.schema.json b/resources/bundled/skills/factory-files/schemas/automation.schema.json deleted file mode 100644 index 578cd69ca7f..00000000000 --- a/resources/bundled/skills/factory-files/schemas/automation.schema.json +++ /dev/null @@ -1,820 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "automation.schema.json", - "$comment": "FORWARD COMPATIBILITY - DO NOT TIGHTEN. additionalProperties is true, and catalogue values are recorded as x-warp-known-values / x-warp-known-max-items annotations rather than enum, const, or maxItems. That is deliberate, not an oversight. This file ships inside a Warp release and is routinely OLDER than the warp-server it validates against, so a closed schema rejects configuration a newer server accepts and pushes agents to delete working fields. When the format gains a value, add it to the annotation; do not convert the annotation back into a rejecting keyword. The two deliberate exceptions are documented in specs/REMOTE-2727/TECH.md: trigger filter keys, which only apply when both provider and event are known, and schemaVersion, which stops validation rather than misapplying v1alpha1 rules. See also references/validation.md.", - "title": "automations//automation.md frontmatter (v1alpha1)", - "description": "YAML frontmatter of an Automation file. The Automation's name comes from its directory, never from a field. The Markdown body after the closing fence is the run prompt.", - "type": "object", - "required": ["triggers"], - "additionalProperties": true, - "properties": { - "enabled": { - "type": "boolean", - "description": "Defaults to true when omitted." - }, - "agent": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "description": "Name of a declared Agent. Defaults to the MAIN/FOREMAN Agent when omitted." - }, - "model": { "$ref": "common.schema.json#/$defs/ozModelOverride" }, - "harness": { "$ref": "common.schema.json#/$defs/harnessOverride" }, - "runner": { "$ref": "common.schema.json#/$defs/runnerRef" }, - "environmentId": { "$ref": "common.schema.json#/$defs/environmentId" }, - "secrets": { "$ref": "common.schema.json#/$defs/secrets" }, - "mcpServers": { "$ref": "common.schema.json#/$defs/mcpServers" }, - "workerHost": { "$ref": "common.schema.json#/$defs/workerHost" }, - "triggers": { - "type": "array", - "minItems": 1, - "items": { "$ref": "#/$defs/trigger" } - } - }, - "allOf": [{ "$ref": "common.schema.json#/$defs/modelXorHarnessOptional" }], - "$defs": { - "stringMatcher": { - "description": "A bare array is sugar for {in: [...]}. Within a field, in matches ANY-of and not_in excludes ANY-of. Null and an empty matcher impose nothing.", - "anyOf": [ - { "type": "null" }, - { "type": "array", "items": { "type": "string" } }, - { - "type": "object", - "additionalProperties": true, - "properties": { - "in": { "type": "array", "items": { "type": "string" } }, - "not_in": { "type": "array", "items": { "type": "string" } } - } - } - ] - }, - "intMatcher": { - "anyOf": [ - { "type": "null" }, - { "type": "array", "items": { "type": "integer" } }, - { - "type": "object", - "additionalProperties": true, - "properties": { - "in": { "type": "array", "items": { "type": "integer" } }, - "not_in": { "type": "array", "items": { "type": "integer" } } - } - } - ] - }, - "scheduleIdsMatcher": { - "description": "schedule_ids supports only the in operator: an exclusion would match every other schedule in scope.", - "anyOf": [ - { "type": "null" }, - { "type": "array", "items": { "type": "string" } }, - { - "type": "object", - "additionalProperties": true, - "properties": { - "in": { "type": "array", "items": { "type": "string" } } - } - } - ] - }, - "nonEmptyScheduleIds": { - "type": "object", - "required": ["filter"], - "properties": { - "filter": { - "type": "object", - "required": ["schedule_ids"], - "properties": { - "schedule_ids": { - "anyOf": [ - { "type": "array", "minItems": 1 }, - { - "type": "object", - "required": ["in"], - "properties": { "in": { "type": "array", "minItems": 1 } } - } - ] - } - } - } - } - }, - "inlineSchedule": { - "type": "object", - "required": ["cron"], - "additionalProperties": true, - "description": "A cron schedule declared inline on a schedule.cron_fired trigger. Only valid on that kind, and mutually exclusive with filter.schedule_ids.", - "properties": { - "name": { - "type": ["string", "null"], - "description": "Stable identity of this declaration within the Automation. At most one inline schedule may omit name. Changing name replaces the schedule; changing only cron updates it in place." - }, - "cron": { - "type": "string", - "pattern": "^\\s*(@(annually|yearly|monthly|weekly|daily|midnight|hourly)|@every\\s+\\S+|(\\S+\\s+){4}\\S+)\\s*$", - "description": "Standard five-field cron expression or a descriptor (@daily, @hourly, @every 1h). Always interpreted in UTC: a CRON_TZ= or TZ= prefix and the six-field seconds form are both rejected." - } - } - }, - "trigger": { - "type": "object", - "required": ["provider", "event"], - "additionalProperties": true, - "properties": { - "provider": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "x-warp-known-values": ["github", "gitlab", "linear", "jira", "slack", "schedule", "factory"] - }, - "event": { "type": "string" }, - "filter": { - "type": "object", - "description": "Filter fields combine with AND. An absent field is a wildcard. The accepted keys depend on the (provider, event) pair." - }, - "schedule": { "$ref": "#/$defs/inlineSchedule" } - }, - "allOf": [ - { - "if": { - "required": ["provider"], - "properties": { "provider": { "const": "github" } } - }, - "then": { - "properties": { - "event": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "x-warp-known-values": [ - "push", - "issue_created", - "issue_labeled", - "issue_mentioned", - "issue_assigned", - "pull_request_opened", - "pull_request_ready", - "pull_request_closed", - "pull_request_merged", - "pull_request_labeled", - "pull_request_synchronized", - "pull_request_reopened", - "pull_request_mentioned", - "pull_request_assigned", - "pull_request_review_requested", - "pull_request_review_submitted", - "check_suite_completed", - "check_suite_rerequested", - "check_run_rerequested", - "workflow_run_completed" - ] - } - } - } - }, - { - "if": { - "required": ["provider"], - "properties": { "provider": { "const": "gitlab" } } - }, - "then": { - "properties": { - "event": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "x-warp-known-values": ["merge_request", "bot_mentioned"] - } - } - } - }, - { - "if": { - "required": ["provider"], - "properties": { "provider": { "const": "factory" } } - }, - "then": { - "properties": { - "event": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "x-warp-known-values": ["work_item_stage_changed"] - } - } - } - }, - { - "if": { - "required": ["provider"], - "properties": { "provider": { "const": "linear" } } - }, - "then": { - "properties": { - "event": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "x-warp-known-values": [ - "issue_created", - "issue_labeled", - "issue_state_changed", - "issue_assigned", - "comment_created", - "agent_session_created" - ] - } - } - } - }, - { - "if": { - "required": ["provider"], - "properties": { "provider": { "const": "jira" } } - }, - "then": { - "properties": { - "event": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "x-warp-known-values": [ - "issue_created", - "issue_labeled", - "status_changed", - "agent_session_created" - ] - } - } - } - }, - { - "if": { - "required": ["provider"], - "properties": { "provider": { "const": "slack" } } - }, - "then": { - "properties": { - "event": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "x-warp-known-values": [ - "app_mention", - "message_dm", - "message_im", - "message_mpim", - "message_posted", - "reaction_added", - "member_joined_channel" - ] - } - } - } - }, - { - "if": { - "required": ["provider"], - "properties": { "provider": { "const": "schedule" } } - }, - "then": { - "properties": { - "event": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "x-warp-known-values": ["cron_fired"] - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "github" }, - "event": { "const": "push" } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "repos": { "$ref": "#/$defs/stringMatcher" }, - "branches": { "$ref": "#/$defs/stringMatcher" }, - "paths": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "github" }, - "event": { "enum": ["issue_created", "issue_labeled"] } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "repos": { "$ref": "#/$defs/stringMatcher" }, - "labels": { "$ref": "#/$defs/stringMatcher" }, - "assignees": { "$ref": "#/$defs/stringMatcher" }, - "authors": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "github" }, - "event": { - "enum": [ - "pull_request_opened", - "pull_request_ready", - "pull_request_closed", - "pull_request_merged", - "pull_request_labeled", - "pull_request_synchronized", - "pull_request_reopened" - ] - } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "repos": { "$ref": "#/$defs/stringMatcher" }, - "base_branches": { "$ref": "#/$defs/stringMatcher" }, - "pr_numbers": { "$ref": "#/$defs/intMatcher" }, - "paths": { "$ref": "#/$defs/stringMatcher" }, - "assignees": { "$ref": "#/$defs/stringMatcher" }, - "authors": { "$ref": "#/$defs/stringMatcher" }, - "labels": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "github" }, - "event": { "enum": ["issue_mentioned", "pull_request_mentioned"] } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "repos": { "$ref": "#/$defs/stringMatcher" }, - "mentioned": { "$ref": "#/$defs/stringMatcher" }, - "labels": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "github" }, - "event": { "enum": ["issue_assigned", "pull_request_assigned"] } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "repos": { "$ref": "#/$defs/stringMatcher" }, - "assignees": { "$ref": "#/$defs/stringMatcher" }, - "labels": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "github" }, - "event": { "const": "pull_request_review_requested" } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "repos": { "$ref": "#/$defs/stringMatcher" }, - "reviewers": { "$ref": "#/$defs/stringMatcher" }, - "reviewer_teams": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "github" }, - "event": { "const": "pull_request_review_submitted" } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "repos": { "$ref": "#/$defs/stringMatcher" }, - "mentioned": { "$ref": "#/$defs/stringMatcher" }, - "labels": { "$ref": "#/$defs/stringMatcher" }, - "review_states": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "github" }, - "event": { "const": "check_suite_completed" } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "repos": { "$ref": "#/$defs/stringMatcher" }, - "conclusions": { "$ref": "#/$defs/stringMatcher" }, - "branches": { "$ref": "#/$defs/stringMatcher" }, - "labels": { "$ref": "#/$defs/stringMatcher" }, - "authors": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "github" }, - "event": { "const": "workflow_run_completed" } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "repos": { "$ref": "#/$defs/stringMatcher" }, - "conclusions": { "$ref": "#/$defs/stringMatcher" }, - "branches": { "$ref": "#/$defs/stringMatcher" }, - "workflows": { "$ref": "#/$defs/stringMatcher" }, - "labels": { "$ref": "#/$defs/stringMatcher" }, - "authors": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "github" }, - "event": { - "enum": ["check_run_rerequested", "check_suite_rerequested"] - } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "repos": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "gitlab" }, - "event": { "const": "merge_request" } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "repos": { "$ref": "#/$defs/stringMatcher" }, - "actions": { "$ref": "#/$defs/stringMatcher" }, - "base_branches": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "gitlab" }, - "event": { "const": "bot_mentioned" } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "repos": { "$ref": "#/$defs/stringMatcher" }, - "mentioned": { - "$ref": "#/$defs/stringMatcher", - "description": "Server-seeded. Declaring it has no effect." - } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "factory" }, - "event": { "const": "work_item_stage_changed" } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "stages": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "linear" }, - "event": { - "enum": [ - "issue_created", - "issue_labeled", - "issue_state_changed", - "issue_assigned" - ] - } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "team_ids": { "$ref": "#/$defs/stringMatcher" }, - "project_ids": { "$ref": "#/$defs/stringMatcher" }, - "labels": { "$ref": "#/$defs/stringMatcher" }, - "state_ids": { "$ref": "#/$defs/stringMatcher" }, - "assignee_ids": { "$ref": "#/$defs/stringMatcher" }, - "mentioned_user_ids": { "$ref": "#/$defs/stringMatcher" }, - "creator_ids": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "linear" }, - "event": { "const": "comment_created" } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "team_ids": { "$ref": "#/$defs/stringMatcher" }, - "project_ids": { "$ref": "#/$defs/stringMatcher" }, - "labels": { "$ref": "#/$defs/stringMatcher" }, - "state_ids": { "$ref": "#/$defs/stringMatcher" }, - "issue_ids": { "$ref": "#/$defs/stringMatcher" }, - "mentioned_user_ids": { "$ref": "#/$defs/stringMatcher" }, - "creator_ids": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "linear" }, - "event": { "const": "agent_session_created" } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "team_ids": { "$ref": "#/$defs/stringMatcher" }, - "creator_ids": { "$ref": "#/$defs/stringMatcher" }, - "keywords": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "jira" }, - "event": { "enum": ["issue_created", "issue_labeled"] } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "project_keys": { "$ref": "#/$defs/stringMatcher" }, - "labels": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "jira" }, - "event": { "const": "status_changed" } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "project_keys": { "$ref": "#/$defs/stringMatcher" }, - "status_ids": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "jira" }, - "event": { "const": "agent_session_created" } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "project_keys": { "$ref": "#/$defs/stringMatcher" }, - "labels": { "$ref": "#/$defs/stringMatcher" }, - "keywords": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "slack" }, - "event": { - "enum": [ - "app_mention", - "message_dm", - "message_im", - "message_mpim", - "message_posted" - ] - } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "channel_ids": { "$ref": "#/$defs/stringMatcher" }, - "user_ids": { "$ref": "#/$defs/stringMatcher" }, - "keywords": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "slack" }, - "event": { "const": "reaction_added" } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "channel_ids": { "$ref": "#/$defs/stringMatcher" }, - "user_ids": { "$ref": "#/$defs/stringMatcher" }, - "emojis": { "$ref": "#/$defs/stringMatcher" }, - "keywords": { "$ref": "#/$defs/stringMatcher" }, - "item_user_ids": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "slack" }, - "event": { "const": "member_joined_channel" } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "channel_ids": { "$ref": "#/$defs/stringMatcher" }, - "user_ids": { "$ref": "#/$defs/stringMatcher" } - } - } - } - } - }, - { - "if": { - "required": ["provider", "event"], - "properties": { - "provider": { "const": "schedule" }, - "event": { "const": "cron_fired" } - } - }, - "then": { - "properties": { - "filter": { - "additionalProperties": false, - "properties": { - "schedule_ids": { "$ref": "#/$defs/scheduleIdsMatcher" } - } - } - }, - "oneOf": [ - { "required": ["schedule"] }, - { "$ref": "#/$defs/nonEmptyScheduleIds" } - ] - } - }, - { - "if": { - "not": { - "allOf": [ - { - "required": ["provider"], - "properties": { "provider": { "const": "schedule" } } - }, - { - "required": ["event"], - "properties": { "event": { "const": "cron_fired" } } - } - ] - } - }, - "then": { "not": { "required": ["schedule"] } } - } - ] - } - } -} diff --git a/resources/bundled/skills/factory-files/schemas/common.schema.json b/resources/bundled/skills/factory-files/schemas/common.schema.json deleted file mode 100644 index 0f8a9c4a171..00000000000 --- a/resources/bundled/skills/factory-files/schemas/common.schema.json +++ /dev/null @@ -1,190 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "common.schema.json", - "$comment": "FORWARD COMPATIBILITY - DO NOT TIGHTEN. additionalProperties is true, and catalogue values are recorded as x-warp-known-values / x-warp-known-max-items annotations rather than enum, const, or maxItems. That is deliberate, not an oversight. This file ships inside a Warp release and is routinely OLDER than the warp-server it validates against, so a closed schema rejects configuration a newer server accepts and pushes agents to delete working fields. When the format gains a value, add it to the annotation; do not convert the annotation back into a rejecting keyword. The two deliberate exceptions are documented in specs/REMOTE-2727/TECH.md: trigger filter keys, which only apply when both provider and event are known, and schemaVersion, which stops validation rather than misapplying v1alpha1 rules. See also references/validation.md.", - "title": "Shared Factory file definitions (v1alpha1)", - "description": "Definitions shared by factory.yaml, Agent, Automation, Scorer, and Runner documents.", - "$defs": { - "nonEmptyString": { - "type": "string", - "minLength": 1, - "pattern": "\\S", - "description": "A string that is non-empty after trimming surrounding whitespace." - }, - "nullableNonEmptyString": { - "type": ["string", "null"], - "if": { "type": "string" }, - "then": { - "minLength": 1, - "pattern": "\\S" - }, - "description": "A non-empty string override, or null to inherit." - }, - "clearableString": { - "type": ["string", "null"], - "description": "A string override. Null or empty explicitly clears an inherited value; omitting the field inherits it." - }, - "alias": { - "type": ["string", "null"], - "description": "Factory display handle, used as the @-mention name on integrated platforms. Letters, digits, spaces, '-', '_', and '.' only, at most 60 characters. Case is preserved; uniqueness is compared case-insensitively across the workspace.", - "x-warp-character-class": "unicode-letters-numbers-space-dot-underscore-hyphen", - "x-warp-max-trimmed-runes": 60 - }, - "credentialStrategy": { - "$ref": "#/$defs/nullableNonEmptyString", - "x-warp-known-values": ["EXECUTOR", "CREATOR"], - "description": "Whose credentials a run executes with. Omit to leave the value already stored on the server untouched." - }, - "agentType": { - "$ref": "#/$defs/nullableNonEmptyString", - "x-warp-known-values": ["CUSTOM", "MAIN", "FOREMAN", "TRIAGE", "SPEC", "IMPLEMENT", "REVIEW", "VERIFY"], - "description": "MAIN is an authoring alias for FOREMAN. Exactly one Agent in the tree must declare MAIN or FOREMAN. Omitting agentType resolves to CUSTOM." - }, - "secretName": { - "$ref": "#/$defs/nonEmptyString", - "description": "Name of a managed secret already registered for the team." - }, - "secrets": { - "type": "array", - "items": { "$ref": "#/$defs/secretName" }, - "description": "Managed secret names. Declaring the field at Agent or Automation level replaces the inherited list rather than adding to it." - }, - "uniqueSecrets": { - "type": "array", - "items": { "$ref": "#/$defs/secretName" }, - "uniqueItems": true, - "description": "Managed secret names. Duplicates are rejected in factory.yaml." - }, - "mcpServers": { - "type": "object", - "propertyNames": { "$ref": "#/$defs/nonEmptyString" }, - "additionalProperties": { - "type": "object", - "required": ["warpId"], - "additionalProperties": true, - "properties": { - "warpId": { - "$ref": "#/$defs/nonEmptyString", - "description": "Warp-managed MCP server ID. This is the only key an entry may carry." - } - } - }, - "description": "Managed MCP servers keyed by the name the agent sees. Declaring the field at Agent or Automation level replaces the inherited map." - }, - "harnessAuth": { - "type": ["object", "null"], - "description": "Credentials for a non-Oz harness. Null explicitly clears inherited auth.", - "additionalProperties": true, - "properties": { - "source": { - "$ref": "#/$defs/nonEmptyString", - "x-warp-known-values": ["managedSecret", "workerEnvironment"] - }, - "secretName": { "$ref": "#/$defs/nonEmptyString" } - }, - "if": { "type": "object" }, - "then": { - "required": ["source"], - "allOf": [ - { - "if": { - "properties": { "source": { "const": "managedSecret" } }, - "required": ["source"] - }, - "then": { "required": ["secretName"] } - }, - { - "if": { - "properties": { "source": { "const": "workerEnvironment" } }, - "required": ["source"] - }, - "then": { "not": { "required": ["secretName"] } } - } - ] - } - }, - "harnessCommonProperties": { - "type": "object", - "additionalProperties": true, - "properties": { - "type": { - "$ref": "#/$defs/nullableNonEmptyString", - "x-warp-known-values": ["oz", "claude", "claude-code", "codex", "gemini"], - "description": "Harness config name. claude-code is an accepted alias for claude." - }, - "model": { "$ref": "#/$defs/nullableNonEmptyString" }, - "reasoningLevel": { - "$ref": "#/$defs/clearableString", - "description": "The current server rejects this on the oz harness. Per-harness capabilities change, so the bundled schema leaves that judgement to the server." - }, - "auth": { "$ref": "#/$defs/harnessAuth" } - } - }, - "harnessOverride": { - "allOf": [ - { "$ref": "#/$defs/harnessCommonProperties" }, - { - "anyOf": [ - { - "required": ["type"], - "properties": { "type": { "type": "string" } } - }, - { - "required": ["model"], - "properties": { "model": { "type": "string" } } - }, - { "required": ["reasoningLevel"] }, - { "required": ["auth"] } - ] - } - ], - "description": "Sparse harness override. Must declare at least one of type, model, reasoningLevel, or auth." - }, - "harnessDefault": { - "allOf": [ - { "$ref": "#/$defs/harnessCommonProperties" }, - { - "required": ["type", "model"], - "properties": { - "type": { - "$ref": "#/$defs/nonEmptyString" - }, - "model": { "$ref": "#/$defs/nonEmptyString" } - } - } - ], - "description": "Factory-level harness default. Both type and model are required." - }, - "modelXorHarnessRequired": { - "oneOf": [ - { "required": ["model"] }, - { "required": ["harness"] } - ], - "description": "Declare exactly one of model (Oz harness shorthand) or harness (explicit harness block)." - }, - "modelXorHarnessOptional": { - "not": { "required": ["model", "harness"] }, - "description": "model and harness are mutually exclusive." - }, - "ozModelDefault": { - "$ref": "#/$defs/nonEmptyString", - "description": "Oz model ID. Shorthand for harness: {type: oz, model: }. Mutually exclusive with harness." - }, - "ozModelOverride": { - "$ref": "#/$defs/nullableNonEmptyString", - "description": "Oz model ID, or null to inherit. Shorthand for harness: {type: oz, model: }. Mutually exclusive with harness." - }, - "runnerRef": { - "$ref": "#/$defs/nullableNonEmptyString", - "description": "Name of a runner declared under runners/.yaml, or null to inherit." - }, - "environmentId": { - "$ref": "#/$defs/nullableNonEmptyString", - "description": "Cloud environment ID the run executes in, or null to inherit." - }, - "workerHost": { - "$ref": "#/$defs/clearableString", - "description": "Self-hosted worker host. Null or empty clears an inherited host and defers to the workspace default." - } - } -} diff --git a/resources/bundled/skills/factory-files/schemas/factory.schema.json b/resources/bundled/skills/factory-files/schemas/factory.schema.json deleted file mode 100644 index 787e5950462..00000000000 --- a/resources/bundled/skills/factory-files/schemas/factory.schema.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "factory.schema.json", - "$comment": "FORWARD COMPATIBILITY - DO NOT TIGHTEN. additionalProperties is true, and catalogue values are recorded as x-warp-known-values / x-warp-known-max-items annotations rather than enum, const, or maxItems. That is deliberate, not an oversight. This file ships inside a Warp release and is routinely OLDER than the warp-server it validates against, so a closed schema rejects configuration a newer server accepts and pushes agents to delete working fields. When the format gains a value, add it to the annotation; do not convert the annotation back into a rejecting keyword. The two deliberate exceptions are documented in specs/REMOTE-2727/TECH.md: trigger filter keys, which only apply when both provider and event are known, and schemaVersion, which stops validation rather than misapplying v1alpha1 rules. See also references/validation.md.", - "title": "factory.yaml (v1alpha1)", - "description": "The Factory root document. Exactly one factory.yaml must exist at the registered Factory root.", - "type": "object", - "required": ["schemaVersion", "name", "repositories", "agentDefaults"], - "additionalProperties": true, - "properties": { - "schemaVersion": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "x-warp-known-values": ["v1alpha1"], - "description": "Tree schema version. These schemas describe v1alpha1. A tree declaring a newer version is reported as unvalidatable rather than validated against v1alpha1 rules." - }, - "name": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "description": "Factory name." - }, - "description": { - "type": ["string", "null"] - }, - "alias": { "$ref": "common.schema.json#/$defs/alias" }, - "credentialStrategy": { "$ref": "common.schema.json#/$defs/credentialStrategy" }, - "repositories": { - "type": "array", - "minItems": 1, - "uniqueItems": true, - "description": "GitHub repositories in the Factory's scope.", - "items": { - "type": "object", - "required": ["owner", "name"], - "additionalProperties": true, - "properties": { - "owner": { "$ref": "common.schema.json#/$defs/nonEmptyString" }, - "name": { "$ref": "common.schema.json#/$defs/nonEmptyString" } - } - } - }, - "secrets": { "$ref": "common.schema.json#/$defs/uniqueSecrets" }, - "mcpServers": { "$ref": "common.schema.json#/$defs/mcpServers" }, - "cloudProviders": { - "$ref": "#/$defs/cloudProviders", - "description": "Current cloud-provider identity federation section." - }, - "providers": { - "$ref": "#/$defs/cloudProviders", - "description": "Legacy read-only alias for cloudProviders. Author cloudProviders in new files." - }, - "integrations": { - "type": "array", - "uniqueItems": true, - "description": "Integration providers attached to the Factory. An empty list explicitly detaches every provider. github is not declarable here: repository access comes from repositories.", - "items": { - "type": "object", - "required": ["type"], - "additionalProperties": true, - "properties": { - "type": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "x-warp-known-values": ["jira", "linear", "slack"], - "description": "Provider slug. The server lowercases and trims the value; author the canonical known spelling." - } - } - } - }, - "agentDefaults": { - "type": "object", - "additionalProperties": true, - "description": "Execution defaults every Agent inherits. Exactly one of model or harness is required.", - "properties": { - "model": { "$ref": "common.schema.json#/$defs/ozModelDefault" }, - "harness": { "$ref": "common.schema.json#/$defs/harnessDefault" }, - "runner": { "$ref": "common.schema.json#/$defs/runnerRef" }, - "environmentId": { "$ref": "common.schema.json#/$defs/environmentId" }, - "secrets": { "$ref": "common.schema.json#/$defs/secrets" }, - "mcpServers": { "$ref": "common.schema.json#/$defs/mcpServers" }, - "workerHost": { "$ref": "common.schema.json#/$defs/workerHost" } - }, - "allOf": [{ "$ref": "common.schema.json#/$defs/modelXorHarnessRequired" }] - } - }, - "$defs": { - "cloudProviders": { - "type": "object", - "additionalProperties": true, - "description": "Cloud provider identity federation used by agent runs.", - "properties": { - "gcp": { - "type": "object", - "required": [ - "projectNumber", - "workloadIdentityFederationPoolId", - "workloadIdentityFederationProviderId" - ], - "additionalProperties": true, - "properties": { - "projectNumber": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "description": "Quote the value so YAML keeps it a string rather than an integer." - }, - "workloadIdentityFederationPoolId": { "$ref": "common.schema.json#/$defs/nonEmptyString" }, - "workloadIdentityFederationProviderId": { "$ref": "common.schema.json#/$defs/nonEmptyString" }, - "serviceAccountEmail": { "$ref": "common.schema.json#/$defs/nonEmptyString" } - } - }, - "aws": { - "type": "object", - "required": ["roleArn"], - "additionalProperties": true, - "properties": { - "roleArn": { "$ref": "common.schema.json#/$defs/nonEmptyString" } - } - } - } - } - } -} diff --git a/resources/bundled/skills/factory-files/schemas/runner.schema.json b/resources/bundled/skills/factory-files/schemas/runner.schema.json deleted file mode 100644 index dc060e2a11b..00000000000 --- a/resources/bundled/skills/factory-files/schemas/runner.schema.json +++ /dev/null @@ -1,135 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "runner.schema.json", - "$comment": "FORWARD COMPATIBILITY - DO NOT TIGHTEN. additionalProperties is true, and catalogue values are recorded as x-warp-known-values / x-warp-known-max-items annotations rather than enum, const, or maxItems. That is deliberate, not an oversight. This file ships inside a Warp release and is routinely OLDER than the warp-server it validates against, so a closed schema rejects configuration a newer server accepts and pushes agents to delete working fields. When the format gains a value, add it to the annotation; do not convert the annotation back into a rejecting keyword. The two deliberate exceptions are documented in specs/REMOTE-2727/TECH.md: trigger filter keys, which only apply when both provider and event are known, and schemaVersion, which stops validation rather than misapplying v1alpha1 rules. See also references/validation.md.", - "title": "runners/.yaml (v1alpha1)", - "description": "A Runner document. The Runner's name comes from the file name, never from a field. Platform and instance-shape rules are enforced when the plan is applied, not by the file parser.", - "type": "object", - "additionalProperties": true, - "properties": { - "description": { - "type": ["string", "null"] - }, - "setupCommands": { - "type": "array", - "items": { "type": "string" }, - "description": "Shell commands run while preparing the sandbox." - }, - "instanceShape": { - "type": "object", - "required": ["vcpus", "memoryGb"], - "additionalProperties": true, - "properties": { - "vcpus": { "type": "integer", "minimum": 1 }, - "memoryGb": { "type": "integer", "minimum": 1 } - } - }, - "platform": { - "type": "object", - "additionalProperties": true, - "properties": { - "os": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "x-warp-known-values": ["linux", "macos"], - "description": "Defaults to linux when omitted." - }, - "arch": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "x-warp-known-values": ["x86_64", "aarch64"], - "description": "Defaults to x86_64 on linux and aarch64 on macos. Supported pairs: linux/x86_64, linux/aarch64, macos/aarch64." - }, - "linux": { - "type": "object", - "additionalProperties": true, - "properties": { - "dockerImage": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "description": "Container image the Linux sandbox boots. Required for every Linux runner." - } - } - }, - "mac": { - "type": "object", - "required": ["version"], - "additionalProperties": true, - "properties": { - "version": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "x-warp-known-values": ["14", "15", "26", "27"], - "description": "Quote the value so YAML keeps it a string. Defaults to 26 when the mac section is omitted." - } - } - } - } - } - }, - "allOf": [ - { - "if": { "$ref": "#/$defs/declaresMacOS" }, - "then": { - "properties": { - "platform": { - "not": { "required": ["linux"] } - } - } - } - }, - { - "if": { "$ref": "#/$defs/declaresLinuxOrDefault" }, - "then": { - "required": ["platform"], - "properties": { - "platform": { - "required": ["linux"], - "not": { "required": ["mac"] }, - "properties": { - "linux": { "required": ["dockerImage"] } - } - }, - "instanceShape": { "$ref": "#/$defs/linuxInstanceShape" } - } - } - } - ], - "$defs": { - "declaresMacOS": { - "type": "object", - "required": ["platform"], - "properties": { - "platform": { - "type": "object", - "required": ["os"], - "properties": { "os": { "const": "macos" } } - } - } - }, - "declaresLinuxOrDefault": { - "anyOf": [ - { "not": { "required": ["platform"] } }, - { - "required": ["platform"], - "properties": { - "platform": { "not": { "required": ["os"] } } - } - }, - { - "required": ["platform"], - "properties": { - "platform": { - "required": ["os"], - "properties": { "os": { "const": "linux" } } - } - } - } - ] - }, - "linuxInstanceShape": { - "required": ["vcpus", "memoryGb"], - "properties": { - "vcpus": { "type": "integer", "minimum": 1 }, - "memoryGb": { "type": "integer", "minimum": 1 } - }, - "description": "Both values must be powers of two; the bundled validator checks that constraint because JSON Schema cannot express it." - } - } -} diff --git a/resources/bundled/skills/factory-files/schemas/scorer.schema.json b/resources/bundled/skills/factory-files/schemas/scorer.schema.json deleted file mode 100644 index 1113d2e9675..00000000000 --- a/resources/bundled/skills/factory-files/schemas/scorer.schema.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "scorer.schema.json", - "$comment": "FORWARD COMPATIBILITY - DO NOT TIGHTEN. additionalProperties is true, and catalogue values are recorded as x-warp-known-values / x-warp-known-max-items annotations rather than enum, const, or maxItems. That is deliberate, not an oversight. This file ships inside a Warp release and is routinely OLDER than the warp-server it validates against, so a closed schema rejects configuration a newer server accepts and pushes agents to delete working fields. When the format gains a value, add it to the annotation; do not convert the annotation back into a rejecting keyword. The two deliberate exceptions are documented in specs/REMOTE-2727/TECH.md: trigger filter keys, which only apply when both provider and event are known, and schemaVersion, which stops validation rather than misapplying v1alpha1 rules. See also references/validation.md.", - "title": "scorers//scorer.md frontmatter (v1alpha1)", - "description": "YAML frontmatter of a Scorer file. The Scorer name comes from its directory, and the Markdown body is its required rubric.", - "type": "object", - "required": ["agents", "labels", "passingScore", "model"], - "additionalProperties": true, - "properties": { - "description": { - "type": ["string", "null"] - }, - "agents": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "common.schema.json#/$defs/nonEmptyString" - }, - "description": "Names of Agents declared by this Factory." - }, - "enabled": { - "type": "boolean", - "description": "Defaults to true." - }, - "output": { - "$ref": "common.schema.json#/$defs/nonEmptyString", - "x-warp-known-values": ["classification"], - "description": "Scorer output form. classification is the current v1 form; newer forms are preserved for forward compatibility." - }, - "labels": { - "type": "array", - "minItems": 1, - "x-warp-known-max-items": 20, - "description": "Classifications this Scorer may return. The current server accepts at most 20; a larger set is left for the server to judge.", - "items": { - "type": "object", - "required": ["value", "score"], - "additionalProperties": true, - "properties": { - "value": { - "$ref": "common.schema.json#/$defs/nonEmptyString" - }, - "description": { - "type": ["string", "null"] - }, - "score": { - "type": "number", - "minimum": 0, - "maximum": 1 - } - } - } - }, - "passingScore": { - "type": "number", - "minimum": 0, - "maximum": 1 - }, - "samplingRate": { - "type": "number", - "minimum": 0, - "maximum": 100, - "description": "Percentage of eligible runs to score. Defaults to 25. Zero is invalid; use enabled: false." - }, - "model": { - "$ref": "common.schema.json#/$defs/nonEmptyString" - }, - "selfImprovement": { - "type": "boolean", - "description": "Defaults to false." - } - } -} diff --git a/resources/bundled/skills/factory-files/scripts/validate_factory_files.py b/resources/bundled/skills/factory-files/scripts/validate_factory_files.py old mode 100755 new mode 100644 index 0ffd313275d..9b528fc8d84 --- a/resources/bundled/skills/factory-files/scripts/validate_factory_files.py +++ b/resources/bundled/skills/factory-files/scripts/validate_factory_files.py @@ -1,41 +1,35 @@ #!/usr/bin/env python3 -"""Validate a Factory file tree against the bundled v1alpha1 JSON Schemas. +"""Validate a Factory file tree against warp-server. Usage: - python3 validate_factory_files.py [FACTORY_ROOT] [--json] [--schemas DIR] + python3 validate_factory_files.py [FACTORY_ROOT] [--json] [--server-root URL] FACTORY_ROOT defaults to the current directory and must contain factory.yaml. -The script is intentionally dependency-free: it ships a restricted YAML reader -for the canonical forms this skill emits (no anchors, aliases, explicit tags, -or multiple documents) and a JSON Schema evaluator covering the keywords the -bundled schemas use. It is not a general-purpose YAML implementation. Anything -it cannot read confidently is reported rather than guessed at. - -It does not replace server-side validation. Provider catalogues, model IDs, -environment IDs, secret names, and runner references are resolved by the -server; this checks structure, field names, enums, and cross-file references. - -FORWARD COMPATIBILITY - DO NOT TIGHTEN --------------------------------------- -This validator and its schemas ship inside a Warp release, so they are -routinely older than the warp-server they are used against. They therefore -accept some input the current server rejects, on purpose. Unknown properties, -agent types, credential strategies, harness types and per-harness -capabilities, integration slugs, trigger providers and events, runner -platforms, Scorer output forms, and the Scorer label cap are all deferred to -the server. - -If you are here because the validator accepted something the server rejects, -the fix is usually a clearer server diagnostic, not a stricter schema. A false -rejection is far more expensive than a false acceptance: it blocks correct -work and invites an agent to "repair" valid configuration by deleting it, -whereas the server revalidates every tree at apply time anyway. - -Two checks are deliberately kept strict, and both are scoped so drift cannot -trip them: trigger filter keys apply only when the provider and event are both -recognized, and an unrecognized schemaVersion stops validation instead of -misapplying v1alpha1 rules. See specs/REMOTE-2727/TECH.md. +warp-server owns the Factory file format, so it is the only thing that decides +whether a tree is valid. This script collects the tree's resource files and +submits them to the validation endpoint, which runs the real parser plus the +state-independent checks the apply path would run next. + +There is deliberately no local fallback. A bundled copy of the format is +routinely older than the server it is used against, and a stale copy does not +degrade gracefully: it reports confident, wrong diagnostics that invite an +agent to "repair" valid configuration by deleting it. Reporting that a tree +could not be checked is strictly safer than reporting the wrong answer, so when +the server cannot be reached this script says so and validates nothing. + +That also means this script never parses YAML. It decides which files are +resource files from their paths alone and sends their bytes verbatim, so it +cannot disagree with the parser about what a document means. + +The endpoint does not resolve server state. Model IDs, environment IDs, secret +names, runner references, MCP server IDs, integration availability, and the +values of provider name aliases are all checked when the plan is applied. + +Exit codes: + 0 the server validated the tree and found no problem + 1 the server validated the tree and reported diagnostics + 2 the tree was not validated; the reason is printed """ from __future__ import annotations @@ -43,25 +37,43 @@ import argparse import json import os -import re import sys -import unicodedata +import urllib.error +import urllib.request from pathlib import Path from typing import Any, Optional -SCHEMA_BY_KIND = { - "factory": "factory.schema.json", - "agent": "agent.schema.json", - "automation": "automation.schema.json", - "runner": "runner.schema.json", - "scorer": "scorer.schema.json", -} +DEFAULT_SERVER_ROOT = "https://app.warp.dev" +VALIDATE_PATH = "/api/v1/factory-files/validate" + +# Bounded so an unreachable or slow server reports quickly rather than stalling +# an authoring session. +REQUEST_TIMEOUT_SECONDS = 10.0 +MAX_RESPONSE_BYTES = 8 * 1024 * 1024 + +# Mirrors the caps the endpoint enforces, so an oversized tree is reported here +# rather than collecting a 400. +MAX_REMOTE_FILES = 256 +MAX_REMOTE_FILE_BYTES = 256 * 1024 +MAX_REMOTE_CONTENT_BYTES = 2 * 1024 * 1024 + +SYMLINK_REFUSED = ( + "resource file is a symlink, or resolves outside the Factory root, and was not " + "read. The server parses the repository tree, so it sees the link itself rather " + "than its target and cannot accept this either. Replace it with a real file." +) + +EXIT_VALID = 0 +EXIT_DIAGNOSTICS = 1 +EXIT_NOT_VALIDATED = 2 -MAIN_AGENT_TYPES = {"MAIN", "FOREMAN"} + +class NotValidated(Exception): + """The tree was not checked. This is never a pass.""" class Problem: - """One validation failure, located as precisely as the input allows.""" + """One reported failure, located as precisely as the server allows.""" def __init__(self, path: str, message: str, line: Optional[int] = None, pointer: str = ""): self.path = path @@ -87,695 +99,17 @@ def render(self) -> str: # --------------------------------------------------------------------------- -# Restricted YAML reader +# Selecting the tree to submit # --------------------------------------------------------------------------- -class YamlError(Exception): - def __init__(self, message: str, line: int): - super().__init__(message) - self.message = message - self.line = line - - -class _Line: - __slots__ = ("number", "indent", "content") - - def __init__(self, number: int, indent: int, content: str): - self.number = number - self.indent = indent - self.content = content - - -# Characters after which a quote opens a quoted scalar. Anywhere else a quote -# is an ordinary character, so "It's a thing" stays a plain scalar rather than -# an unterminated string. -_VALUE_START_CHARS = ":,[{-" - - -def _strip_comment(raw: str, line_number: int) -> str: - """Remove a trailing comment, honoring quoted scalars.""" - out: list[str] = [] - quote: Optional[str] = None - previous = "" - index = 0 - while index < len(raw): - char = raw[index] - if quote: - out.append(char) - if char == "\\" and quote == '"' and index + 1 < len(raw): - out.append(raw[index + 1]) - index += 2 - continue - if char == quote: - if quote == "'" and index + 1 < len(raw) and raw[index + 1] == "'": - out.append("'") - index += 2 - continue - quote = None - index += 1 - continue - if char in "\"'" and (previous == "" or previous in _VALUE_START_CHARS): - quote = char - out.append(char) - previous = char - index += 1 - continue - if char == "#" and (index == 0 or raw[index - 1] in " \t"): - break - out.append(char) - if char not in " \t": - previous = char - index += 1 - if quote: - raise YamlError("unterminated quoted string", line_number) - return "".join(out).rstrip() - - -def _reject_unsupported(content: str, line_number: int) -> None: - """Reject line-level constructs the Factory file parser does not accept. +def classify(relative: str) -> tuple[str, str]: + """Mirror the server's path classification. Returns (kind, name). - Anchors, aliases, and tags are checked in [_parse_scalar] instead, because - they are only meaningful where a node begins; scanning the whole line - rejects ordinary prose such as "A & B" or "see *this*". + This decides only which files are worth submitting. The server classifies + them again and owns the verdict, so a disagreement here costs a wasted + upload rather than a wrong answer. """ - if content.strip() in ("---", "..."): - raise YamlError("multiple YAML documents are not permitted", line_number) - if re.match(r"^\s*<<\s*:", content): - raise YamlError("yaml merge keys are not permitted", line_number) - - -def _opens_block_scalar(content: str, line_number: int) -> bool: - while content.startswith("- "): - content = content[2:].lstrip() - entry = _split_key(content, line_number) - value = entry[1] if entry is not None else content - return value[:1] in ("|", ">") - - -def _skip_block_scalar_body(raw_lines: list[str], index: int, header_indent: int) -> int: - """Return the index of the first line after a block scalar's body.""" - while index < len(raw_lines): - raw = raw_lines[index] - if raw.strip() == "": - index += 1 - continue - if len(raw) - len(raw.lstrip(" ")) <= header_indent: - break - index += 1 - return index - - -def _read_lines(text: str) -> list[_Line]: - raw_lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") - lines: list[_Line] = [] - index = 0 - while index < len(raw_lines): - raw = raw_lines[index] - number = index + 1 - index += 1 - if "\t" in raw[: len(raw) - len(raw.lstrip(" \t"))]: - raise YamlError("tabs are not permitted for indentation", number) - content = _strip_comment(raw, number) - if not content.strip(): - continue - _reject_unsupported(content, number) - indent = len(content) - len(content.lstrip(" ")) - stripped = content.strip() - lines.append(_Line(number, indent, stripped)) - # A block scalar's body is opaque text. Leaving it out of the - # structural line list keeps its content from being read as YAML. - if _opens_block_scalar(stripped, number): - index = _skip_block_scalar_body(raw_lines, index, indent) - return lines - - -_INT_RE = re.compile(r"^[-+]?[0-9]+$") -_HEX_RE = re.compile(r"^[-+]?0x[0-9a-fA-F]+$") -_OCT_RE = re.compile(r"^[-+]?0o[0-7]+$") -_FLOAT_RE = re.compile(r"^[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?$") -_YAML_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}(?:[Tt ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[ \t]*(?:Z|[-+]\d{1,2}(?::\d{2})?))?)?$") - - -def _parse_scalar(token: str, line_number: int) -> Any: - token = token.strip() - if token == "" or token == "~" or token in ("null", "Null", "NULL"): - return None - if token.startswith("'"): - if not token.endswith("'") or len(token) < 2: - raise YamlError("invalid single-quoted string", line_number) - return token[1:-1].replace("''", "'") - if token.startswith('"'): - try: - return json.loads(token) - except json.JSONDecodeError as error: - raise YamlError(f"invalid double-quoted string: {error.msg}", line_number) from error - if token[0] in "&*": - raise YamlError("yaml anchors and aliases are not permitted", line_number) - if token[0] == "!": - raise YamlError("explicit yaml tags are not permitted", line_number) - if token in ("true", "True", "TRUE"): - return True - if token in ("false", "False", "FALSE"): - return False - if _INT_RE.match(token): - return int(token, 10) - if _HEX_RE.match(token): - return int(token, 16) - if _OCT_RE.match(token): - return int(token, 8) - if _FLOAT_RE.match(token): - return float(token) - if _YAML_DATE_RE.match(token): - raise YamlError("timestamps must be quoted so YAML keeps them as strings", line_number) - if token.lower() in {".inf", "+.inf", "-.inf", ".nan"}: - raise YamlError("non-finite YAML numbers are not permitted", line_number) - return token - - -def _split_key(content: str, line_number: int) -> Optional[tuple[str, str]]: - """Split `key: value`, honoring quoted keys. Returns None when absent.""" - quote: Optional[str] = None - index = 0 - while index < len(content): - char = content[index] - if quote: - if char == "\\" and quote == '"' and index + 1 < len(content): - index += 2 - continue - if char == quote: - if quote == "'" and index + 1 < len(content) and content[index + 1] == "'": - index += 2 - continue - quote = None - index += 1 - continue - if char in "\"'": - quote = char - index += 1 - continue - if char in "[{": - return None - if char == ":" and (index + 1 == len(content) or content[index + 1] in " \t"): - key_token = content[:index].strip() - key = _parse_scalar(key_token, line_number) - if not isinstance(key, str): - key = key_token - return key, content[index + 1 :].strip() - index += 1 - return None - - -def _parse_flow(text: str, line_number: int) -> Any: - value, rest = _parse_flow_value(text.strip(), line_number) - if rest.strip(): - raise YamlError("unexpected trailing content after flow collection", line_number) - return value - - -def _parse_flow_value(text: str, line_number: int) -> tuple[Any, str]: - text = text.lstrip() - if not text: - raise YamlError("unexpected end of flow collection", line_number) - if text[0] == "[": - items: list[Any] = [] - rest = text[1:].lstrip() - if rest.startswith("]"): - return items, rest[1:] - while True: - item, rest = _parse_flow_value(rest, line_number) - items.append(item) - rest = rest.lstrip() - if rest.startswith(","): - rest = rest[1:].lstrip() - if rest.startswith("]"): - return items, rest[1:] - continue - if rest.startswith("]"): - return items, rest[1:] - raise YamlError("unterminated flow sequence", line_number) - if text[0] == "{": - mapping: dict[str, Any] = {} - rest = text[1:].lstrip() - if rest.startswith("}"): - return mapping, rest[1:] - while True: - key_text, rest = _read_flow_scalar(rest, line_number) - rest = rest.lstrip() - if not rest.startswith(":"): - raise YamlError("flow mapping entry is missing ':'", line_number) - value, rest = _parse_flow_value(rest[1:], line_number) - key = _parse_scalar(key_text, line_number) - if not isinstance(key, str): - key = key_text.strip() - if key in mapping: - raise YamlError(f'duplicate key "{key}"', line_number) - mapping[key] = value - rest = rest.lstrip() - if rest.startswith(","): - rest = rest[1:].lstrip() - if rest.startswith("}"): - return mapping, rest[1:] - continue - if rest.startswith("}"): - return mapping, rest[1:] - raise YamlError("unterminated flow mapping", line_number) - token, rest = _read_flow_scalar(text, line_number) - return _parse_scalar(token, line_number), rest - - -def _read_flow_scalar(text: str, line_number: int) -> tuple[str, str]: - text = text.lstrip() - if text[:1] in ("'", '"'): - quote = text[0] - index = 1 - while index < len(text): - if text[index] == "\\" and quote == '"': - index += 2 - continue - if text[index] == quote: - if quote == "'" and text[index + 1 : index + 2] == "'": - index += 2 - continue - return text[: index + 1], text[index + 1 :] - index += 1 - raise YamlError("unterminated quoted string in flow collection", line_number) - index = 0 - while index < len(text) and text[index] not in ",]}:": - index += 1 - return text[:index].strip(), text[index:] - - -class _Reader: - def __init__(self, lines: list[_Line], raw_lines: list[str]): - self.lines = lines - self.raw_lines = raw_lines - self.index = 0 - - def peek(self) -> Optional[_Line]: - return self.lines[self.index] if self.index < len(self.lines) else None - - def parse_block(self, indent: int) -> Any: - line = self.peek() - if line is None or line.indent < indent: - return None - if line.content.startswith("- ") or line.content == "-": - return self._parse_sequence(line.indent) - return self._parse_mapping(line.indent) - - def _parse_sequence(self, indent: int) -> list[Any]: - items: list[Any] = [] - while True: - line = self.peek() - if line is None or line.indent != indent: - break - if not (line.content.startswith("- ") or line.content == "-"): - break - self.index += 1 - remainder = line.content[1:].strip() - if remainder == "": - items.append(self.parse_block(indent + 1)) - continue - entry = _split_key(remainder, line.number) - if entry is not None: - # An inline mapping entry opens a mapping whose remaining keys - # are indented to the column the first key started at. - after_dash = line.content[1:] - lead = len(after_dash) - len(after_dash.lstrip(" ")) - inline_indent = indent + 1 + lead - items.append(self._parse_inline_mapping(entry, line.number, inline_indent)) - continue - items.append(self._parse_value(remainder, line.number, indent)) - return items - - def _parse_inline_mapping( - self, entry: tuple[str, str], line_number: int, indent: int - ) -> dict[str, Any]: - key, value_token = entry - mapping: dict[str, Any] = {key: self._parse_value(value_token, line_number, indent)} - return self._parse_mapping(indent, existing=mapping) - - def _parse_mapping( - self, indent: int, existing: Optional[dict[str, Any]] = None - ) -> dict[str, Any]: - mapping: dict[str, Any] = existing if existing is not None else {} - while True: - line = self.peek() - if line is None or line.indent != indent: - break - if line.content.startswith("- "): - break - entry = _split_key(line.content, line.number) - if entry is None: - raise YamlError(f"expected 'key: value', found {line.content!r}", line.number) - self.index += 1 - key, value_token = entry - if key in mapping: - raise YamlError(f'duplicate key "{key}"', line.number) - mapping[key] = self._parse_value(value_token, line.number, indent) - return mapping - - def _parse_value(self, token: str, line_number: int, indent: int) -> Any: - if token.startswith("|") or token.startswith(">"): - return self._parse_block_scalar(token, line_number, indent) - if token.startswith("[") or token.startswith("{"): - return _parse_flow(token, line_number) - if token != "": - return _parse_scalar(token, line_number) - nested = self.peek() - if nested is None or nested.indent <= indent: - return None - return self.parse_block(nested.indent) - - def _parse_block_scalar(self, header: str, line_number: int, indent: int) -> str: - style = header[0] - chomp = "clip" - if "-" in header[1:]: - chomp = "strip" - elif "+" in header[1:]: - chomp = "keep" - collected: list[str] = [] - # line_number is 1-based, so it indexes the line after the header. - cursor = line_number - block_indent: Optional[int] = None - while cursor < len(self.raw_lines): - raw = self.raw_lines[cursor] - if raw.strip() == "": - collected.append("") - cursor += 1 - continue - current_indent = len(raw) - len(raw.lstrip(" ")) - if current_indent <= indent: - break - if block_indent is None: - block_indent = current_indent - collected.append(raw[block_indent:]) - cursor += 1 - if chomp != "keep": - while collected and collected[-1] == "": - collected.pop() - if style == "|": - text = "\n".join(collected) - else: - text = " ".join(part.strip() for part in collected if part.strip()) - if chomp == "strip" or not text: - return text - return text + "\n" - - -def load_yaml(text: str) -> Any: - """Parse the restricted YAML subset the Factory file format accepts.""" - raw_lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") - lines = _read_lines(text) - if not lines: - return None - reader = _Reader(lines, raw_lines) - value = reader.parse_block(lines[0].indent) - remaining = reader.peek() - if remaining is not None: - raise YamlError(f"unexpected content {remaining.content!r}", remaining.number) - return value - - -def split_frontmatter(text: str) -> tuple[str, str, int]: - """Return a Markdown resource's frontmatter, body, and line offset.""" - normalized = text.replace("\r\n", "\n").replace("\r", "\n") - lines = normalized.split("\n") - if not lines or lines[0].rstrip() != "---": - raise YamlError("resource file must start with a frontmatter fence (---)", 1) - for index in range(1, len(lines)): - if lines[index].rstrip() == "---": - return "\n".join(lines[1:index]), "\n".join(lines[index + 1 :]), 1 - raise YamlError("frontmatter is missing a closing fence (---)", 1) - - -# --------------------------------------------------------------------------- -# JSON Schema evaluator -# --------------------------------------------------------------------------- - -_TYPE_CHECKS = { - "object": lambda value: isinstance(value, dict), - "array": lambda value: isinstance(value, list), - "string": lambda value: isinstance(value, str), - "integer": lambda value: isinstance(value, int) and not isinstance(value, bool), - "number": lambda value: isinstance(value, (int, float)) and not isinstance(value, bool), - "boolean": lambda value: isinstance(value, bool), - "null": lambda value: value is None, -} - - -class SchemaStore: - """Loads sibling schema files and resolves local and relative $refs.""" - - def __init__(self, directory: Path): - self.directory = directory - self._cache: dict[str, Any] = {} - - def document(self, filename: str) -> Any: - if filename not in self._cache: - path = self.directory / filename - self._cache[filename] = json.loads(path.read_text(encoding="utf-8")) - return self._cache[filename] - - def resolve(self, ref: str, current: str) -> tuple[Any, str]: - filename, _, pointer = ref.partition("#") - target = filename or current - document = self.document(target) - node = document - for token in [segment for segment in pointer.split("/") if segment]: - token = token.replace("~1", "/").replace("~0", "~") - node = node[token] - return node, target - - -# Keywords the evaluator implements, and the annotations it may ignore. A -# keyword in neither set is reported rather than skipped: silently ignoring an -# unimplemented keyword would under-validate without any signal. -_SUPPORTED_KEYWORDS = frozenset( - { - "$ref", - "type", - "const", - "enum", - "minLength", - "maxLength", - "pattern", - "minimum", - "maximum", - "minItems", - "maxItems", - "uniqueItems", - "items", - "required", - "minProperties", - "properties", - "additionalProperties", - "propertyNames", - "allOf", - "anyOf", - "oneOf", - "not", - "if", - "then", - "else", - } -) -_ANNOTATION_KEYWORDS = frozenset( - { - "$schema", - "$id", - "$defs", - "$comment", - "title", - "description", - "x-warp-character-class", - "x-warp-known-max-items", - "x-warp-known-values", - "x-warp-max-trimmed-runes", - } -) - - - -def _matches_pattern(pattern: str, value: str) -> bool: - return re.search(pattern, value) is not None - - -def _describe(value: Any) -> str: - for name, check in _TYPE_CHECKS.items(): - if name != "number" and check(value): - return name - return type(value).__name__ - - -def validate_instance( - instance: Any, - schema: Any, - store: SchemaStore, - document: str, - pointer: str = "", -) -> list[str]: - """Evaluate the JSON Schema keywords used by the bundled schemas.""" - if schema is True or schema == {}: - return [] - if schema is False: - return [f"{pointer or '/'}: no value is allowed here"] - - errors: list[str] = [] - - unsupported = set(schema) - _SUPPORTED_KEYWORDS - _ANNOTATION_KEYWORDS - if unsupported: - listed = ", ".join(sorted(unsupported)) - errors.append( - f"{pointer or '/'}: this validator does not implement schema keyword(s) {listed}; " - "its result is incomplete until they are added" - ) - - if "$ref" in schema: - target, target_document = store.resolve(schema["$ref"], document) - errors.extend(validate_instance(instance, target, store, target_document, pointer)) - - if "type" in schema: - declared = schema["type"] - names = declared if isinstance(declared, list) else [declared] - if not any(_TYPE_CHECKS[name](instance) for name in names): - errors.append( - f"{pointer or '/'}: expected {' or '.join(names)}, found {_describe(instance)}" - ) - return errors - - if "const" in schema and instance != schema["const"]: - errors.append(f"{pointer or '/'}: must be {json.dumps(schema['const'])}") - if "enum" in schema and instance not in schema["enum"]: - allowed = ", ".join(json.dumps(option) for option in schema["enum"]) - errors.append(f"{pointer or '/'}: {json.dumps(instance)} must be one of {allowed}") - - if isinstance(instance, str): - if "minLength" in schema and len(instance) < schema["minLength"]: - required = schema["minLength"] - detail = "must not be empty" if required == 1 else f"must be at least {required} characters" - errors.append(f"{pointer or '/'}: {detail}") - if "maxLength" in schema and len(instance) > schema["maxLength"]: - errors.append(f"{pointer or '/'}: must be at most {schema['maxLength']} characters") - if "pattern" in schema and not _matches_pattern(schema["pattern"], instance): - errors.append(f"{pointer or '/'}: {json.dumps(instance)} does not match the required format") - - if isinstance(instance, (int, float)) and not isinstance(instance, bool): - if "minimum" in schema and instance < schema["minimum"]: - errors.append(f"{pointer or '/'}: must be at least {schema['minimum']}") - if "maximum" in schema and instance > schema["maximum"]: - errors.append(f"{pointer or '/'}: must be at most {schema['maximum']}") - - if isinstance(instance, list): - if "minItems" in schema and len(instance) < schema["minItems"]: - errors.append(f"{pointer or '/'}: must contain at least {schema['minItems']} entries") - if "maxItems" in schema and len(instance) > schema["maxItems"]: - errors.append(f"{pointer or '/'}: must contain at most {schema['maxItems']} entries") - if schema.get("uniqueItems") and _has_duplicates(instance): - errors.append(f"{pointer or '/'}: entries must be unique") - if "items" in schema: - for index, item in enumerate(instance): - errors.extend( - validate_instance(item, schema["items"], store, document, f"{pointer}/{index}") - ) - - if isinstance(instance, dict): - for name in schema.get("required", []): - if name not in instance: - errors.append(f"{pointer or '/'}: {name} is required") - if "minProperties" in schema and len(instance) < schema["minProperties"]: - errors.append(f"{pointer or '/'}: must declare at least {schema['minProperties']} field") - properties = schema.get("properties", {}) - for name, value in instance.items(): - if name in properties: - errors.extend( - validate_instance(value, properties[name], store, document, f"{pointer}/{name}") - ) - elif "additionalProperties" in schema: - additional = schema["additionalProperties"] - if additional is False: - known = ", ".join(sorted(properties)) or "none" - errors.append( - f"{pointer or '/'}: unknown field {json.dumps(name)} (accepted: {known})" - ) - else: - errors.extend( - validate_instance(value, additional, store, document, f"{pointer}/{name}") - ) - if "propertyNames" in schema: - for name in instance: - errors.extend( - validate_instance( - name, schema["propertyNames"], store, document, f"{pointer}/{name}" - ) - ) - - for subschema in schema.get("allOf", []): - errors.extend(validate_instance(instance, subschema, store, document, pointer)) - - if "anyOf" in schema: - branches = [ - validate_instance(instance, subschema, store, document, pointer) - for subschema in schema["anyOf"] - ] - if all(branch for branch in branches): - errors.append(_combine(pointer, schema, branches, "does not match any accepted form")) - - if "oneOf" in schema: - branches = [ - validate_instance(instance, subschema, store, document, pointer) - for subschema in schema["oneOf"] - ] - matched = [index for index, branch in enumerate(branches) if not branch] - if len(matched) == 0: - errors.append(_combine(pointer, schema, branches, "does not match any accepted form")) - elif len(matched) > 1: - errors.append( - f"{pointer or '/'}: matches more than one mutually exclusive form" - + (f" ({schema['description']})" if "description" in schema else "") - ) - - if "not" in schema and not validate_instance(instance, schema["not"], store, document, pointer): - errors.append( - f"{pointer or '/'}: " - + (schema.get("description") or "this form is not allowed here") - ) - - if "if" in schema: - matched = not validate_instance(instance, schema["if"], store, document, pointer) - branch = schema.get("then") if matched else schema.get("else") - if branch is not None: - errors.extend(validate_instance(instance, branch, store, document, pointer)) - - return errors - - -def _combine(pointer: str, schema: Any, branches: list[list[str]], summary: str) -> str: - detail = schema.get("description") - head = f"{pointer or '/'}: {detail or summary}" - nested = sorted({message for branch in branches for message in branch}) - if not nested: - return head - return head + " — " + "; ".join(nested[:4]) - - -def _has_duplicates(items: list[Any]) -> bool: - seen: list[str] = [] - for item in items: - key = json.dumps(item, sort_keys=True) - if key in seen: - return True - seen.append(key) - return False - - -# --------------------------------------------------------------------------- -# Factory tree traversal -# --------------------------------------------------------------------------- - - -def classify(relative: str) -> tuple[str, str]: - """Mirror the server's path classification. Returns (kind, name).""" if relative == "factory.yaml": return "factory", "" segments = relative.split("/") @@ -812,6 +146,7 @@ def classify(relative: str) -> tuple[str, str]: def _valid_name(name: str) -> bool: return name not in ("", ".", "..") and "/" not in name + def _resource_files(root: Path) -> list[Path]: files = [root / "factory.yaml"] for directory_name in ("agents", "automations", "runners", "scorers"): @@ -834,8 +169,7 @@ def _leaves_factory_root(path: Path, root: Path) -> bool: symlink is a blob whose content is the target path, so it sees the link itself. Following one here would both diverge from that and read a file the Factory does not contain - an untrusted repository could otherwise point a - resource at any readable path and have its content echoed back in a parse - error. + resource at any readable path and have its content uploaded. """ if path.is_symlink(): return True @@ -846,500 +180,135 @@ def _leaves_factory_root(path: Path, root: Path) -> bool: return False -SUPPORTED_SCHEMA_VERSION = "v1alpha1" - - -def _unsupported_schema_version(root: Path) -> Optional[Problem]: - """Report a tree whose schemaVersion these schemas do not describe. - - Validating a newer tree against v1alpha1 rules would bury the one useful - fact under a cascade of bogus unknown-field reports, so stop instead and - say the server is the authority. - """ - factory_file = root / "factory.yaml" - if _leaves_factory_root(factory_file, root): - # Leave the report to validate_tree, which names it as a link. - return None - try: - parsed = load_yaml(factory_file.read_text(encoding="utf-8")) - except (OSError, UnicodeError, YamlError): - return None - if not isinstance(parsed, dict): - return None - declared = parsed.get("schemaVersion") - if not isinstance(declared, str) or declared.strip() in ("", SUPPORTED_SCHEMA_VERSION): - return None - return Problem( - "factory.yaml", - f"these bundled schemas describe {SUPPORTED_SCHEMA_VERSION}, not " - f"{declared.strip()!r}, so this tree was not validated locally; check it " - "with the server instead of downgrading schemaVersion", - pointer="schemaVersion", - ) - - -def validate_tree(root: Path, store: SchemaStore) -> list[Problem]: - problems: list[Problem] = [] - documents: dict[str, tuple[str, str, Any]] = {} - seen_names: dict[tuple[str, str], str] = {} - +def collect_tree(root: Path) -> tuple[list[dict[str, str]], list[Problem]]: + """Collect the resource files to submit, refusing symlinks as the server does.""" if not (root / "factory.yaml").is_file(): - return [Problem("factory.yaml", "factory.yaml is required at the Factory root")] - - unsupported = _unsupported_schema_version(root) - if unsupported is not None: - return [unsupported] - + raise NotValidated(f"{root} has no factory.yaml, so it is not a Factory root") + files: list[dict[str, str]] = [] + problems: list[Problem] = [] + total = 0 for absolute in _resource_files(root): relative = absolute.relative_to(root).as_posix() - kind, name = classify(relative) - if kind in ("unrelated", "skill"): + kind, _ = classify(relative) + if kind in ("unrelated", "skill", "invalid"): continue - if kind == "invalid": - problems.append( - Problem( - relative, - "resource files must use factory.yaml, agents//agent.md, " - "automations//automation.md, runners/.yaml, " - "or scorers//scorer.md", - ) - ) - continue - if kind in ("automation", "runner", "scorer"): - previous = seen_names.get((kind, name)) - if previous is not None: - problems.append( - Problem(relative, f'{kind} "{name}" is also declared by {previous}') - ) - continue - seen_names[(kind, name)] = relative - if _leaves_factory_root(absolute, root): - problems.append( - Problem( - relative, - "resource file is a symlink, or resolves outside the Factory root, " - "and was not read. The server parses the repository tree, so it sees " - "the link itself rather than its target and cannot accept this " - "either. Replace it with a real file.", - ) - ) + problems.append(Problem(relative, SYMLINK_REFUSED)) continue - try: - text = absolute.read_text(encoding="utf-8") + content = absolute.read_text(encoding="utf-8") except (OSError, UnicodeError) as error: problems.append(Problem(relative, f"could not read UTF-8 resource: {error}")) continue - offset = 0 - body = "" - try: - if kind in ("agent", "automation", "scorer"): - frontmatter, body, offset = split_frontmatter(text) - parsed = load_yaml(frontmatter) if frontmatter.strip() else {} - else: - parsed = load_yaml(text) - except YamlError as error: - problems.append(Problem(relative, error.message, error.line + offset)) - continue - - if parsed is None: - parsed = {} - if not isinstance(parsed, dict): - problems.append(Problem(relative, "document root must be a YAML mapping")) - continue + encoded = len(content.encode("utf-8")) + if encoded > MAX_REMOTE_FILE_BYTES: + raise NotValidated(f"{relative} is larger than the endpoint accepts") + total += encoded + if total > MAX_REMOTE_CONTENT_BYTES or len(files) >= MAX_REMOTE_FILES: + raise NotValidated("the tree is larger than the endpoint accepts") + files.append({"path": relative, "content": content}) + if not files: + raise NotValidated("the tree has no resource files to submit") + return files, problems - documents[relative] = (kind, name, parsed) - schema = store.document(SCHEMA_BY_KIND[kind]) - for message in validate_instance(parsed, schema, store, SCHEMA_BY_KIND[kind]): - pointer, _, detail = message.partition(": ") - problems.append(Problem(relative, detail, pointer=pointer.lstrip("/").replace("/", "."))) - if kind == "automation": - problems.extend(_automation_semantics(relative, parsed)) - elif kind == "factory": - problems.extend(_factory_semantics(relative, parsed)) - elif kind == "runner": - problems.extend(_runner_semantics(relative, parsed)) - elif kind == "scorer": - problems.extend(_scorer_semantics(relative, parsed, body)) - - problems.extend(_validate_cross_file(documents)) - return problems +# --------------------------------------------------------------------------- +# Server-backed validation +# --------------------------------------------------------------------------- -def _scorer_semantics(relative: str, parsed: dict[str, Any], body: str) -> list[Problem]: - problems: list[Problem] = [] - if not body.strip(): - problems.append(Problem(relative, "the Markdown body is the rubric and must not be empty")) - agents = parsed.get("agents") - if isinstance(agents, list): - normalized_agents = [value.strip() for value in agents if isinstance(value, str)] - if len(set(normalized_agents)) != len(normalized_agents): - problems.append(Problem(relative, "agent names must be unique after trimming", pointer="agents")) +class Outcome: + """What the server found, and what it deliberately did not check.""" - labels = parsed.get("labels") - threshold = parsed.get("passingScore") - numeric_scores: list[float] = [] - if isinstance(labels, list): - seen_labels: set[str] = set() - for index, label in enumerate(labels): - if not isinstance(label, dict): - continue - value = label.get("value") - if isinstance(value, str): - normalized_value = value.strip() - if normalized_value in seen_labels: - problems.append( - Problem( - relative, - f'duplicate label "{normalized_value}"', - pointer=f"labels.{index}.value", - ) - ) - seen_labels.add(normalized_value) - score = label.get("score") - if isinstance(score, (int, float)) and not isinstance(score, bool): - numeric_scores.append(float(score)) - if ( - isinstance(threshold, (int, float)) - and not isinstance(threshold, bool) - and numeric_scores + def __init__( + self, + schema_version: str, + problems: list[Problem], + deferred: Optional[list[dict[str, Any]]] = None, ): - threshold_value = float(threshold) - if not any(score >= threshold_value for score in numeric_scores): - problems.append( - Problem( - relative, - "at least one label score must be at or above passingScore", - pointer="passingScore", - ) - ) - if not any(score < threshold_value for score in numeric_scores): - problems.append( - Problem( - relative, - "at least one label score must be below passingScore", - pointer="passingScore", - ) - ) - - sampling_rate = parsed.get("samplingRate") - if isinstance(sampling_rate, (int, float)) and not isinstance(sampling_rate, bool): - if float(sampling_rate) == 0: - problems.append( - Problem( - relative, - "samplingRate must not be 0; use enabled: false to stop scoring", - pointer="samplingRate", - ) - ) - return problems - - -def _factory_semantics(relative: str, parsed: dict[str, Any]) -> list[Problem]: - problems: list[Problem] = [] - alias = parsed.get("alias") - if isinstance(alias, str): - normalized_alias = alias.strip() - if len(normalized_alias) > 60: - problems.append(Problem(relative, "alias must not exceed 60 characters", pointer="alias")) - if any( - unicodedata.category(character)[:1] not in {"L", "N"} and character not in " _.-" - for character in normalized_alias - ): - problems.append( - Problem( - relative, - "alias may only contain letters, digits, spaces, '-', '_', and '.'", - pointer="alias", - ) - ) - - secrets = parsed.get("secrets") - if isinstance(secrets, list): - normalized = [value.strip() for value in secrets if isinstance(value, str)] - if len(set(normalized)) != len(normalized): - problems.append( - Problem(relative, "secret names must be unique after trimming", pointer="secrets") - ) - - repositories = parsed.get("repositories") - if isinstance(repositories, list): - seen: set[tuple[str, str]] = set() - for index, repository in enumerate(repositories): - if not isinstance(repository, dict): - continue - owner, name = repository.get("owner"), repository.get("name") - if not isinstance(owner, str) or not isinstance(name, str): - continue - key = (owner.strip(), name.strip()) - if key in seen: - problems.append( - Problem( - relative, - f"duplicate repository {key[0]}/{key[1]} after trimming", - pointer=f"repositories.{index}", - ) - ) - seen.add(key) - return problems - - -def _runner_semantics(relative: str, parsed: dict[str, Any]) -> list[Problem]: - shape = parsed.get("instanceShape") - platform = parsed.get("platform") - os_name = platform.get("os", "linux") if isinstance(platform, dict) else "linux" - if os_name != "linux" or not isinstance(shape, dict): - return [] - problems: list[Problem] = [] - for field in ("vcpus", "memoryGb"): - value = shape.get(field) - if isinstance(value, int) and not isinstance(value, bool) and value > 0: - if value & (value - 1): - problems.append( - Problem( - relative, - f"{field} must be a power of two for Linux runners", - pointer=f"instanceShape.{field}", - ) - ) - return problems - - -_CRON_DESCRIPTORS = { - "@yearly", - "@annually", - "@monthly", - "@weekly", - "@daily", - "@midnight", - "@hourly", -} -_DURATION_RE = re.compile( - r"^[+-]?(?:0|(?:(?:\d+(?:\.\d*)?|\.\d+)(?:ns|us|µs|μs|ms|s|m|h))+)$" -) -_MONTH_NAMES = { - "jan": 1, - "feb": 2, - "mar": 3, - "apr": 4, - "may": 5, - "jun": 6, - "jul": 7, - "aug": 8, - "sep": 9, - "oct": 10, - "nov": 11, - "dec": 12, -} -_DAY_NAMES = {"sun": 0, "mon": 1, "tue": 2, "wed": 3, "thu": 4, "fri": 5, "sat": 6} - - -def _cron_number(value: str, names: Optional[dict[str, int]]) -> Optional[int]: - if names is not None and value.lower() in names: - return names[value.lower()] - if not re.fullmatch(r"\d+", value): - return None - return int(value) - - -def _valid_cron_field( - field: str, minimum: int, maximum: int, names: Optional[dict[str, int]] = None -) -> bool: - for expression in filter(None, field.split(",")): - parts = expression.split("/") - if len(parts) > 2: - return False - base = parts[0] - if len(parts) == 2 and (not parts[1].isdigit() or int(parts[1]) == 0): - return False - if base in {"*", "?"}: - continue - bounds = base.split("-") - if len(bounds) > 2: - return False - start = _cron_number(bounds[0], names) - end = _cron_number(bounds[-1], names) - if start is None or end is None: - return False - if start < minimum or end > maximum or start > end: - return False - return bool(field) - - -def _valid_cron(expression: str) -> bool: - expression = expression.strip() - if expression in _CRON_DESCRIPTORS: - return True - if expression.startswith("@every "): - return _DURATION_RE.fullmatch(expression[len("@every ") :]) is not None - fields = expression.split() - if len(fields) != 5: - return False - return all( - validator - for validator in ( - _valid_cron_field(fields[0], 0, 59), - _valid_cron_field(fields[1], 0, 23), - _valid_cron_field(fields[2], 1, 31), - _valid_cron_field(fields[3], 1, 12, _MONTH_NAMES), - _valid_cron_field(fields[4], 0, 6, _DAY_NAMES), + self.schema_version = schema_version + self.problems = problems + self.deferred = deferred or [] + + def disclosure(self) -> str: + """The sentence the agent must repeat. Never claim more than ran.""" + return ( + f"Validated with the warp-server parser for {self.schema_version}; " + "state-dependent apply checks were not run." ) - ) -def _automation_semantics(relative: str, parsed: dict[str, Any]) -> list[Problem]: - """Report filter values listed in both in and not_in. - - Such a filter can never match, so the server rejects it rather than - persisting a silently dead subscription. JSON Schema cannot compare two - sibling arrays, so the check lives here. - """ - problems: list[Problem] = [] - triggers = parsed.get("triggers") - if not isinstance(triggers, list): - return problems - schedule_keys: set[str] = set() - for index, trigger in enumerate(triggers): - if not isinstance(trigger, dict): - continue - schedule = trigger.get("schedule") - if isinstance(schedule, dict): - name = schedule.get("name") - normalized_name = name.strip() if isinstance(name, str) else "" - key = f"name:{normalized_name}" if normalized_name else "unnamed" - if key in schedule_keys: - detail = ( - f'duplicate inline schedule name "{normalized_name}"' - if normalized_name - else "at most one inline schedule may omit name" - ) - problems.append( - Problem(relative, detail, pointer=f"triggers.{index}.schedule") - ) - schedule_keys.add(key) - cron = schedule.get("cron") - if isinstance(cron, str) and not _valid_cron(cron): - problems.append( - Problem( - relative, - f"invalid cron expression {json.dumps(cron)}", - pointer=f"triggers.{index}.schedule.cron", - ) - ) - - declared = trigger.get("filter") - if not isinstance(declared, dict): - continue - provider, event = trigger.get("provider"), trigger.get("event") - for field, matcher in declared.items(): - if not isinstance(matcher, dict): - continue - included = matcher.get("in") - excluded = matcher.get("not_in") - if not isinstance(included, list) or not isinstance(excluded, list): - continue - normalize = _matcher_normalizer(provider, event, field) - excluded_keys = {normalize(value) for value in excluded} - for value in included: - if normalize(value) in excluded_keys: - problems.append( - Problem( - relative, - f"{json.dumps(value)} is present in, or equivalent to a value in, " - "both in and not_in, " - "so this filter can never match", - pointer=f"triggers.{index}.filter.{field}", - ) - ) - return problems +def server_root(argument: Optional[str]) -> str: + """Resolve the server to ask, so a local or staging root needs no code change.""" + chosen = argument or os.environ.get("WARP_SERVER_ROOT") or DEFAULT_SERVER_ROOT + return chosen.rstrip("/") -def _matcher_normalizer(provider: Any, event: Any, field: str): - lowercase_fields: set[tuple[str, str]] = { - ("github", "assignees"), - ("github", "authors"), - ("github", "mentioned"), - ("github", "reviewers"), - ("github", "reviewer_teams"), - ("github", "review_states"), - ("github", "conclusions"), - ("github", "workflows"), - ("gitlab", "repos"), - ("gitlab", "actions"), - ("gitlab", "mentioned"), - ("linear", "mentioned_user_ids"), - ("linear", "labels"), - } - - def normalize(value: Any) -> Any: - if not isinstance(value, str): - return value - if provider == "github" and event == "push" and field == "branches": - return value[len("refs/heads/") :] if value.startswith("refs/heads/") else value - if provider == "slack" and field == "emojis": - emoji = value.strip().strip(":") - skin_tone = emoji.find("::skin-tone-") - if skin_tone >= 0: - emoji = emoji[:skin_tone] - return emoji.lower() - if field == "keywords" and provider in {"linear", "slack", "jira"}: - return value.strip().lower() - if (provider, field) in lowercase_fields: - return value.lower() - return value +def _request_json(url: str, token: Optional[str] = None, payload: Optional[Any] = None) -> Any: + """Post or fetch JSON, turning every failure class into NotValidated.""" + data = None + headers = {"Accept": "application/json"} + if payload is not None: + data = json.dumps(payload).encode("utf-8") + headers["Content-Type"] = "application/json" + if token: + headers["Authorization"] = "Bearer " + token + request = urllib.request.Request(url, data=data, headers=headers) + try: + with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response: + body = response.read(MAX_RESPONSE_BYTES + 1) + except urllib.error.HTTPError as error: + raise NotValidated(f"the server answered HTTP {error.code}") from error + except Exception as error: # DNS, TLS, connection, timeout, proxy, ... + raise NotValidated(f"the server could not be reached: {error}") from error + if len(body) > MAX_RESPONSE_BYTES: + raise NotValidated("the server response was implausibly large") + try: + return json.loads(body.decode("utf-8")) + except (UnicodeError, ValueError) as error: + raise NotValidated(f"the server response was not JSON: {error}") from error - return normalize +def validate(root: Path, base_url: str) -> Outcome: + """Submit the tree to the server, or raise NotValidated. -def _validate_cross_file(documents: dict[str, tuple[str, str, Any]]) -> list[Problem]: - """Check the tree-level rules that no single-document schema can express. + The endpoint reads the declared schemaVersion itself and reports an + unrecognized one as a diagnostic, so there is nothing to pre-flight and no + reason for this script to read the tree's YAML. - Runner references are deliberately not checked: a name the tree does not - declare legitimately resolves to an existing team runner on the server. + The endpoint needs no credential. WARP_API_KEY is forwarded when the + environment already carries one, as an Oz sandbox does, so the request is + attributable there; nothing requires it, because a local authoring agent + runs in a shell that cannot see the Warp client's session. """ - problems: list[Problem] = [] - agent_names: set[str] = set() - main_agents: list[str] = [] - - for relative, (kind, name, parsed) in documents.items(): - if kind == "agent": - agent_names.add(name) - if str(parsed.get("agentType", "")) in MAIN_AGENT_TYPES: - main_agents.append(relative) - - if not main_agents: + token = os.environ.get("WARP_API_KEY") + files, problems = collect_tree(root) + response = _request_json(base_url + VALIDATE_PATH, token=token, payload={"files": files}) + if not isinstance(response, dict) or not isinstance(response.get("diagnostics"), list): + raise NotValidated("the validation response was malformed") + + for diagnostic in response["diagnostics"]: + if not isinstance(diagnostic, dict): + raise NotValidated("the validation response was malformed") + code = str(diagnostic.get("code", "")) + message = str(diagnostic.get("message", "")) + line = diagnostic.get("line") problems.append( - Problem("factory.yaml", "exactly one Agent must declare agentType MAIN or FOREMAN") - ) - elif len(main_agents) > 1: - for relative in sorted(main_agents): - problems.append( - Problem(relative, "only one Agent may declare agentType MAIN or FOREMAN") + Problem( + str(diagnostic.get("path", "")), + f"{code}: {message}" if code else message, + line=line if isinstance(line, int) else None, ) - - for relative, (kind, _, parsed) in documents.items(): - if kind == "automation": - agent = parsed.get("agent") - if isinstance(agent, str) and agent not in agent_names: - problems.append( - Problem(relative, f'agent "{agent}" must name a declared Agent', pointer="agent") - ) - elif kind == "scorer": - agents = parsed.get("agents") - if not isinstance(agents, list): - continue - for index, agent in enumerate(agents): - if isinstance(agent, str) and agent.strip() not in agent_names: - problems.append( - Problem( - relative, - f'agent "{agent.strip()}" must name a declared Agent', - pointer=f"agents.{index}", - ) - ) - return problems + ) + deferred = [ + entry for entry in response.get("deferred_resolutions", []) if isinstance(entry, dict) + ] + reported = response.get("schema_version") + return Outcome( + reported if isinstance(reported, str) and reported else "unknown", + problems, + deferred, + ) def main() -> int: @@ -1347,25 +316,60 @@ def main() -> int: parser.add_argument("root", nargs="?", default=".", help="Factory root containing factory.yaml") parser.add_argument("--json", action="store_true", help="emit machine-readable output") parser.add_argument( - "--schemas", - default=str(Path(__file__).resolve().parent.parent / "schemas"), - help="directory holding the bundled JSON Schemas", + "--server-root", + default=None, + help="warp-server root to validate against; defaults to $WARP_SERVER_ROOT then " + + DEFAULT_SERVER_ROOT, ) args = parser.parse_args() root = Path(args.root).resolve() - store = SchemaStore(Path(args.schemas).resolve()) - problems = validate_tree(root, store) + try: + outcome = validate(root, server_root(args.server_root)) + except NotValidated as reason: + report = ( + f"This tree was NOT validated: {reason}. Nothing here says the files are " + "correct or incorrect. Validate against a reachable warp-server, and say " + "plainly that validation did not run." + ) + if args.json: + print(json.dumps({"validated": False, "reason": str(reason)}, indent=2)) + else: + print(report, file=sys.stderr) + return EXIT_NOT_VALIDATED + problems = outcome.problems if args.json: - print(json.dumps({"valid": not problems, "problems": [p.as_dict() for p in problems]}, indent=2)) - elif problems: + print( + json.dumps( + { + "validated": True, + "valid": not problems, + "schema_version": outcome.schema_version, + "disclosure": outcome.disclosure(), + "problems": [problem.as_dict() for problem in problems], + "deferred_resolutions": outcome.deferred, + }, + indent=2, + ) + ) + return EXIT_DIAGNOSTICS if problems else EXIT_VALID + + if problems: print(f"{len(problems)} problem(s) in {root}:", file=sys.stderr) for problem in problems: print(f" {problem.render()}", file=sys.stderr) - else: - print(f"{root}: factory files are valid against the v1alpha1 schemas") - return 1 if problems else 0 + print(outcome.disclosure(), file=sys.stderr) + return EXIT_DIAGNOSTICS + + print(f"{root}: factory files are valid.") + print(outcome.disclosure()) + for entry in outcome.deferred: + print( + f" deferred: {entry.get('path', '')} {entry.get('field', '')} " + f"({entry.get('kind', '')}) is resolved when the plan is applied" + ) + return EXIT_VALID if __name__ == "__main__": diff --git a/script/test_factory_files_skill.py b/script/test_factory_files_skill.py index 23e6f44ff82..a09be47a74b 100755 --- a/script/test_factory_files_skill.py +++ b/script/test_factory_files_skill.py @@ -1,40 +1,48 @@ #!/usr/bin/env python3 -"""Regression and packaging checks for the bundled factory-files skill. +"""Behavioral and packaging checks for the bundled factory-files skill. -Each case builds a throwaway Factory tree and asserts whether -resources/bundled/skills/factory-files/scripts/validate_factory_files.py -accepts it. The expected verdicts were verified against the authoritative Go -implementation in warp-server (factoryfile.ParseTree and -triggers.ValidateFilter), so a change here should be made only alongside a -matching change to the Factory file format. +The skill no longer carries a copy of the Factory file format. warp-server owns +the format and decides whether a tree is valid, so there is nothing here that +asserts which documents the format accepts - those cases live beside the parser +in warp-server (logic/factoryfile). -The final check runs prepare_bundled_resources and compares the packaged skill -tree byte-for-byte with the canonical source. +What is left to check is everything the client is still responsible for: -Some VALID_CASES assert that the validator TOLERATES input the current server -rejects. Those are not mistakes and must not be "corrected" into INVALID_CASES: -the schemas ship inside a Warp release and are routinely older than the server, -so they defer catalogue and limit decisions rather than rejecting a tree built -for a newer server. If one of them starts failing, a schema was tightened; -reopen the schema rather than moving the case. See specs/REMOTE-2727/TECH.md. +- which files it selects and uploads, and which it refuses to read +- that it reports the server's verdict as the server worded it +- that every way of not reaching a verdict is reported as "not validated" + rather than as a pass + +That last one is the point of the whole design. A stale local copy of the +format used to answer confidently and wrongly; the replacement must never +imply a tree was checked when it was not. Run directly, or via script/presubmit. """ from __future__ import annotations +import contextlib +import http.server +import json import os -import re import shutil import subprocess import sys import tempfile +import threading from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent SKILL = REPO_ROOT / "resources" / "bundled" / "skills" / "factory-files" VALIDATOR = SKILL / "scripts" / "validate_factory_files.py" -EXAMPLES = SKILL / "references" / "examples.md" + +EXIT_VALID = 0 +EXIT_DIAGNOSTICS = 1 +EXIT_NOT_VALIDATED = 2 + +DISCLOSURE = "Validated with the warp-server parser" +NOT_VALIDATED = "was NOT validated" FACTORY = """schemaVersion: v1alpha1 name: demo @@ -47,774 +55,376 @@ MAIN_AGENT = "---\nagentType: MAIN\n---\nDo the thing.\n" +CLEAN_RESPONSE = {"schema_version": "v1alpha1", "valid": True, "diagnostics": []} -def tree(**files: str) -> dict[str, str]: - """A minimal valid tree, overridden by the given files.""" - base = {"factory.yaml": FACTORY, "agents/main/agent.md": MAIN_AGENT} - base.update(files) - return base +class _FakeServer: + """A warp-server stand-in. -VALID_CASES: list[tuple[str, dict[str, str]]] = [ - ("minimal", tree()), - ( - "empty-agent-frontmatter", - tree( - **{ - "agents/main/agent.md": "---\nagentType: MAIN\n---\n", - "agents/aux/agent.md": "---\n---\nhelp out\n", - } - ), - ), - ( - "harness-claude-full", - tree( - **{ - "agents/main/agent.md": "---\nagentType: MAIN\nharness:\n type: claude\n" - " model: opus\n reasoningLevel: high\n auth:\n source: managedSecret\n" - " secretName: ANTHROPIC_KEY\n---\nx\n" - } - ), - ), - ( - "harness-claude-code-alias", - tree( - **{ - "agents/main/agent.md": "---\nagentType: MAIN\nharness:\n type: claude-code\n" - " model: opus\n---\nx\n" - } - ), - ), - ( - "harness-auth-clear", - tree( - **{ - "agents/main/agent.md": "---\nagentType: MAIN\nharness:\n type: codex\n" - " model: gpt-5.5\n auth: null\n---\nx\n" - } - ), - ), - ( - "harness-oz-model-only", - tree( - **{ - "agents/main/agent.md": "---\nagentType: MAIN\nharness:\n type: oz\n" - " model: auto\n---\nx\n" - } - ), - ), - ( - "agentdefaults-harness", - tree( - **{ - "factory.yaml": FACTORY.replace( - " model: auto\n", " harness:\n type: claude\n model: opus\n" - ) - } - ), - ), - ( - "inline-schedule", - tree( - **{ - "automations/nightly/automation.md": "---\ntriggers:\n - provider: schedule\n" - " event: cron_fired\n schedule:\n name: nightly\n" - " cron: 0 3 * * *\n---\nrun\n" - } - ), - ), - ( - "schedule-ids-not-in", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: schedule\n" - " event: cron_fired\n filter:\n schedule_ids: [sched_1]\n---\nrun\n" - } - ), - ), - ( - "descriptor-cron", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: schedule\n" - " event: cron_fired\n schedule:\n cron: '@daily'\n---\nrun\n" - } - ), - ), - ( - "macos-runner", - tree( - **{ - "runners/mac.yaml": "platform:\n os: macos\n arch: aarch64\n mac:\n" - " version: '15'\ninstanceShape:\n vcpus: 6\n memoryGb: 14\n" - } - ), - ), - ( - "linux-runner", - tree( - **{ - "runners/lin.yaml": "description: CI\nsetupCommands:\n - apt-get update -y\n" - "instanceShape:\n vcpus: 4\n memoryGb: 8\nplatform:\n os: linux\n" - " arch: x86_64\n linux:\n dockerImage: ubuntu:22.04\n" - } - ), - ), - ( - "linux-runner-default-os", - tree(**{"runners/lin.yaml": "platform:\n linux:\n dockerImage: ubuntu:22.04\n"}), - ), - ( - "integrations", - tree( - **{ - "factory.yaml": FACTORY - + "integrations:\n - type: slack\n - type: linear\n - type: jira\n" - } - ), - ), - ("integrations-empty", tree(**{"factory.yaml": FACTORY + "integrations: []\n"})), - ( - "cloud-providers-current-key", - tree( - **{ - "factory.yaml": FACTORY - + "cloudProviders:\n aws:\n roleArn: arn:aws:iam::123456789012:role/factory\n" - } - ), - ), - ( - "cloud-providers-current-and-legacy", - tree( - **{ - "factory.yaml": FACTORY - + "cloudProviders:\n aws:\n roleArn: arn:aws:iam::123456789012:role/current\n" - + "providers:\n aws:\n roleArn: arn:aws:iam::123456789012:role/legacy\n" - } - ), - ), - ( - "integrations-normalize-case-and-space", - tree(**{"factory.yaml": FACTORY + "integrations:\n - type: ' Slack '\n"}), - ), - ( - "workerhost-clear", - tree(**{"agents/main/agent.md": "---\nagentType: MAIN\nworkerHost: null\n---\nx\n"}), - ), - ( - "legacy-flat-automation", - tree( - **{ - "automations/flat.md": "---\ntriggers:\n - provider: github\n event: push\n" - " filter:\n repos: [warpdotdev/warp]\n---\nrun\n" - } - ), - ), - ( - "not-in-matcher", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: github\n" - " event: pull_request_opened\n filter:\n labels:\n in: [ready]\n" - " not_in: [wip]\n pr_numbers: [12, 13]\n---\nrun\n" - } - ), - ), - ("alias-ok", tree(**{"factory.yaml": FACTORY + "alias: Demo Factory-1.0_x\n"})), - ( - "skills-are-ignored", - tree( - **{ - "agents/main/skills/x/SKILL.md": "---\nname: x\n---\n", - "skills/y/SKILL.md": "---\nname: y\n---\n", - } - ), - ), - ( - "block-scalar-description", - tree( - **{ - "factory.yaml": "schemaVersion: v1alpha1\nname: demo\ndescription: |\n line one\n" - " line two\nrepositories:\n - owner: warpdotdev\n name: warp\n" - "agentDefaults:\n model: auto\n" - } - ), - ), - ( - "comments-and-quotes", - tree( - **{ - "factory.yaml": "# leading comment\nschemaVersion: v1alpha1 # trailing\n" - 'name: "demo: with colon"\nrepositories:\n - owner: warpdotdev\n name: warp\n' - "agentDefaults:\n model: auto\n" - } - ), - ), - # Prose in a plain scalar is not a quoted string, so an apostrophe must not - # read as an unterminated quote. - ( - "apostrophe-in-prose", - tree(**{"factory.yaml": FACTORY + "description: It's Warp's factory\n"}), - ), - # A block scalar's body is opaque text: emphasis, a document marker, and an - # ampersand are all literal there. - ( - "block-scalar-prose", - tree( - **{ - "factory.yaml": "schemaVersion: v1alpha1\nname: demo\ndescription: |\n" - " It's a summary with *emphasis*\n ---\n A & B\n" - "repositories:\n - owner: warpdotdev\n name: warp\n" - "agentDefaults:\n model: auto\n" - } - ), - ), - ( - "gitlab-and-factory-providers", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: gitlab\n" - " event: merge_request\n filter:\n repos: [acme/platform]\n" - " actions: [open]\n - provider: factory\n" - " event: work_item_stage_changed\n filter:\n stages: [REVIEW]\n---\nrun\n" - } - ), - ), - ( - "jira-agent-session-labels", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: jira\n" - " event: agent_session_created\n filter:\n project_keys: [ENG]\n" - " labels: [triage]\n keywords: [urgent]\n---\nrun\n" - } - ), - ), - # Jira label matching is case-sensitive, so values differing only in case - # are two distinct labels rather than a conflict. - ( - "jira-labels-are-case-sensitive", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: jira\n" - " event: issue_labeled\n filter:\n labels:\n in: [Bug]\n" - " not_in: [bug]\n---\nrun\n" - } - ), - ), - ( - "empty-and-null-matchers", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: github\n" - " event: pull_request_opened\n filter:\n labels: {}\n" - " authors: null\n---\nrun\n" - } - ), - ), - ( - "nullable-overrides", - tree( - **{ - "factory.yaml": FACTORY + "credentialStrategy: null\n", - "agents/main/agent.md": "---\nagentType: MAIN\nmodel: null\n" - "credentialStrategy: null\nrunner: null\nenvironmentId: null\n---\nx\n", - } - ), - ), - ( - "sparse-harness-null-fields", - tree( - **{ - "agents/main/agent.md": "---\nagentType: MAIN\nharness:\n type: claude\n" - " model: null\n auth: null\n---\nx\n", - } - ), - ), - ( - "unicode-and-padded-alias", - tree(**{"factory.yaml": FACTORY + "alias: ' café '\n"}), - ), - ( - "large-linux-power-of-two-shape", - tree( - **{ - "runners/large.yaml": "instanceShape:\n vcpus: 2048\n memoryGb: 2048\n" - "platform:\n os: linux\n linux:\n dockerImage: ubuntu:24.04\n" - } - ), - ), - ( - "macos-default-version", - tree(**{"runners/mac.yaml": "platform:\n os: macos\n arch: aarch64\n"}), - ), - ( - "named-cron-fields", - tree( - **{ - "automations/monthly/automation.md": "---\ntriggers:\n - provider: schedule\n" - " event: cron_fired\n schedule:\n cron: 0 3 1 JAN MON-FRI\n---\nx\n" - } - ), - ), - ( - "scorer-complete", - tree( - **{ - "agents/implementer/agent.md": "---\n---\nImplement.\n", - "scorers/tests/scorer.md": "---\nagents:\n - implementer\nlabels:\n" - " - value: pass\n score: 1\n - value: fail\n score: 0\n" - "passingScore: 1\nsamplingRate: 25\nmodel: claude-4-5-haiku\n" - "selfImprovement: true\n---\nEvaluate the run.\n", - } - ), - ), - # --------------------------------------------------------------- - # Deliberate forward-compatibility tolerances. - # - # Each case below is input the CURRENT server rejects and this - # validator accepts anyway, so that a Warp release older than the - # server does not block valid work. Do not move these to - # INVALID_CASES to "match the server" - that reintroduces exactly the - # false rejections these were added to prevent. - # --------------------------------------------------------------- - ( - "new-event-on-known-provider-leaves-filter-open", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: github\n" - " event: brand_new_github_event\n filter:\n" - " brand_new_filter_key: [x]\n---\nrun\n" - } - ), - ), - ( - "oz-harness-capability-limits-are-server-owned", - tree( - **{ - "agents/main/agent.md": "---\nagentType: MAIN\nharness:\n type: oz\n" - " model: auto\n reasoningLevel: high\n" - " auth:\n source: managedSecret\n secretName: K\n---\nx\n" - } - ), - ), - ( - "scorer-label-count-above-current-server-cap", - tree( - **{ - "scorers/many/scorer.md": "---\nagents: [main]\nlabels:\n" - + "".join( - f" - value: label_{index}\n score: {1 if index == 0 else 0}\n" - for index in range(21) - ) - + "passingScore: 1\nmodel: m\n---\nRubric.\n", - } - ), - ), - ( - "forward-compatible-unknowns", - tree( - **{ - "factory.yaml": FACTORY - + "futureFactoryField: enabled\n" - + "integrations:\n - type: future-provider\n", - "agents/main/agent.md": "---\nagentType: MAIN\nfutureAgentField: true\n" - "credentialStrategy: FUTURE\nharness:\n type: future-harness\n" - " model: future-model\nmcpServers:\n future:\n warpId: future\n" - " futureMcpField: true\n---\nx\n", - "agents/future/agent.md": "---\nagentType: FUTURE\n---\nx\n", - "automations/future/automation.md": "---\nfutureAutomationField: true\n" - "triggers:\n - provider: future-provider\n event: future_event\n" - " filter:\n future_filter: [value]\n---\nx\n", - "runners/future.yaml": "futureRunnerField: true\nplatform:\n" - " os: future-os\n arch: future-arch\n", - "scorers/future/scorer.md": "---\nagents: [main]\nlabels:\n" - " - value: pass\n score: 1\n - value: fail\n score: 0\n" - "passingScore: 1\nmodel: future-model\nfutureScorerField: true\n---\nRubric.\n", - } - ), - ), -] - -INVALID_CASES: list[tuple[str, dict[str, str]]] = [ - ( - "model-and-harness", - tree( - **{ - "agents/main/agent.md": "---\nagentType: MAIN\nmodel: auto\nharness:\n" - " type: oz\n model: auto\n---\nx\n" - } - ), - ), - ( - "agentdefaults-neither", - tree( - **{ - "factory.yaml": FACTORY.replace( - "agentDefaults:\n model: auto\n", "agentDefaults:\n runner: r\n" - ) - } - ), - ), - ( - "worker-env-with-secret", - tree( - **{ - "agents/main/agent.md": "---\nagentType: MAIN\nharness:\n type: claude\n" - " model: opus\n auth:\n source: workerEnvironment\n secretName: K\n---\nx\n" - } - ), - ), - ( - "managed-secret-without-name", - tree( - **{ - "agents/main/agent.md": "---\nagentType: MAIN\nharness:\n type: claude\n" - " model: opus\n auth:\n source: managedSecret\n---\nx\n" - } - ), - ), - ("empty-harness", tree(**{"agents/main/agent.md": "---\nagentType: MAIN\nharness: {}\n---\nx\n"})), - ("alias-bad-char", tree(**{"factory.yaml": FACTORY + "alias: demo/factory\n"})), - ("alias-too-long", tree(**{"factory.yaml": FACTORY + "alias: " + "a" * 61 + "\n"})), - ("two-main-agents", tree(**{"agents/other/agent.md": "---\nagentType: FOREMAN\n---\nx\n"})), - ( - "no-main-agent", - {"factory.yaml": FACTORY, "agents/main/agent.md": "---\nagentType: REVIEW\n---\nx\n"}, - ), - ( - "unknown-agent-ref", - tree( - **{ - "automations/n/automation.md": "---\nagent: nope\ntriggers:\n - provider: github\n" - " event: push\n---\nrun\n" - } - ), - ), - ( - "unknown-filter-key-on-known-provider-and-event", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: slack\n" - " event: app_mention\n filter:\n channels: [C1]\n---\nrun\n" - } - ), - ), - ( - "schedule-on-wrong-trigger", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: github\n" - " event: push\n schedule:\n cron: 0 3 * * *\n---\nrun\n" - } - ), - ), - ( - "schedule-and-schedule-ids", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: schedule\n" - " event: cron_fired\n filter:\n schedule_ids: [s1]\n schedule:\n" - " cron: 0 3 * * *\n---\nrun\n" - } - ), - ), - ( - "schedule-neither", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: schedule\n" - " event: cron_fired\n---\nrun\n" - } - ), - ), - ( - "cron-six-fields", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: schedule\n" - " event: cron_fired\n schedule:\n cron: 0 0 3 * * *\n---\nrun\n" - } - ), - ), - ( - "cron-tz-prefix", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: schedule\n" - " event: cron_fired\n schedule:\n cron: CRON_TZ=UTC 0 3 * * *\n---\nrun\n" - } - ), - ), - ("no-triggers", tree(**{"automations/n/automation.md": "---\nenabled: true\n---\nrun\n"})), - ("empty-triggers", tree(**{"automations/n/automation.md": "---\ntriggers: []\n---\nrun\n"})), - ("duplicate-secrets", tree(**{"factory.yaml": FACTORY + "secrets:\n - A\n - A\n"})), - ( - "empty-repositories", - tree( - **{ - "factory.yaml": "schemaVersion: v1alpha1\nname: demo\nrepositories: []\n" - "agentDefaults:\n model: auto\n" - } - ), - ), - ("bad-schema-version", tree(**{"factory.yaml": FACTORY.replace("v1alpha1", "v1beta1")})), - ("linux-no-docker-image", tree(**{"runners/lin.yaml": "platform:\n os: linux\n arch: x86_64\n"})), - ("runner-no-platform", tree(**{"runners/lin.yaml": "description: nothing\n"})), - ( - "linux-shape-not-power-of-two", - tree( - **{ - "runners/lin.yaml": "instanceShape:\n vcpus: 3\n memoryGb: 8\nplatform:\n" - " os: linux\n linux:\n dockerImage: u\n" - } - ), - ), - ( - "macos-with-linux-section", - tree(**{"runners/mac.yaml": "platform:\n os: macos\n linux:\n dockerImage: u\n"}), - ), - ("bad-agent-type", tree(**{"agents/main/agent.md": "---\nagentType: BOSS\n---\nx\n"})), - ("nested-agent-path", tree(**{"agents/team/extra/agent.md": "---\n---\nx\n"})), - ( - "duplicate-automation-name", - tree( - **{ - "automations/dup.md": "---\ntriggers:\n - provider: github\n event: push\n---\nrun\n", - "automations/dup/automation.md": "---\ntriggers:\n - provider: github\n" - " event: push\n---\nrun\n", - } - ), - ), - ( - "yaml-anchor", - tree( - **{ - "factory.yaml": "schemaVersion: v1alpha1\nname: demo\nrepositories:\n - &base\n" - " owner: warpdotdev\n name: warp\nagentDefaults:\n model: auto\n" - } - ), - ), - ( - "yaml-alias", - tree(**{"agents/main/agent.md": "---\nagentType: MAIN\nrunner: *base\n---\nx\n"}), - ), - ( - "yaml-tag", - tree(**{"agents/main/agent.md": "---\nagentType: MAIN\nrunner: !!str lin\n---\nx\n"}), - ), - ( - "matcher-in-and-not-in-conflict", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: github\n" - " event: pull_request_opened\n filter:\n labels:\n" - " in: [ready]\n not_in: [ready]\n---\nrun\n" - } - ), - ), - ( - "canonical-matcher-conflict", - tree( - **{ - "automations/n/automation.md": "---\ntriggers:\n - provider: slack\n" - " event: reaction_added\n filter:\n emojis:\n" - " in: [':eyes:']\n not_in: [eyes]\n---\nrun\n" - } - ), - ), - ("alias-emoji", tree(**{"factory.yaml": FACTORY + "alias: factory🚀\n"})), - ( - "trimmed-alias-too-long", - tree(**{"factory.yaml": FACTORY + "alias: ' " + "a" * 61 + " '\n"}), - ), - ( - "empty-harness-model", - tree( - **{ - "agents/main/agent.md": "---\nagentType: MAIN\nharness:\n" - " type: claude\n model: ''\n---\nx\n" - } - ), - ), - ( - "partial-runner-shape", - tree( - **{ - "runners/partial.yaml": "instanceShape:\n vcpus: 4\nplatform:\n os: linux\n" - " linux:\n dockerImage: ubuntu:24.04\n" - } - ), - ), - ( - "empty-macos-config", - tree(**{"runners/mac.yaml": "platform:\n os: macos\n arch: aarch64\n mac: {}\n"}), - ), - ( - "cron-field-out-of-range", - tree( - **{ - "automations/bad/automation.md": "---\ntriggers:\n - provider: schedule\n" - " event: cron_fired\n schedule:\n cron: 99 25 32 13 8\n---\nx\n" - } - ), - ), - ( - "cron-invalid-every-duration", - tree( - **{ - "automations/bad/automation.md": "---\ntriggers:\n - provider: schedule\n" - " event: cron_fired\n schedule:\n cron: '@every someday'\n---\nx\n" - } - ), - ), - ( - "duplicate-inline-schedule-name", - tree( - **{ - "automations/dup/automation.md": "---\ntriggers:\n" - " - provider: schedule\n event: cron_fired\n schedule:\n" - " name: same\n cron: 0 1 * * *\n" - " - provider: schedule\n event: cron_fired\n schedule:\n" - " name: same\n cron: 0 2 * * *\n---\nx\n" - } - ), - ), - ( - "duplicate-trimmed-secrets", - tree(**{"factory.yaml": FACTORY + "secrets: [' A ', A]\n"}), - ), - ( - "duplicate-trimmed-repositories", - tree( - **{ - "factory.yaml": "schemaVersion: v1alpha1\nname: demo\nrepositories:\n" - " - owner: ' warp '\n name: repo\n" - " - owner: warp\n name: repo\nagentDefaults:\n model: auto\n" - } - ), - ), - ( - "quoted-scalar-with-trailing-junk", - tree(**{"factory.yaml": FACTORY + 'description: "quoted" junk\n'}), - ), - ( - "unquoted-yaml-timestamp", - tree(**{"factory.yaml": FACTORY + "description: 2026-08-15\n"}), - ), - ( - "scorer-empty-rubric", - tree( - **{ - "scorers/empty/scorer.md": "---\nagents: [main]\nlabels:\n" - " - value: pass\n score: 1\n - value: fail\n score: 0\n" - "passingScore: 1\nmodel: m\n---\n", - } - ), - ), - ( - "scorer-unknown-agent", - tree( - **{ - "scorers/unknown/scorer.md": "---\nagents: [missing]\nlabels:\n" - " - value: pass\n score: 1\n - value: fail\n score: 0\n" - "passingScore: 1\nmodel: m\n---\nRubric.\n", - } - ), - ), - ( - "scorer-all-pass", - tree( - **{ - "scorers/all-pass/scorer.md": "---\nagents: [main]\nlabels:\n" - " - value: pass\n score: 1\n - value: better\n score: 0.9\n" - "passingScore: 0.5\nmodel: m\n---\nRubric.\n", - } - ), - ), - ( - "scorer-zero-sampling", - tree( - **{ - "scorers/zero/scorer.md": "---\nagents: [main]\nlabels:\n" - " - value: pass\n score: 1\n - value: fail\n score: 0\n" - "passingScore: 1\nsamplingRate: 0\nmodel: m\n---\nRubric.\n", - } - ), - ), - ( - "scorer-flat-path", - tree( - **{ - "scorers/flat.md": "---\nagents: [main]\nlabels:\n" - " - value: pass\n score: 1\n - value: fail\n score: 0\n" - "passingScore: 1\nmodel: m\n---\nRubric.\n", - } - ), - ), -] - + Records what the validator submitted so a test can assert the tree it sent, + not just the verdict it printed. + """ -def run_case(name: str, expect_valid: bool, files: dict[str, str]) -> bool: - root = Path(tempfile.mkdtemp(prefix="factory-files-case-")) + def __init__(self, httpd): + self._httpd = httpd + self.url = f"http://127.0.0.1:{httpd.server_address[1]}" + + def submitted_files(self) -> list[dict]: + return self._httpd.submitted_files + + def authorization(self) -> str: + return self._httpd.authorization + + +@contextlib.contextmanager +def fake_server(validate_body, validate_status: int = 200, raw_body: bytes | None = None): + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *_args): # keep the corpus output readable + pass + + def do_POST(self): # noqa: N802 - BaseHTTPRequestHandler's naming + length = int(self.headers.get("Content-Length", "0")) + payload = json.loads(self.rfile.read(length) or b"{}") + self.server.submitted_files.extend(payload.get("files", [])) + self.server.authorization = self.headers.get("Authorization", "") + if raw_body is not None: + encoded = raw_body + else: + encoded = json.dumps(validate_body).encode("utf-8") + self.send_response(validate_status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + httpd.submitted_files = [] + httpd.authorization = "" + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() try: - for relative, content in files.items(): + yield _FakeServer(httpd) + finally: + httpd.shutdown() + httpd.server_close() + thread.join(timeout=5) + + +def run_validator(root: Path, server_url: str, api_key: str = "", extra: list[str] | None = None): + environment = os.environ.copy() + environment.pop("WARP_API_KEY", None) + environment.pop("WARP_SERVER_ROOT", None) + if api_key: + environment["WARP_API_KEY"] = api_key + return subprocess.run( + [sys.executable, str(VALIDATOR), str(root), "--server-root", server_url] + + (extra or []), + capture_output=True, + text=True, + check=False, + env=environment, + ) + + +@contextlib.contextmanager +def factory_tree(**files: str): + """A minimal Factory root, plus any extra files, cleaned up afterwards.""" + root = Path(tempfile.mkdtemp(prefix="factory-files-")) + try: + contents = {"factory.yaml": FACTORY, "agents/main/agent.md": MAIN_AGENT} + contents.update(files) + for relative, content in contents.items(): path = root / relative path.parent.mkdir(parents=True, exist_ok=True) path.write_text(content, encoding="utf-8") - result = subprocess.run( - [sys.executable, str(VALIDATOR), str(root)], - capture_output=True, - text=True, - check=False, - ) - if (result.returncode == 0) == expect_valid: - return True - expected = "valid" if expect_valid else "invalid" - print(f"FAIL {name}: expected {expected}", file=sys.stderr) - output = (result.stdout + result.stderr).rstrip() - print(output or " (no output)", file=sys.stderr) - return False + yield root finally: shutil.rmtree(root, ignore_errors=True) -def documented_example_cases() -> list[tuple[str, bool, dict[str, str]]]: - """Assemble the code blocks in references/examples.md into valid trees. +def assert_submits_only_resource_files() -> None: + """Only canonical resource paths are uploaded. - The reference teaches by example, so an example that no longer validates is - a defect in the documentation. Block indices are positional: adding or - reordering a block in the reference means updating this mapping. + Skills can hold anything a repository wants to give an agent, and unrelated + files are none of the server's business, so neither is sent. """ - blocks = [ - body - for _, body in re.findall( - r"```(yaml|markdown)\n(.*?)```", EXAMPLES.read_text(encoding="utf-8"), re.S - ) - ] - expected_blocks = 11 - if len(blocks) != expected_blocks: - raise SystemExit( - f"examples.md has {len(blocks)} example blocks, expected {expected_blocks}; " - "update documented_example_cases()" - ) - minimal = {"factory.yaml": blocks[0], "agents/foreman/agent.md": blocks[1]} - full = { - "factory.yaml": blocks[2], - "agents/foreman/agent.md": blocks[1], - "agents/implementer/agent.md": blocks[3], - "agents/reviewer/agent.md": blocks[4], - "agents/investigator/agent.md": blocks[5], - "automations/pr-review/automation.md": blocks[6], - "automations/nightly-sweep/automation.md": blocks[7], - "runners/linux-standard.yaml": blocks[8], - "runners/macos-standard.yaml": blocks[9], - "scorers/tests-run/scorer.md": blocks[10], + with factory_tree( + **{ + "automations/nightly/automation.md": "---\nagent: main\n---\nrun\n", + "runners/linux.yaml": "platform:\n linux:\n dockerImage: ubuntu:24.04\n", + "scorers/tests/scorer.md": "---\nagents: [main]\n---\nRubric.\n", + "skills/helper/SKILL.md": "---\nname: helper\n---\nhelp\n", + "agents/main/skills/inner/SKILL.md": "---\nname: inner\n---\nhelp\n", + "README.md": "unrelated", + "agents/main/notes.txt": "unrelated", + } + ) as root: + with fake_server(CLEAN_RESPONSE) as server: + result = run_validator(root, server.url) + if result.returncode != EXIT_VALID: + raise RuntimeError(f"expected a clean pass, got: {result.stdout}{result.stderr}") + submitted = {entry["path"] for entry in server.submitted_files()} + expected = { + "factory.yaml", + "agents/main/agent.md", + "automations/nightly/automation.md", + "runners/linux.yaml", + "scorers/tests/scorer.md", + } + if submitted != expected: + raise RuntimeError( + f"unexpected submitted tree:\n missing={sorted(expected - submitted)}" + f"\n extra={sorted(submitted - expected)}" + ) + + +def assert_reports_the_servers_verdict() -> None: + """Diagnostics are relayed as the server worded them, never reinterpreted.""" + with factory_tree() as root: + response = { + "schema_version": "v1alpha1", + "valid": False, + "diagnostics": [ + { + "path": "factory.yaml", + "line": 2, + "column": 1, + "code": "FF_UNKNOWN_FIELD", + "message": 'unknown field "bogus"', + } + ], + } + with fake_server(response) as server: + result = run_validator(root, server.url) + if result.returncode != EXIT_DIAGNOSTICS: + raise RuntimeError(f"expected exit {EXIT_DIAGNOSTICS}, got {result.returncode}") + for fragment in ("FF_UNKNOWN_FIELD", 'unknown field "bogus"', "factory.yaml:2"): + if fragment not in result.stderr: + raise RuntimeError(f"the server diagnostic was not relayed: {result.stderr}") + + # An unrecognized schemaVersion is just a server diagnostic now. The + # client has no opinion about versions because it no longer reads YAML. + unsupported = { + "schema_version": "v9alpha1", + "valid": False, + "diagnostics": [ + { + "path": "factory.yaml", + "line": 1, + "column": 1, + "code": "FF_UNSUPPORTED_VERSION", + "message": "unsupported schemaVersion v9alpha1", + } + ], + } + with fake_server(unsupported) as server: + result = run_validator(root, server.url) + if result.returncode != EXIT_DIAGNOSTICS or "FF_UNSUPPORTED_VERSION" not in result.stderr: + raise RuntimeError(f"an unsupported version was not relayed: {result.stderr}") + + +def assert_surfaces_deferred_resolutions() -> None: + """A deferred provider alias is reported, not silently dropped. + + These are the values the endpoint deliberately did not prove. Hiding them + would let a clean result read as a guarantee the tree will apply. + """ + with factory_tree() as root: + response = { + "schema_version": "v1alpha1", + "valid": True, + "diagnostics": [], + "deferred_resolutions": [ + { + "path": "automations/t/automation.md", + "field": "triggers[0].filter.teams", + "kind": "linear_name_alias", + } + ], + } + with fake_server(response) as server: + result = run_validator(root, server.url) + if result.returncode != EXIT_VALID: + raise RuntimeError("a deferred alias should not fail the tree") + for fragment in ("deferred", "linear_name_alias", "triggers[0].filter.teams"): + if fragment not in result.stdout: + raise RuntimeError(f"deferred resolutions were not reported: {result.stdout}") + + +def assert_unreached_verdicts_are_never_a_pass() -> None: + """Every way of failing to reach the server exits 2 and says so. + + This is the check that matters most. The previous design answered from a + bundled copy of the format when the server was unavailable, and a stale + copy produced confident, wrong diagnostics. Silence is the safe failure, + but only if it is loud about being silence. + """ + cases = { + "http 401": dict(validate_status=401), + "http 429": dict(validate_status=429), + "http 500": dict(validate_status=500), + "response is not JSON": dict(raw_body=b"nope"), + "response is missing diagnostics": dict(validate_body={"schema_version": "v1alpha1"}), + "diagnostics are not objects": dict( + validate_body={"schema_version": "v1alpha1", "diagnostics": ["nope"]} + ), } - return [ - ("documented-minimal-example", True, minimal), - ("documented-full-example", True, full), - ] + with factory_tree() as root: + for name, options in cases.items(): + body = options.pop("validate_body", CLEAN_RESPONSE) + with fake_server(body, **options) as server: + result = run_validator(root, server.url) + if result.returncode != EXIT_NOT_VALIDATED: + raise RuntimeError( + f"{name}: expected exit {EXIT_NOT_VALIDATED}, got {result.returncode}" + ) + combined = result.stdout + result.stderr + if NOT_VALIDATED not in combined: + raise RuntimeError(f"{name}: did not report that nothing was validated") + if DISCLOSURE in combined: + raise RuntimeError(f"{name}: claimed a server verdict it never received") + + # An unreachable server is the same case, and must not hang. + result = run_validator(root, "http://127.0.0.1:9") + if result.returncode != EXIT_NOT_VALIDATED or NOT_VALIDATED not in result.stderr: + raise RuntimeError(f"an unreachable server was not reported: {result.stderr}") + + # A directory that is not a Factory root is also not a verdict. + empty = Path(tempfile.mkdtemp(prefix="factory-files-empty-")) + try: + result = run_validator(empty, "http://127.0.0.1:9") + if result.returncode != EXIT_NOT_VALIDATED or "factory.yaml" not in result.stderr: + raise RuntimeError(f"a non-Factory directory was not reported: {result.stderr}") + finally: + shutil.rmtree(empty, ignore_errors=True) + + +def assert_credentials_are_optional_but_forwarded() -> None: + """The endpoint needs no key; one is forwarded when the environment has it. + + Requiring a key would disable validation for local authoring agents, whose + shell cannot see the Warp client's session. + """ + with factory_tree() as root: + with fake_server(CLEAN_RESPONSE) as server: + result = run_validator(root, server.url) + if result.returncode != EXIT_VALID: + raise RuntimeError("validation should not require a credential") + if server.authorization(): + raise RuntimeError("an Authorization header was sent without a key") + + with fake_server(CLEAN_RESPONSE) as server: + result = run_validator(root, server.url, api_key="wk-1.abc") + if result.returncode != EXIT_VALID: + raise RuntimeError("validation failed when a credential was present") + if server.authorization() != "Bearer wk-1.abc": + raise RuntimeError(f"the key was not forwarded: {server.authorization()!r}") + + +def assert_symlinked_resources_are_refused() -> None: + """A symlinked resource is reported and never uploaded. + + git stores a symlink as a blob holding the target path, so the server sees + the link rather than its target and cannot accept one either. Following it + here would both diverge from that and upload a file the Factory does not + contain. + """ + canary = "CANARY-SHOULD-NEVER-BE-UPLOADED" + outside = Path(tempfile.mkdtemp(prefix="factory-files-outside-")) + try: + secret = outside / "secret.txt" + secret.write_text(f"{canary}\n", encoding="utf-8") + + # A link out of the tree. + with factory_tree() as root: + (root / "runners").mkdir(exist_ok=True) + (root / "runners" / "escapes.yaml").symlink_to(secret) + with fake_server(CLEAN_RESPONSE) as server: + result = run_validator(root, server.url) + uploaded = json.dumps(server.submitted_files()) + combined = result.stdout + result.stderr + if result.returncode != EXIT_DIAGNOSTICS or "symlink" not in combined: + raise RuntimeError(f"an escaping symlink was not refused: {combined}") + if canary in combined or canary in uploaded: + raise RuntimeError("the symlink target's content escaped") + + # A link whose target is inside the tree is refused too: the server + # still sees the link, not the file it points at. + with factory_tree() as root: + (root / "runners").mkdir(exist_ok=True) + (root / "runners" / "real.yaml").write_text( + "platform:\n linux:\n dockerImage: ubuntu:24.04\n", encoding="utf-8" + ) + (root / "runners" / "alias.yaml").symlink_to(root / "runners" / "real.yaml") + with fake_server(CLEAN_RESPONSE) as server: + result = run_validator(root, server.url) + submitted = {entry["path"] for entry in server.submitted_files()} + if result.returncode != EXIT_DIAGNOSTICS: + raise RuntimeError("an in-tree symlink was accepted") + if "runners/alias.yaml" in submitted: + raise RuntimeError("an in-tree symlink was uploaded") + if "runners/real.yaml" not in submitted: + raise RuntimeError("the real file beside the symlink was not uploaded") + finally: + shutil.rmtree(outside, ignore_errors=True) + + +def assert_oversized_trees_are_reported() -> None: + """A tree past the endpoint's caps is reported here, not sent and rejected.""" + with factory_tree(**{"runners/big.yaml": "#" * (256 * 1024 + 1)}) as root: + with fake_server(CLEAN_RESPONSE) as server: + result = run_validator(root, server.url) + if server.submitted_files(): + raise RuntimeError("an oversized tree was uploaded anyway") + if result.returncode != EXIT_NOT_VALIDATED or "larger than" not in result.stderr: + raise RuntimeError(f"an oversized file was not reported: {result.stderr}") + + +def assert_json_output_never_implies_a_verdict() -> None: + """--json distinguishes "checked and clean" from "not checked".""" + with factory_tree() as root: + with fake_server(CLEAN_RESPONSE) as server: + result = run_validator(root, server.url, extra=["--json"]) + payload = json.loads(result.stdout) + if payload != { + "validated": True, + "valid": True, + "schema_version": "v1alpha1", + "disclosure": payload["disclosure"], + "problems": [], + "deferred_resolutions": [], + }: + raise RuntimeError(f"unexpected clean payload: {payload}") + + result = run_validator(root, "http://127.0.0.1:9", extra=["--json"]) + payload = json.loads(result.stdout) + if payload.get("validated") is not False or "valid" in payload: + raise RuntimeError( + f"an unvalidated tree must not report a validity verdict: {payload}" + ) + + +def assert_no_format_copy_remains() -> None: + """The skill must not regrow a local copy of the Factory file format. + + A bundled copy ships inside a Warp release and goes stale against the + server, which is what produced confidently wrong diagnostics before. If a + future change needs the format, fetch it from the server. + """ + schemas = list(SKILL.rglob("*.schema.json")) + if schemas: + raise RuntimeError( + "the skill has regrown bundled schemas, which go stale against the " + "server and produce false rejections; fetch the format instead:\n " + + "\n ".join(str(path.relative_to(SKILL)) for path in schemas) + ) + source = VALIDATOR.read_text(encoding="utf-8") + for banned in ("import yaml", "def load_yaml", "jsonschema"): + if banned in source: + raise RuntimeError( + f"the validator parses the format again ({banned!r}); it should send " + "bytes to the server and relay the verdict" + ) def assert_packaged_skill_matches() -> None: @@ -830,15 +440,9 @@ def assert_packaged_skill_matches() -> None: check=True, ) packaged = Path(destination) / "bundled" / "skills" / "factory-files" - source_files = { - path.relative_to(SKILL) - for path in SKILL.rglob("*") - if path.is_file() - } + source_files = {path.relative_to(SKILL) for path in SKILL.rglob("*") if path.is_file()} packaged_files = { - path.relative_to(packaged) - for path in packaged.rglob("*") - if path.is_file() + path.relative_to(packaged) for path in packaged.rglob("*") if path.is_file() } if source_files != packaged_files: raise RuntimeError( @@ -849,145 +453,34 @@ def assert_packaged_skill_matches() -> None: if (SKILL / relative).read_bytes() != (packaged / relative).read_bytes(): raise RuntimeError(f"packaged content differs for {relative}") -# Keywords that reject an otherwise well-formed value because it is not in a -# hard-coded list. Every one of these is a place a newer server could legitimately -# widen, so they are banned outside the narrow exceptions below. -_CLOSED_KEYWORDS = ("enum", "const", "maxItems") - - -def _walk_schema(node, path, found): - if isinstance(node, dict): - for keyword in _CLOSED_KEYWORDS: - if keyword in node: - found.append((path, keyword)) - if node.get("additionalProperties") is False: - found.append((path, "additionalProperties:false")) - for key, value in node.items(): - _walk_schema(value, f"{path}/{key}", found) - elif isinstance(node, list): - for index, value in enumerate(node): - _walk_schema(value, f"{path}/{index}", found) - - -def assert_schemas_stay_forward_compatible() -> None: - """Fail if a schema was closed back up against future server changes. - - The bundled schemas ship inside a Warp release and are routinely older than - the warp-server they validate against, so rejecting unknown properties or - unknown catalogue values would block configuration a newer server accepts. - New values belong in an x-warp-known-values annotation instead. - - Three exceptions are allowed, all scoped so drift cannot trip them: - - - `if` conditions select which rule applies; they never reject on their own. - - `$defs` named `declares*` exist only to be referenced from an `if`, so they - are conditions too. Keep that naming convention for new ones. - - Per-(provider, event) trigger filter objects close their key set, because - a misspelled filter key is a common mistake that otherwise survives until - apply. Those rules only fire when both provider and event are recognized. - """ - import json - - offenders: list[str] = [] - for schema_path in sorted((SKILL / "schemas").glob("*.schema.json")): - document = json.loads(schema_path.read_text(encoding="utf-8")) - found: list[tuple[str, str]] = [] - _walk_schema(document, "", found) - for path, keyword in found: - if "/if/" in path or path.endswith("/if"): - continue - if path.startswith("/$defs/declares"): - continue - if keyword == "additionalProperties:false" and path.endswith("/then/properties/filter"): - continue - offenders.append(f"{schema_path.name}{path or '/'} uses {keyword}") - if offenders: - raise RuntimeError( - "these schemas were tightened against future server changes, which " - "would reject trees a newer server accepts; record new values in an " - "x-warp-known-values annotation instead (see " - "specs/REMOTE-2727/TECH.md):\n " + "\n ".join(offenders) - ) - -def assert_symlinked_resources_are_refused() -> None: - """A resource file that links out of the tree must not be read. - - The dict-based cases above cannot express a symlink, so this builds one - directly. Two things are asserted: the tree is rejected, and the link - target's content never reaches the output. The server parses an in-memory - git tree, where a symlink is a blob holding the target path, so it never - reads the target either. - """ - canary = "CANARY-SHOULD-NEVER-BE-ECHOED" - root = Path(tempfile.mkdtemp(prefix="factory-files-symlink-")) - try: - (root / "agents" / "main").mkdir(parents=True) - (root / "runners").mkdir() - (root / "factory.yaml").write_text(FACTORY, encoding="utf-8") - (root / "agents" / "main" / "agent.md").write_text(MAIN_AGENT, encoding="utf-8") - - outside = Path(tempfile.mkdtemp(prefix="factory-files-outside-")) - try: - secret = outside / "secret.txt" - secret.write_text(f"{canary}\nnot a mapping: [\n", encoding="utf-8") - (root / "runners" / "linked.yaml").symlink_to(secret) - - result = subprocess.run( - [sys.executable, str(VALIDATOR), str(root)], - capture_output=True, - text=True, - check=False, - ) - output = result.stdout + result.stderr - if result.returncode == 0: - raise RuntimeError("a symlinked resource file was accepted") - if canary in output: - raise RuntimeError( - "the validator echoed the content of a file outside the Factory root" - ) - if "symlink" not in output: - raise RuntimeError(f"expected a symlink diagnostic, got: {output.strip()}") - finally: - shutil.rmtree(outside, ignore_errors=True) - - # A link whose target is inside the root is refused too. The escape - # check cannot see this one, so it exercises the is_symlink branch: the - # server still reads the link rather than its target. - (root / "runners" / "linked.yaml").unlink() - (root / "runners" / "real.yaml").write_text( - "platform:\n linux:\n dockerImage: ubuntu:24.04\n", encoding="utf-8" - ) - (root / "runners" / "alias.yaml").symlink_to(root / "runners" / "real.yaml") - result = subprocess.run( - [sys.executable, str(VALIDATOR), str(root)], - capture_output=True, - text=True, - check=False, - ) - if result.returncode == 0: - raise RuntimeError("a symlink pointing inside the Factory root was accepted") - finally: - shutil.rmtree(root, ignore_errors=True) +CHECKS = ( + ("only resource files are submitted", assert_submits_only_resource_files), + ("the server's verdict is relayed verbatim", assert_reports_the_servers_verdict), + ("deferred resolutions are surfaced", assert_surfaces_deferred_resolutions), + ("an unreached verdict is never a pass", assert_unreached_verdicts_are_never_a_pass), + ("credentials are optional but forwarded", assert_credentials_are_optional_but_forwarded), + ("symlinked resources are refused", assert_symlinked_resources_are_refused), + ("oversized trees are reported", assert_oversized_trees_are_reported), + ("json output never implies a verdict", assert_json_output_never_implies_a_verdict), + ("no local copy of the format remains", assert_no_format_copy_remains), + ("source and bundled trees match", assert_packaged_skill_matches), +) def main() -> int: - # Run first: tightening a schema also fails several corpus cases, and this - # explains why rather than leaving the reader to infer it from a rejection. - assert_schemas_stay_forward_compatible() - cases = [(name, True, files) for name, files in VALID_CASES] - cases += [(name, False, files) for name, files in INVALID_CASES] - cases += documented_example_cases() - failures = [name for name, expect, files in cases if not run_case(name, expect, files)] + failures = 0 + for description, check in CHECKS: + try: + check() + except Exception as error: # noqa: BLE001 - the report is the point + failures += 1 + print(f"FAIL factory-files: {description}\n {error}", file=sys.stderr) + else: + print(f"factory-files: {description}") if failures: - print(f"{len(failures)}/{len(cases)} factory-files validator cases failed", file=sys.stderr) + print(f"{failures}/{len(CHECKS)} factory-files checks failed", file=sys.stderr) return 1 - assert_symlinked_resources_are_refused() - assert_packaged_skill_matches() - print(f"factory-files validator: {len(cases)}/{len(cases)} cases passed") - print("factory-files schemas: still open to future server changes") - print("factory-files resources: symlinks out of the tree are refused") - print("factory-files packaging: source and bundled trees match") return 0 diff --git a/specs/REMOTE-2727/TECH.md b/specs/REMOTE-2727/TECH.md index bc6efc9bd29..7faecfb7f4b 100644 --- a/specs/REMOTE-2727/TECH.md +++ b/specs/REMOTE-2727/TECH.md @@ -8,6 +8,8 @@ This work needs only a technical spec. The user workflow is already defined, and The implementation uses `warpdotdev/warp` as its primary repository. Warp owns the canonical skill copy and the shared distribution path for three of the four requested surfaces. `warp-server` remains authoritative for the Factory file schema. The Claude Code and Codex plugin repositories contain downstream mirrors. +The schema-ownership, drift, trigger-filter validation, and follow-up design in this document is superseded by [`warpdotdev/warp-server/specs/REMOTE-2868/TECH.md`](https://github.com/warpdotdev/warp-server/blob/develop/specs/REMOTE-2868/TECH.md). REMOTE-2868 moves schema generation and parser-backed validation to `warp-server` while retaining this skill's permissive offline fallback. This document remains authoritative for the skill trigger, authoring workflow, canonical paths, symlink policy, and boundary against `factory-mcp`. + ## Context The Factory file parser accepts a versioned, path-derived tree: @@ -152,7 +154,9 @@ The openness is easy to mistake for an unfinished edge and quietly undo, so it i Two rules are deliberately not checked, because the server accepts them and a check would produce false failures: a `runner` name may resolve to an existing team runner the tree does not declare, and every server-resolved value (model IDs, environment IDs, secret names, MCP IDs) needs state the validator does not have. `SKILL.md` and `references/validation.md` both state this boundary so the agent does not overstate what a clean run proves. -The trigger filter catalogue is the one catalogue still enforced, and it is the highest-value check here: the parser accepts any mapping as a `filter` and defers key validation to apply time, so a wrong filter key currently survives review. It is kept drift-safe by scoping it, because every filter rule fires only when both the provider and the event match values these schemas know. A newer provider, or a newer event on a known provider, matches no rule and leaves its filter unconstrained, so the check cannot reject a tree built for a newer server. The residual gap is a new filter key added to an existing provider/event pair, which the server would still catch at apply time. The `warp-server` fixture at `logic/factoryfile/testdata/valid/automations/triage/automation.md` contains exactly this defect today: it uses `teams`, `projects`, `states`, `issues`, `baseBranches`, `channels`, `users`, and `itemUsers`, none of which `triggers.CanonicalizeFilter` accepts. That fixture should be corrected separately. +The trigger filter catalogue is the one catalogue still enforced, and it is the highest-value check here: the parser accepts any mapping as a `filter` and defers key validation to apply time, so a wrong filter key currently survives review. It is kept drift-safe by scoping it, because every filter rule fires only when both the provider and the event match values these schemas know. A newer provider, or a newer event on a known provider, matches no rule and leaves its filter unconstrained, so the check cannot reject a tree built for a newer server. The residual gap is a new filter key added to an existing provider/event pair, which the server would still catch at apply time. + +Correction: `teams`, `projects`, `states`, `issues`, `baseBranches`, `channels`, `users`, and `itemUsers` in `logic/factoryfile/testdata/valid/automations/triage/automation.md` are supported authoring aliases, not a defective fixture. Apply rewrites the GitHub aliases locally and resolves the Linear and Slack name aliases through provider snapshots before `triggers.CanonicalizeFilter`. The validator must not pass the authored keys directly to canonical validation. REMOTE-2868 defines the key-by-key policy. A parser-backed CLI or an API that validates an arbitrary source tree remains a follow-up. Do not add either endpoint in this work item. @@ -303,7 +307,7 @@ No visual recording is required. This feature changes agent context and generate ## Findings worth acting on separately - The `alias` rule is not what the bug-bash notes assumed. `factoryalias.Normalize` accepts Unicode letters, digits, spaces, `-`, `_`, and `.` up to 60 runes, and preserves case; uniqueness folds case in the comparison key only. There is no lowercase or hyphen-separated requirement. The schemas and reference encode the implemented rule. If the intended product rule really is lowercase-and-hyphenated, that is a server change, not a schema change. -- `logic/factoryfile/testdata/valid/automations/triage/automation.md` uses filter keys the apply step rejects. It passes today only because `ParseTree` does not validate filter keys. Worth fixing in `warp-server` so the fixture stops teaching the wrong spelling. +- `logic/factoryfile/testdata/valid/automations/triage/automation.md` uses supported authoring aliases. The earlier claim that apply rejects them was incorrect; apply rewrites or resolves them before canonical validation. - Filter keys being parser-accepted and apply-rejected is the underlying gap. Validating filters during parse, or at least during plan, would move the error to where the author can see it. ## Risks and mitigations