diff --git a/.changeset/agent-skills-token-based-length.md b/.changeset/agent-skills-token-based-length.md new file mode 100644 index 00000000..468f2f40 --- /dev/null +++ b/.changeset/agent-skills-token-based-length.md @@ -0,0 +1,30 @@ +--- +'@gtbuchanan/eslint-plugin-agent-skills': major +--- + +Gate skill length on tokens rather than lines + +`configs.recommended` now caps both `SKILL.md` and +`**/skills/*/references/**/*.md` with `agent-skills/max-tokens` at 5000, +and applies `agent-skills/max-lines` to neither. The rule still ships, so +a repo wanting the spec's 500-line figure can wire it explicitly: + +```js +{ + files: ['**/skills/*/SKILL.md'], + rules: { 'agent-skills/max-lines': ['warn', { max: 500 }] }, +} +``` + +Both rules proxy for how much context a skill costs to load, and tokens +measure it directly. At the ~10-14 tokens per line typical of prose +skills, 500 lines works out to 5200-7000 tokens, so the token cap is the +one that fires. The line cap leads only below ~10 tokens per line, where +it reports formatting — semantic line breaks and dense lists raise a line +count without changing what the agent loads. + +The 300-line cap on `references/` had no upstream basis: the spec's third +tier is "Resources (as needed)" and `skill-creator` calls bundled +resources "unlimited, loaded as needed". The 5000 there is a backstop for +a reference file costing more to load than the instructions tier it was +split out of. diff --git a/packages/eslint-config/e2e/skill.test.ts b/packages/eslint-config/e2e/skill.test.ts index 7633721d..43dda430 100644 --- a/packages/eslint-config/e2e/skill.test.ts +++ b/packages/eslint-config/e2e/skill.test.ts @@ -154,7 +154,31 @@ describe.concurrent('SKILL.md validation', () => { expect(result.stdout).toMatch(/not found.*references\/MISSING\.md/v); }); - it('caps file at 500 lines per the spec', async ({ fixture, expect }) => { + it('caps file at the spec token budget', async ({ fixture, expect }) => { + const name = skillName(); + /* Four UTF-8 bytes per estimated token, so this body lands one + token past the 5000 the spec recommends for the instructions + tier. One line, so a line count can't stand in for it. */ + const result = await fixture.run({ + files: { + [`skills/${name}/SKILL.md`]: skill( + [ + `name: ${name}`, + `description: ${faker.lorem.sentence()}`, + ], + ['# Example skill', '', 'word'.repeat(5001), ''], + ), + }, + }); + + expect(result.stdout).toMatch(/max-tokens/v); + expect(result.stdout).toMatch(/Maximum recommended is 5000/v); + }); + + /* The recommended config gates length on tokens alone. `max-lines` + still ships, but a repo has to opt into it, so a long file of cheap + lines lints clean. */ + it('does not cap file by line count', async ({ fixture, expect }) => { const name = skillName(); const lines = Array.from( { length: 600 }, @@ -167,12 +191,12 @@ describe.concurrent('SKILL.md validation', () => { `name: ${name}`, `description: ${faker.lorem.sentence()}`, ], - [...lines, ''], + ['# Example skill', '', ...lines, ''], ), }, }); - expect(result.stdout).toMatch(/max-lines/v); - expect(result.stdout).toMatch(/Maximum allowed is 500/v); + expect(result).toMatchObject({ exitCode: 0 }); + expect(result.stdout).not.toMatch(/max-lines/v); }); }); diff --git a/packages/eslint-config/src/plugins/agent-skills.ts b/packages/eslint-config/src/plugins/agent-skills.ts index b89c7c5b..a9b8dfa9 100644 --- a/packages/eslint-config/src/plugins/agent-skills.ts +++ b/packages/eslint-config/src/plugins/agent-skills.ts @@ -17,7 +17,7 @@ const toSources = ( /** * Agent Skills `SKILL.md` validation. Delegates the entire wiring - * (frontmatter schema rule, name-matches-dir rule, file-length cap) + * (frontmatter schema rule, name-matches-dir rule, token budget) * to the plugin's recommended flat-config block, then layers on the * frontmatter extensions of whichever hosts are selected. */ diff --git a/packages/eslint-plugin-agent-skills/README.md b/packages/eslint-plugin-agent-skills/README.md index d531ea5c..8901db46 100644 --- a/packages/eslint-plugin-agent-skills/README.md +++ b/packages/eslint-plugin-agent-skills/README.md @@ -19,17 +19,19 @@ constraints that pure schemas can't express. This package ships: spec's "one level deep" depth guidance. - `agent-skills/max-lines` — markdown-aware version of core's `max-lines` that fires under any markdown parser/language. - - `agent-skills/max-tokens` — the `SKILL.md` body must stay under - the spec's recommended instruction-tier token budget. + - `agent-skills/max-tokens` — a markdown body must stay under an + estimated token budget, defaulting to the spec's recommended + instruction-tier figure. - `agent-skills/min-evals` — each skill must ship at least N eval cases in `evals/evals.json`. - `agent-skills/evals-schema` — `evals/evals.json` matches the canonical schema, with sequential unique `id` values and a `skill_name` that matches the sibling `SKILL.md`. - A `configs.recommended` flat-config that wires the plugin's rules - for `**/skills/*/SKILL.md` (500 lines, 5000 estimated tokens), - `**/skills/*/references/**/*.md` (a tighter 300-line `max-lines` - cap, no token cap), and `**/skills/*/evals/evals.json`. + for `**/skills/*/SKILL.md` (5000 estimated tokens), + `**/skills/*/references/**/*.md` (a 5000-token backstop), and + `**/skills/*/evals/evals.json`. Length is gated on tokens alone; + `max-lines` ships but is opt-in. - A `defineSkillFrontmatterConfig` composer that overlays the frontmatter extensions of one or more agent hosts. @@ -234,17 +236,28 @@ Caps a markdown file at a maximum line count. Mirrors core ESLint's rule's `Program` visitor never runs against `@eslint/markdown`'s `root` mdast node). -The recommended config applies it twice, with different caps for -different file roles per the spec's -[Progressive disclosure](https://agentskills.io/specification#progressive-disclosure) -guidance: +**Not enabled by `configs.recommended`** — opt in explicitly if you +want it. It remains exported because the spec does state a line figure +("Keep your main `SKILL.md` under 500 lines."), so a repo that wants +that enforced literally can wire it: + +```js +{ + files: ['**/skills/*/SKILL.md'], + rules: { 'agent-skills/max-lines': ['warn', { max: 500 }] }, +} +``` + +The recommended config gates length on `max-tokens` alone. Both rules +proxy for how much context a skill costs to load, and tokens measure it +directly. At the ~10–14 tokens per line typical of prose skills, 500 +lines works out to 5200–7000 tokens, so `max-tokens` is the one that +fires. The line cap leads only below ~10 tokens per line — semantic line +breaks, one-item-per-line lists, reflowed tables — and there it reports +formatting, since none of those change what the agent loads. -- `**/skills/*/SKILL.md` — 500 lines ("Keep your main `SKILL.md` under - 500 lines."). -- `**/skills/*/references/**/*.md` — 300 lines, since the spec calls - for ancillary reference files to be focused and smaller than - `SKILL.md`. 300 sits just above the p90 line count observed across - popular published skills. +Note that this rule counts physical lines: there is no +`skipBlankLines` / `skipComments` equivalent to core's `max-lines`. Options: @@ -256,19 +269,19 @@ Caps the `SKILL.md` body at a maximum estimated token count, per the spec's [Progressive disclosure](https://agentskills.io/specification#progressive-disclosure) budget for the instructions tier ("**Instructions** (< 5000 tokens -recommended)"). The recommended config applies it to -`**/skills/*/SKILL.md` only — the spec names a token figure for that -tier alone, and gives none for `references/`. +recommended)"). + +The recommended config applies it to `**/skills/*/SKILL.md` and to +`**/skills/*/references/**/*.md`, both at 5000. On `SKILL.md` that is +the spec's own figure. On `references/` it is a backstop rather than a +spec limit — the spec caps no reference file, so the threshold worth +flagging is a reference file that costs more to load than the entire +instructions tier it was split out of. Frontmatter is excluded from the count. The spec accounts for `name` and `description` separately, as the ~100-token metadata tier loaded at startup, so a long description shouldn't eat the instruction budget. -This complements `max-lines` rather than duplicating it. The two catch -different shapes of the same problem: a table- or code-heavy skill can -sit well under 500 lines and still blow the token budget, and a skill -of many short lines can do the reverse. - Options: - `max` — maximum estimated token count. Defaults to `5000`. diff --git a/packages/eslint-plugin-agent-skills/src/index.ts b/packages/eslint-plugin-agent-skills/src/index.ts index e77d4479..11fa89b1 100644 --- a/packages/eslint-plugin-agent-skills/src/index.ts +++ b/packages/eslint-plugin-agent-skills/src/index.ts @@ -93,7 +93,15 @@ export const configs: { }, rules: { 'agent-skills/file-references': 'warn', - 'agent-skills/max-lines': ['warn', { max: 500 }], + /* + * `max-lines` is deliberately absent, though the spec does say + * "Keep your main `SKILL.md` under 500 lines". At the ~10-14 + * tokens/line typical of prose skills, 500 lines is 5200-7000 + * tokens, so `max-tokens` is the one that fires; the line cap + * leads only below ~10 tokens/line, where semantic line breaks + * or dense lists raise it without changing what the agent loads. + * The rule stays exported for repos wanting the spec's figure. + */ 'agent-skills/max-tokens': ['warn', { max: 5000 }], 'agent-skills/min-evals': 'warn', 'agent-skills/name-matches-dir': 'warn', @@ -105,7 +113,15 @@ export const configs: { language: 'markdown/commonmark', plugins: { 'agent-skills': plugin, markdown }, rules: { - 'agent-skills/max-lines': ['warn', { max: 300 }], + /* + * A backstop, not a spec limit. Nothing upstream caps a + * reference file: the spec's third tier is "Resources (as + * needed)" and `skill-creator` calls bundled resources + * "unlimited, loaded as needed". What is still worth flagging is + * one costing more to load than the instructions tier it was + * split out of — hence the same 5000 tokens. + */ + 'agent-skills/max-tokens': ['warn', { max: 5000 }], }, }, { diff --git a/packages/eslint-plugin-agent-skills/src/rules/max-tokens.ts b/packages/eslint-plugin-agent-skills/src/rules/max-tokens.ts index b4d3c7e7..f83647f1 100644 --- a/packages/eslint-plugin-agent-skills/src/rules/max-tokens.ts +++ b/packages/eslint-plugin-agent-skills/src/rules/max-tokens.ts @@ -83,9 +83,10 @@ const measureBody = ( * as the ~100-token metadata tier loaded at startup, so a long * description shouldn't eat the instruction budget. * - * Complements `max-lines` rather than duplicating it — a table- or - * code-heavy skill can sit well under 500 lines and still blow the - * token budget, and a skill of many short lines can do the reverse. + * Supersedes `max-lines` in the recommended config rather than + * complementing it: both proxy for context cost and this one measures + * it directly, so at typical prose density the line cap cannot fire + * first, and where it does move independently it reports formatting. * * The count is an estimate — see `bytesPerToken` for how it is derived * and where it stops holding. Every agent host tokenizes differently diff --git a/packages/eslint-plugin-agent-skills/test/max-lines.test.ts b/packages/eslint-plugin-agent-skills/test/max-lines.test.ts new file mode 100644 index 00000000..c5ae3f36 --- /dev/null +++ b/packages/eslint-plugin-agent-skills/test/max-lines.test.ts @@ -0,0 +1,99 @@ +import { RuleTester } from 'eslint'; +import { describe, it } from 'vitest'; +import { maxLines } from '#src/rules/max-lines.js'; +import * as parser from './parser.js'; + +const ruleTester = new RuleTester({ + languageOptions: { parser }, +}); + +const filename = '/repo/skills/my-skill/SKILL.md'; + +/* Numbered so a miscount shows up as the wrong number in the message + rather than as an off-by-one nobody can see. */ +const body = (lineCount: number): string => Array + .from({ length: lineCount }, (_unused, index) => `Line ${(index + 1).toString()}.`) + .join('\n'); + +describe.concurrent('agent-skills/max-lines', () => { + it('passes when the file is at the limit', ({ expect }) => { + expect(() => { + ruleTester.run('agent-skills/max-lines', maxLines, { + invalid: [], + valid: [{ code: body(500), filename }], + }); + }).not.toThrow(); + }); + + it('flags when the file exceeds the limit', ({ expect }) => { + expect(() => { + ruleTester.run('agent-skills/max-lines', maxLines, { + invalid: [ + { + code: body(501), + errors: [{ message: /501.*Maximum allowed is 500/v }], + filename, + }, + ], + valid: [], + }); + }).not.toThrow(); + }); + + it('honors an explicit max option', ({ expect }) => { + expect(() => { + ruleTester.run('agent-skills/max-lines', maxLines, { + invalid: [ + { + code: body(11), + errors: [{ message: /11.*Maximum allowed is 10/v }], + filename, + options: [{ max: 10 }], + }, + ], + valid: [{ code: body(10), filename, options: [{ max: 10 }] }], + }); + }).not.toThrow(); + }); + + /* Unlike core's `max-lines`, this rule takes the file's physical line + count — there is no `skipBlankLines`/`skipComments` equivalent, so + structural lines are counted like any other. */ + it('counts blank lines', ({ expect }) => { + expect(() => { + ruleTester.run('agent-skills/max-lines', maxLines, { + invalid: [ + { + code: '\n'.repeat(10), + errors: [{ message: /11.*Maximum allowed is 10/v }], + filename, + options: [{ max: 10 }], + }, + ], + valid: [], + }); + }).not.toThrow(); + }); + + it('reports from the first line past the limit to end of file', ({ expect }) => { + expect(() => { + ruleTester.run('agent-skills/max-lines', maxLines, { + invalid: [ + { + code: 'a\nb\nc', + errors: [{ + column: 1, + endColumn: 2, + endLine: 3, + line: 3, + messageId: 'tooLong', + }], + filename, + options: [{ max: 2 }], + }, + ], + valid: [], + }); + }).not.toThrow(); + }); +}); diff --git a/packages/eslint-plugin-agent-skills/test/recommended.test.ts b/packages/eslint-plugin-agent-skills/test/recommended.test.ts index 547dc5be..632d5bf6 100644 --- a/packages/eslint-plugin-agent-skills/test/recommended.test.ts +++ b/packages/eslint-plugin-agent-skills/test/recommended.test.ts @@ -16,6 +16,10 @@ const buildBody = (lineCount: number): string => Array .from({ length: lineCount }, (_unused, index) => `Line ${(index + 1).toString()}.`) .join('\n'); +/* One estimated token per 4 characters. Kept on a single line so a line + count can never stand in for the token count being asserted. */ +const buildTokens = (tokenCount: number): string => 'word'.repeat(tokenCount); + const lint = ( code: string, filename = referencesFile, @@ -33,37 +37,37 @@ const maxTokensMessages = ( const frontmatter = '---\nname: my-skill\ndescription: ok.\n---\n'; -/* One estimated token per 4 characters. Kept on a single line so the - 500-line `max-lines` cap can't fire alongside the token cap. */ -const buildTokens = (tokenCount: number): string => 'word'.repeat(tokenCount); - -describe('configs.recommended references/ max-lines', () => { - it('passes when the file is at the 300-line limit', ({ expect }) => { - const messages = maxLinesMessages(lint(buildBody(300))); +describe('configs.recommended references/ max-tokens', () => { + it('passes when the file is at the 5000-token limit', ({ expect }) => { + const messages = maxTokensMessages(lint(buildTokens(5000))); expect(messages).toStrictEqual([]); }); - it('flags when the file exceeds the 300-line limit', ({ expect }) => { - const [message, ...rest] = maxLinesMessages(lint(buildBody(301))); + it('flags when the file exceeds the 5000-token limit', ({ expect }) => { + const [message, ...rest] = maxTokensMessages( + lint(`${buildTokens(5000)}over`), + ); expect(rest).toStrictEqual([]); - expect(message?.message).toMatch(/301.*Maximum allowed is 300/v); + expect(message?.message).toMatch(/~5001 tokens.*recommended is 5000/v); expect(message?.severity).toBe(1); }); it('honors a top-of-file eslint-disable HTML comment', ({ expect }) => { const code = - '\n' + - buildBody(500); + '\n' + + buildTokens(6000); - const messages = maxLinesMessages(lint(code)); + const messages = maxTokensMessages(lint(code)); expect(messages).toStrictEqual([]); }); - it('does not apply the 300-line cap to SKILL.md', ({ expect }) => { - const messages = maxLinesMessages(lint(buildBody(400), skillFile)); + /* The spec caps the instructions tier but leaves resources "as needed", + so a reference file is bounded by context cost alone. */ + it('does not cap references/ by line count', ({ expect }) => { + const messages = maxLinesMessages(lint(buildBody(400))); expect(messages).toStrictEqual([]); }); @@ -88,8 +92,14 @@ describe('configs.recommended SKILL.md max-tokens', () => { expect(message?.severity).toBe(1); }); - it('does not apply a token cap to references/', ({ expect }) => { - const messages = maxTokensMessages(lint(buildTokens(6000))); + /* `max-lines` stays exported for repos that want the spec's 500-line + figure enforced literally, but the recommended config gates on + tokens alone — see the rule's own docs for why the two aren't the + complementary pair they look like. */ + it('does not cap SKILL.md by line count', ({ expect }) => { + const messages = maxLinesMessages( + lint(frontmatter + buildBody(600), skillFile), + ); expect(messages).toStrictEqual([]); });