From f85beee113ee0160291e2e7535efc50f63f1e028 Mon Sep 17 00:00:00 2001 From: Taylor Buchanan Date: Tue, 1 Sep 2026 10:08:12 -0500 Subject: [PATCH 1/3] Cap references/ files by tokens instead of lines The recommended config capped references/ at 300 lines, which read as a hard limit but has no upstream basis. The spec's third progressive- disclosure tier is "Resources (as needed)" and its references/ guidance is qualitative; Anthropic's skill-creator calls bundled resources "unlimited, loaded as needed". The only numeric threshold anywhere near 300 is a table-of-contents trigger, not a cap. It also bounded the wrong quantity. What a reference file costs is the context it occupies once loaded, and a line count tracks that poorly -- semantic line breaks and dense lists both move it without changing what the agent reads. Swap in max-tokens at 5000, the same figure the spec recommends for an instructions tier, so the rule flags a reference file that costs more to load than the SKILL.md it was split out of. --- .../agent-skills-references-token-cap.md | 18 ++++++++ packages/eslint-plugin-agent-skills/README.md | 43 ++++++++++--------- .../eslint-plugin-agent-skills/src/index.ts | 21 ++++++++- .../test/recommended.test.ts | 40 ++++++++--------- 4 files changed, 79 insertions(+), 43 deletions(-) create mode 100644 .changeset/agent-skills-references-token-cap.md diff --git a/.changeset/agent-skills-references-token-cap.md b/.changeset/agent-skills-references-token-cap.md new file mode 100644 index 0000000..7eb0e73 --- /dev/null +++ b/.changeset/agent-skills-references-token-cap.md @@ -0,0 +1,18 @@ +--- +'@gtbuchanan/eslint-plugin-agent-skills': minor +--- + +Cap `references/` files by estimated tokens instead of lines + +`configs.recommended` now applies `agent-skills/max-tokens` to +`**/skills/*/references/**/*.md` at 5000 and no longer applies +`agent-skills/max-lines` there. + +Nothing upstream caps a reference file: the spec's third +progressive-disclosure tier is "Resources (as needed)" and its +`references/` guidance is qualitative, while Anthropic's `skill-creator` +calls bundled resources "unlimited, loaded as needed". The previous +300-line cap read as a hard limit derived from a table-of-contents +threshold, and it bounded the wrong quantity — the cost of a reference +file is the context it occupies when loaded, which a line count tracks +poorly. diff --git a/packages/eslint-plugin-agent-skills/README.md b/packages/eslint-plugin-agent-skills/README.md index d531ea5..8ce7970 100644 --- a/packages/eslint-plugin-agent-skills/README.md +++ b/packages/eslint-plugin-agent-skills/README.md @@ -19,8 +19,9 @@ constraints that pure schemas can't express. This package ships: spec's "one level deep" depth guidance. - `agent-skills/max-lines` — markdown-aware version of core's `max-lines` that fires under any markdown parser/language. - - `agent-skills/max-tokens` — the `SKILL.md` body must stay under - the spec's recommended instruction-tier token budget. + - `agent-skills/max-tokens` — a markdown body must stay under an + estimated token budget, defaulting to the spec's recommended + instruction-tier figure. - `agent-skills/min-evals` — each skill must ship at least N eval cases in `evals/evals.json`. - `agent-skills/evals-schema` — `evals/evals.json` matches the @@ -28,8 +29,8 @@ constraints that pure schemas can't express. This package ships: `skill_name` that matches the sibling `SKILL.md`. - A `configs.recommended` flat-config that wires the plugin's rules for `**/skills/*/SKILL.md` (500 lines, 5000 estimated tokens), - `**/skills/*/references/**/*.md` (a tighter 300-line `max-lines` - cap, no token cap), and `**/skills/*/evals/evals.json`. + `**/skills/*/references/**/*.md` (a 5000-token backstop, no line + cap), and `**/skills/*/evals/evals.json`. - A `defineSkillFrontmatterConfig` composer that overlays the frontmatter extensions of one or more agent hosts. @@ -234,17 +235,17 @@ Caps a markdown file at a maximum line count. Mirrors core ESLint's rule's `Program` visitor never runs against `@eslint/markdown`'s `root` mdast node). -The recommended config applies it twice, with different caps for -different file roles per the spec's +The recommended config applies it to `**/skills/*/SKILL.md` at 500 +lines, per the spec's [Progressive disclosure](https://agentskills.io/specification#progressive-disclosure) -guidance: +guidance ("Keep your main `SKILL.md` under 500 lines."). -- `**/skills/*/SKILL.md` — 500 lines ("Keep your main `SKILL.md` under - 500 lines."). -- `**/skills/*/references/**/*.md` — 300 lines, since the spec calls - for ancillary reference files to be focused and smaller than - `SKILL.md`. 300 sits just above the p90 line count observed across - popular published skills. +It is not applied to `**/skills/*/references/**/*.md`. Nothing upstream +caps a reference file — the spec's third tier is "Resources (as +needed)" and its `references/` guidance is qualitative, while +Anthropic's `skill-creator` calls bundled resources "unlimited, loaded +as needed". Those files are bounded by `max-tokens` instead, since the +cost worth flagging there is context, not length. Options: @@ -256,19 +257,19 @@ Caps the `SKILL.md` body at a maximum estimated token count, per the spec's [Progressive disclosure](https://agentskills.io/specification#progressive-disclosure) budget for the instructions tier ("**Instructions** (< 5000 tokens -recommended)"). The recommended config applies it to -`**/skills/*/SKILL.md` only — the spec names a token figure for that -tier alone, and gives none for `references/`. +recommended)"). + +The recommended config applies it to `**/skills/*/SKILL.md` and to +`**/skills/*/references/**/*.md`, both at 5000. On `SKILL.md` that is +the spec's own figure. On `references/` it is a backstop rather than a +spec limit — the spec caps no reference file, so the threshold worth +flagging is a reference file that costs more to load than the entire +instructions tier it was split out of. Frontmatter is excluded from the count. The spec accounts for `name` and `description` separately, as the ~100-token metadata tier loaded at startup, so a long description shouldn't eat the instruction budget. -This complements `max-lines` rather than duplicating it. The two catch -different shapes of the same problem: a table- or code-heavy skill can -sit well under 500 lines and still blow the token budget, and a skill -of many short lines can do the reverse. - Options: - `max` — maximum estimated token count. Defaults to `5000`. diff --git a/packages/eslint-plugin-agent-skills/src/index.ts b/packages/eslint-plugin-agent-skills/src/index.ts index e77d447..cdb0cb6 100644 --- a/packages/eslint-plugin-agent-skills/src/index.ts +++ b/packages/eslint-plugin-agent-skills/src/index.ts @@ -105,7 +105,26 @@ export const configs: { language: 'markdown/commonmark', plugins: { 'agent-skills': plugin, markdown }, rules: { - 'agent-skills/max-lines': ['warn', { max: 300 }], + /* + * A backstop, not a spec limit. The spec's third progressive- + * disclosure tier is "Resources (as needed)" and its `references/` + * guidance is qualitative ("keep individual reference files + * focused"); Anthropic's own `skill-creator` calls bundled + * resources "unlimited, loaded as needed". So nothing upstream + * caps a reference file, and capping one tightly would work + * against the point of moving material out of `SKILL.md`. + * + * What remains worth flagging is a reference file that costs more + * to load than the entire instructions tier it was split out of, + * which is where the limit comes from: the same 5000 tokens the + * spec recommends for a `SKILL.md` body. + * + * Tokens rather than lines because the cost being bounded is + * context, and a line count is a poor proxy for it — semantic + * line breaks and dense lists both move it without changing what + * the agent loads. + */ + 'agent-skills/max-tokens': ['warn', { max: 5000 }], }, }, { diff --git a/packages/eslint-plugin-agent-skills/test/recommended.test.ts b/packages/eslint-plugin-agent-skills/test/recommended.test.ts index 547dc5b..8288ada 100644 --- a/packages/eslint-plugin-agent-skills/test/recommended.test.ts +++ b/packages/eslint-plugin-agent-skills/test/recommended.test.ts @@ -16,6 +16,10 @@ const buildBody = (lineCount: number): string => Array .from({ length: lineCount }, (_unused, index) => `Line ${(index + 1).toString()}.`) .join('\n'); +/* One estimated token per 4 characters. Kept on a single line so a line + count can never stand in for the token count being asserted. */ +const buildTokens = (tokenCount: number): string => 'word'.repeat(tokenCount); + const lint = ( code: string, filename = referencesFile, @@ -33,37 +37,37 @@ const maxTokensMessages = ( const frontmatter = '---\nname: my-skill\ndescription: ok.\n---\n'; -/* One estimated token per 4 characters. Kept on a single line so the - 500-line `max-lines` cap can't fire alongside the token cap. */ -const buildTokens = (tokenCount: number): string => 'word'.repeat(tokenCount); - -describe('configs.recommended references/ max-lines', () => { - it('passes when the file is at the 300-line limit', ({ expect }) => { - const messages = maxLinesMessages(lint(buildBody(300))); +describe('configs.recommended references/ max-tokens', () => { + it('passes when the file is at the 5000-token limit', ({ expect }) => { + const messages = maxTokensMessages(lint(buildTokens(5000))); expect(messages).toStrictEqual([]); }); - it('flags when the file exceeds the 300-line limit', ({ expect }) => { - const [message, ...rest] = maxLinesMessages(lint(buildBody(301))); + it('flags when the file exceeds the 5000-token limit', ({ expect }) => { + const [message, ...rest] = maxTokensMessages( + lint(`${buildTokens(5000)}over`), + ); expect(rest).toStrictEqual([]); - expect(message?.message).toMatch(/301.*Maximum allowed is 300/v); + expect(message?.message).toMatch(/~5001 tokens.*recommended is 5000/v); expect(message?.severity).toBe(1); }); it('honors a top-of-file eslint-disable HTML comment', ({ expect }) => { const code = - '\n' + - buildBody(500); + '\n' + + buildTokens(6000); - const messages = maxLinesMessages(lint(code)); + const messages = maxTokensMessages(lint(code)); expect(messages).toStrictEqual([]); }); - it('does not apply the 300-line cap to SKILL.md', ({ expect }) => { - const messages = maxLinesMessages(lint(buildBody(400), skillFile)); + /* The spec caps the instructions tier but leaves resources "as needed", + so a reference file is bounded by context cost alone. */ + it('does not cap references/ by line count', ({ expect }) => { + const messages = maxLinesMessages(lint(buildBody(400))); expect(messages).toStrictEqual([]); }); @@ -87,10 +91,4 @@ describe('configs.recommended SKILL.md max-tokens', () => { expect(message?.message).toMatch(/~5001 tokens.*recommended is 5000/v); expect(message?.severity).toBe(1); }); - - it('does not apply a token cap to references/', ({ expect }) => { - const messages = maxTokensMessages(lint(buildTokens(6000))); - - expect(messages).toStrictEqual([]); - }); }); From 8651b4d33faa8c4549650d0067a42c20ef5f69f2 Mon Sep 17 00:00:00 2001 From: Taylor Buchanan Date: Tue, 1 Sep 2026 10:19:34 -0500 Subject: [PATCH 2/3] Drop max-lines from the recommended config Length is now gated on max-tokens alone. Both rules proxy for how much context a skill costs to load, and tokens measure that directly while lines only correlate. At the ~10-14 tokens per line typical of prose skills, a 500-line cap does not bind until well past 5000 tokens, so max-lines is dominated -- it cannot fire before max-tokens does. Where the correlation breaks, it breaks against the line cap. Semantic line breaks, one-item-per-line lists, and reflowed tables all raise the line count without changing what the agent loads, so the rule reports formatting rather than cost. Keep the rule exported and documented: the spec does state a line figure, so a repo wanting it enforced literally can still wire it. It loses its coverage from the recommended config, so give it a direct unit test. --- .../agent-skills-references-token-cap.md | 18 ---- .changeset/agent-skills-token-based-length.md | 31 ++++++ packages/eslint-config/e2e/skill.test.ts | 32 +++++- .../eslint-config/src/plugins/agent-skills.ts | 2 +- packages/eslint-plugin-agent-skills/README.md | 40 +++++--- .../eslint-plugin-agent-skills/src/index.ts | 32 +++--- .../src/rules/max-tokens.ts | 7 +- .../test/max-lines.test.ts | 99 +++++++++++++++++++ .../test/recommended.test.ts | 12 +++ 9 files changed, 215 insertions(+), 58 deletions(-) delete mode 100644 .changeset/agent-skills-references-token-cap.md create mode 100644 .changeset/agent-skills-token-based-length.md create mode 100644 packages/eslint-plugin-agent-skills/test/max-lines.test.ts diff --git a/.changeset/agent-skills-references-token-cap.md b/.changeset/agent-skills-references-token-cap.md deleted file mode 100644 index 7eb0e73..0000000 --- a/.changeset/agent-skills-references-token-cap.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -'@gtbuchanan/eslint-plugin-agent-skills': minor ---- - -Cap `references/` files by estimated tokens instead of lines - -`configs.recommended` now applies `agent-skills/max-tokens` to -`**/skills/*/references/**/*.md` at 5000 and no longer applies -`agent-skills/max-lines` there. - -Nothing upstream caps a reference file: the spec's third -progressive-disclosure tier is "Resources (as needed)" and its -`references/` guidance is qualitative, while Anthropic's `skill-creator` -calls bundled resources "unlimited, loaded as needed". The previous -300-line cap read as a hard limit derived from a table-of-contents -threshold, and it bounded the wrong quantity — the cost of a reference -file is the context it occupies when loaded, which a line count tracks -poorly. diff --git a/.changeset/agent-skills-token-based-length.md b/.changeset/agent-skills-token-based-length.md new file mode 100644 index 0000000..8ddbb43 --- /dev/null +++ b/.changeset/agent-skills-token-based-length.md @@ -0,0 +1,31 @@ +--- +'@gtbuchanan/eslint-plugin-agent-skills': major +--- + +Gate skill length on tokens rather than lines + +`configs.recommended` now caps both `SKILL.md` and +`**/skills/*/references/**/*.md` with `agent-skills/max-tokens` at 5000, +and applies `agent-skills/max-lines` to neither. The rule still ships, so +a repo wanting the spec's 500-line figure can wire it explicitly: + +```js +{ + files: ['**/skills/*/SKILL.md'], + rules: { 'agent-skills/max-lines': ['warn', { max: 500 }] }, +} +``` + +Both rules proxy for how much context a skill costs to load, and tokens +measure it directly. At the ~10-14 tokens per line typical of prose +skills, 500 lines does not bind until well past 5000 tokens, so the line +cap cannot fire first — and where a line count does move independently +(semantic line breaks, one-item-per-line lists, reflowed tables) it moves +without changing what the agent loads. + +The 300-line cap on `references/` had no upstream basis. The spec's third +progressive-disclosure tier is "Resources (as needed)" and `skill-creator` +calls bundled resources "unlimited, loaded as needed"; the only nearby +number is a table-of-contents threshold, not a cap. The 5000 there is a +backstop for a reference file costing more to load than the instructions +tier it was split out of. diff --git a/packages/eslint-config/e2e/skill.test.ts b/packages/eslint-config/e2e/skill.test.ts index 7633721..43dda43 100644 --- a/packages/eslint-config/e2e/skill.test.ts +++ b/packages/eslint-config/e2e/skill.test.ts @@ -154,7 +154,31 @@ describe.concurrent('SKILL.md validation', () => { expect(result.stdout).toMatch(/not found.*references\/MISSING\.md/v); }); - it('caps file at 500 lines per the spec', async ({ fixture, expect }) => { + it('caps file at the spec token budget', async ({ fixture, expect }) => { + const name = skillName(); + /* Four UTF-8 bytes per estimated token, so this body lands one + token past the 5000 the spec recommends for the instructions + tier. One line, so a line count can't stand in for it. */ + const result = await fixture.run({ + files: { + [`skills/${name}/SKILL.md`]: skill( + [ + `name: ${name}`, + `description: ${faker.lorem.sentence()}`, + ], + ['# Example skill', '', 'word'.repeat(5001), ''], + ), + }, + }); + + expect(result.stdout).toMatch(/max-tokens/v); + expect(result.stdout).toMatch(/Maximum recommended is 5000/v); + }); + + /* The recommended config gates length on tokens alone. `max-lines` + still ships, but a repo has to opt into it, so a long file of cheap + lines lints clean. */ + it('does not cap file by line count', async ({ fixture, expect }) => { const name = skillName(); const lines = Array.from( { length: 600 }, @@ -167,12 +191,12 @@ describe.concurrent('SKILL.md validation', () => { `name: ${name}`, `description: ${faker.lorem.sentence()}`, ], - [...lines, ''], + ['# Example skill', '', ...lines, ''], ), }, }); - expect(result.stdout).toMatch(/max-lines/v); - expect(result.stdout).toMatch(/Maximum allowed is 500/v); + expect(result).toMatchObject({ exitCode: 0 }); + expect(result.stdout).not.toMatch(/max-lines/v); }); }); diff --git a/packages/eslint-config/src/plugins/agent-skills.ts b/packages/eslint-config/src/plugins/agent-skills.ts index b89c7c5..a9b8dfa 100644 --- a/packages/eslint-config/src/plugins/agent-skills.ts +++ b/packages/eslint-config/src/plugins/agent-skills.ts @@ -17,7 +17,7 @@ const toSources = ( /** * Agent Skills `SKILL.md` validation. Delegates the entire wiring - * (frontmatter schema rule, name-matches-dir rule, file-length cap) + * (frontmatter schema rule, name-matches-dir rule, token budget) * to the plugin's recommended flat-config block, then layers on the * frontmatter extensions of whichever hosts are selected. */ diff --git a/packages/eslint-plugin-agent-skills/README.md b/packages/eslint-plugin-agent-skills/README.md index 8ce7970..d3e20ea 100644 --- a/packages/eslint-plugin-agent-skills/README.md +++ b/packages/eslint-plugin-agent-skills/README.md @@ -28,9 +28,10 @@ constraints that pure schemas can't express. This package ships: canonical schema, with sequential unique `id` values and a `skill_name` that matches the sibling `SKILL.md`. - A `configs.recommended` flat-config that wires the plugin's rules - for `**/skills/*/SKILL.md` (500 lines, 5000 estimated tokens), - `**/skills/*/references/**/*.md` (a 5000-token backstop, no line - cap), and `**/skills/*/evals/evals.json`. + for `**/skills/*/SKILL.md` (5000 estimated tokens), + `**/skills/*/references/**/*.md` (a 5000-token backstop), and + `**/skills/*/evals/evals.json`. Length is gated on tokens alone; + `max-lines` ships but is opt-in. - A `defineSkillFrontmatterConfig` composer that overlays the frontmatter extensions of one or more agent hosts. @@ -235,17 +236,28 @@ Caps a markdown file at a maximum line count. Mirrors core ESLint's rule's `Program` visitor never runs against `@eslint/markdown`'s `root` mdast node). -The recommended config applies it to `**/skills/*/SKILL.md` at 500 -lines, per the spec's -[Progressive disclosure](https://agentskills.io/specification#progressive-disclosure) -guidance ("Keep your main `SKILL.md` under 500 lines."). - -It is not applied to `**/skills/*/references/**/*.md`. Nothing upstream -caps a reference file — the spec's third tier is "Resources (as -needed)" and its `references/` guidance is qualitative, while -Anthropic's `skill-creator` calls bundled resources "unlimited, loaded -as needed". Those files are bounded by `max-tokens` instead, since the -cost worth flagging there is context, not length. +**Not enabled by `configs.recommended`** — opt in explicitly if you +want it. It remains exported because the spec does state a line figure +("Keep your main `SKILL.md` under 500 lines."), so a repo that wants +that enforced literally can wire it: + +```js +{ + files: ['**/skills/*/SKILL.md'], + rules: { 'agent-skills/max-lines': ['warn', { max: 500 }] }, +} +``` + +The recommended config gates length on `max-tokens` alone. Both rules +proxy for how much context a skill costs to load, and tokens measure it +directly. At the ~10–14 tokens per line typical of prose skills, 500 +lines doesn't bind until well past 5000 tokens, so `max-lines` cannot +fire first — and where a line count does move independently (semantic +line breaks, one-item-per-line lists, reflowed tables) it moves without +changing what the agent loads. + +Note that this rule counts physical lines: there is no +`skipBlankLines` / `skipComments` equivalent to core's `max-lines`. Options: diff --git a/packages/eslint-plugin-agent-skills/src/index.ts b/packages/eslint-plugin-agent-skills/src/index.ts index cdb0cb6..729bd69 100644 --- a/packages/eslint-plugin-agent-skills/src/index.ts +++ b/packages/eslint-plugin-agent-skills/src/index.ts @@ -93,7 +93,14 @@ export const configs: { }, rules: { 'agent-skills/file-references': 'warn', - 'agent-skills/max-lines': ['warn', { max: 500 }], + /* + * `max-lines` is deliberately absent, though the spec does say + * "Keep your main `SKILL.md` under 500 lines". At the ~10-14 + * tokens/line typical of prose skills it cannot fire before + * `max-tokens` does, and semantic line breaks or dense lists + * raise it without changing what the agent loads. The rule + * stays exported for repos wanting the spec's figure literally. + */ 'agent-skills/max-tokens': ['warn', { max: 5000 }], 'agent-skills/min-evals': 'warn', 'agent-skills/name-matches-dir': 'warn', @@ -106,23 +113,12 @@ export const configs: { plugins: { 'agent-skills': plugin, markdown }, rules: { /* - * A backstop, not a spec limit. The spec's third progressive- - * disclosure tier is "Resources (as needed)" and its `references/` - * guidance is qualitative ("keep individual reference files - * focused"); Anthropic's own `skill-creator` calls bundled - * resources "unlimited, loaded as needed". So nothing upstream - * caps a reference file, and capping one tightly would work - * against the point of moving material out of `SKILL.md`. - * - * What remains worth flagging is a reference file that costs more - * to load than the entire instructions tier it was split out of, - * which is where the limit comes from: the same 5000 tokens the - * spec recommends for a `SKILL.md` body. - * - * Tokens rather than lines because the cost being bounded is - * context, and a line count is a poor proxy for it — semantic - * line breaks and dense lists both move it without changing what - * the agent loads. + * A backstop, not a spec limit. Nothing upstream caps a + * reference file: the spec's third tier is "Resources (as + * needed)" and `skill-creator` calls bundled resources + * "unlimited, loaded as needed". What is still worth flagging is + * one costing more to load than the instructions tier it was + * split out of — hence the same 5000 tokens. */ 'agent-skills/max-tokens': ['warn', { max: 5000 }], }, diff --git a/packages/eslint-plugin-agent-skills/src/rules/max-tokens.ts b/packages/eslint-plugin-agent-skills/src/rules/max-tokens.ts index b4d3c7e..f83647f 100644 --- a/packages/eslint-plugin-agent-skills/src/rules/max-tokens.ts +++ b/packages/eslint-plugin-agent-skills/src/rules/max-tokens.ts @@ -83,9 +83,10 @@ const measureBody = ( * as the ~100-token metadata tier loaded at startup, so a long * description shouldn't eat the instruction budget. * - * Complements `max-lines` rather than duplicating it — a table- or - * code-heavy skill can sit well under 500 lines and still blow the - * token budget, and a skill of many short lines can do the reverse. + * Supersedes `max-lines` in the recommended config rather than + * complementing it: both proxy for context cost and this one measures + * it directly, so at typical prose density the line cap cannot fire + * first, and where it does move independently it reports formatting. * * The count is an estimate — see `bytesPerToken` for how it is derived * and where it stops holding. Every agent host tokenizes differently diff --git a/packages/eslint-plugin-agent-skills/test/max-lines.test.ts b/packages/eslint-plugin-agent-skills/test/max-lines.test.ts new file mode 100644 index 0000000..c5ae3f3 --- /dev/null +++ b/packages/eslint-plugin-agent-skills/test/max-lines.test.ts @@ -0,0 +1,99 @@ +import { RuleTester } from 'eslint'; +import { describe, it } from 'vitest'; +import { maxLines } from '#src/rules/max-lines.js'; +import * as parser from './parser.js'; + +const ruleTester = new RuleTester({ + languageOptions: { parser }, +}); + +const filename = '/repo/skills/my-skill/SKILL.md'; + +/* Numbered so a miscount shows up as the wrong number in the message + rather than as an off-by-one nobody can see. */ +const body = (lineCount: number): string => Array + .from({ length: lineCount }, (_unused, index) => `Line ${(index + 1).toString()}.`) + .join('\n'); + +describe.concurrent('agent-skills/max-lines', () => { + it('passes when the file is at the limit', ({ expect }) => { + expect(() => { + ruleTester.run('agent-skills/max-lines', maxLines, { + invalid: [], + valid: [{ code: body(500), filename }], + }); + }).not.toThrow(); + }); + + it('flags when the file exceeds the limit', ({ expect }) => { + expect(() => { + ruleTester.run('agent-skills/max-lines', maxLines, { + invalid: [ + { + code: body(501), + errors: [{ message: /501.*Maximum allowed is 500/v }], + filename, + }, + ], + valid: [], + }); + }).not.toThrow(); + }); + + it('honors an explicit max option', ({ expect }) => { + expect(() => { + ruleTester.run('agent-skills/max-lines', maxLines, { + invalid: [ + { + code: body(11), + errors: [{ message: /11.*Maximum allowed is 10/v }], + filename, + options: [{ max: 10 }], + }, + ], + valid: [{ code: body(10), filename, options: [{ max: 10 }] }], + }); + }).not.toThrow(); + }); + + /* Unlike core's `max-lines`, this rule takes the file's physical line + count — there is no `skipBlankLines`/`skipComments` equivalent, so + structural lines are counted like any other. */ + it('counts blank lines', ({ expect }) => { + expect(() => { + ruleTester.run('agent-skills/max-lines', maxLines, { + invalid: [ + { + code: '\n'.repeat(10), + errors: [{ message: /11.*Maximum allowed is 10/v }], + filename, + options: [{ max: 10 }], + }, + ], + valid: [], + }); + }).not.toThrow(); + }); + + it('reports from the first line past the limit to end of file', ({ expect }) => { + expect(() => { + ruleTester.run('agent-skills/max-lines', maxLines, { + invalid: [ + { + code: 'a\nb\nc', + errors: [{ + column: 1, + endColumn: 2, + endLine: 3, + line: 3, + messageId: 'tooLong', + }], + filename, + options: [{ max: 2 }], + }, + ], + valid: [], + }); + }).not.toThrow(); + }); +}); diff --git a/packages/eslint-plugin-agent-skills/test/recommended.test.ts b/packages/eslint-plugin-agent-skills/test/recommended.test.ts index 8288ada..632d5bf 100644 --- a/packages/eslint-plugin-agent-skills/test/recommended.test.ts +++ b/packages/eslint-plugin-agent-skills/test/recommended.test.ts @@ -91,4 +91,16 @@ describe('configs.recommended SKILL.md max-tokens', () => { expect(message?.message).toMatch(/~5001 tokens.*recommended is 5000/v); expect(message?.severity).toBe(1); }); + + /* `max-lines` stays exported for repos that want the spec's 500-line + figure enforced literally, but the recommended config gates on + tokens alone — see the rule's own docs for why the two aren't the + complementary pair they look like. */ + it('does not cap SKILL.md by line count', ({ expect }) => { + const messages = maxLinesMessages( + lint(frontmatter + buildBody(600), skillFile), + ); + + expect(messages).toStrictEqual([]); + }); }); From dff14285ab2773893cfd1d69fd7779402722bde9 Mon Sep 17 00:00:00 2001 From: Taylor Buchanan Date: Tue, 1 Sep 2026 14:59:11 -0500 Subject: [PATCH 3/3] Qualify the max-lines ordering claim The docs said max-lines "cannot fire first", which overstates it. At exactly 10 tokens per line 500 lines is 5000 tokens, and the measured floor across real skills was 10.45, so the line cap trails by 4% there rather than by a wide margin. The claim was also unconditional where the mechanism is not: below ~10 tokens per line the line cap does lead. That case is the whole reason for the change, so asserting it away argued against the point. --- .changeset/agent-skills-token-based-length.md | 19 +++++++++---------- packages/eslint-plugin-agent-skills/README.md | 8 ++++---- .../eslint-plugin-agent-skills/src/index.ts | 9 +++++---- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.changeset/agent-skills-token-based-length.md b/.changeset/agent-skills-token-based-length.md index 8ddbb43..468f2f4 100644 --- a/.changeset/agent-skills-token-based-length.md +++ b/.changeset/agent-skills-token-based-length.md @@ -18,14 +18,13 @@ a repo wanting the spec's 500-line figure can wire it explicitly: Both rules proxy for how much context a skill costs to load, and tokens measure it directly. At the ~10-14 tokens per line typical of prose -skills, 500 lines does not bind until well past 5000 tokens, so the line -cap cannot fire first — and where a line count does move independently -(semantic line breaks, one-item-per-line lists, reflowed tables) it moves -without changing what the agent loads. +skills, 500 lines works out to 5200-7000 tokens, so the token cap is the +one that fires. The line cap leads only below ~10 tokens per line, where +it reports formatting — semantic line breaks and dense lists raise a line +count without changing what the agent loads. -The 300-line cap on `references/` had no upstream basis. The spec's third -progressive-disclosure tier is "Resources (as needed)" and `skill-creator` -calls bundled resources "unlimited, loaded as needed"; the only nearby -number is a table-of-contents threshold, not a cap. The 5000 there is a -backstop for a reference file costing more to load than the instructions -tier it was split out of. +The 300-line cap on `references/` had no upstream basis: the spec's third +tier is "Resources (as needed)" and `skill-creator` calls bundled +resources "unlimited, loaded as needed". The 5000 there is a backstop for +a reference file costing more to load than the instructions tier it was +split out of. diff --git a/packages/eslint-plugin-agent-skills/README.md b/packages/eslint-plugin-agent-skills/README.md index d3e20ea..8901db4 100644 --- a/packages/eslint-plugin-agent-skills/README.md +++ b/packages/eslint-plugin-agent-skills/README.md @@ -251,10 +251,10 @@ that enforced literally can wire it: The recommended config gates length on `max-tokens` alone. Both rules proxy for how much context a skill costs to load, and tokens measure it directly. At the ~10–14 tokens per line typical of prose skills, 500 -lines doesn't bind until well past 5000 tokens, so `max-lines` cannot -fire first — and where a line count does move independently (semantic -line breaks, one-item-per-line lists, reflowed tables) it moves without -changing what the agent loads. +lines works out to 5200–7000 tokens, so `max-tokens` is the one that +fires. The line cap leads only below ~10 tokens per line — semantic line +breaks, one-item-per-line lists, reflowed tables — and there it reports +formatting, since none of those change what the agent loads. Note that this rule counts physical lines: there is no `skipBlankLines` / `skipComments` equivalent to core's `max-lines`. diff --git a/packages/eslint-plugin-agent-skills/src/index.ts b/packages/eslint-plugin-agent-skills/src/index.ts index 729bd69..11fa89b 100644 --- a/packages/eslint-plugin-agent-skills/src/index.ts +++ b/packages/eslint-plugin-agent-skills/src/index.ts @@ -96,10 +96,11 @@ export const configs: { /* * `max-lines` is deliberately absent, though the spec does say * "Keep your main `SKILL.md` under 500 lines". At the ~10-14 - * tokens/line typical of prose skills it cannot fire before - * `max-tokens` does, and semantic line breaks or dense lists - * raise it without changing what the agent loads. The rule - * stays exported for repos wanting the spec's figure literally. + * tokens/line typical of prose skills, 500 lines is 5200-7000 + * tokens, so `max-tokens` is the one that fires; the line cap + * leads only below ~10 tokens/line, where semantic line breaks + * or dense lists raise it without changing what the agent loads. + * The rule stays exported for repos wanting the spec's figure. */ 'agent-skills/max-tokens': ['warn', { max: 5000 }], 'agent-skills/min-evals': 'warn',