From bef239ea17aa64e47d203586198c11c1d1235bb1 Mon Sep 17 00:00:00 2001 From: Taarik <147209483+taarikashenafi@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:41:54 -0500 Subject: [PATCH] Add advanced alt-text demo fixture Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 40132dd1-60d2-4441-ab8b-e8c5dc8e049e --- example/site-with-errors/README.md | 27 ++++++ .../advanced-alt-text-errors.html | 28 ++++++ .../assets/img/test-image.png | Bin 0 -> 1628 bytes package.json | 4 +- scripts/demo-support.ts | 80 +++++++++++++++ scripts/run-live-demo.ts | 52 ++++++++++ scripts/verify-demo.ts | 91 ++++++++++++++++++ tests/advanced-example.test.ts | 66 +++++++++++++ 8 files changed, 347 insertions(+), 1 deletion(-) create mode 100644 example/site-with-errors/advanced-alt-text-errors.html create mode 100644 example/site-with-errors/assets/img/test-image.png create mode 100644 scripts/demo-support.ts create mode 100644 scripts/run-live-demo.ts create mode 100644 scripts/verify-demo.ts create mode 100644 tests/advanced-example.test.ts diff --git a/example/site-with-errors/README.md b/example/site-with-errors/README.md index 0a1138a..7eb950a 100644 --- a/example/site-with-errors/README.md +++ b/example/site-with-errors/README.md @@ -7,6 +7,12 @@ A small Jekyll site that demos the and adds a [page whose images each intentionally trip one of the plugin's rules](alt-text-errors.html). +The site also includes an +[advanced model-backed demo page](advanced-alt-text-errors.html). Its three +synthetic cases demonstrate keyword stuffing, plausible but contextually wrong +alt text, and an accurate control. None of those cases trips a deterministic +rule; enable `alt-text-quality` to evaluate them. + Use it for: - **Manual testing** — build and serve the site, then point the scanner at it. @@ -47,6 +53,7 @@ TEST_USERNAME=demo TEST_PASSWORD=demo bundle exec rackup ``` The errors page is then available at `/alt-text-errors/`. +The advanced page is available at `/advanced-alt-text-errors/`. ## Scan it with the plugin @@ -63,3 +70,23 @@ The `example site-with-errors` test loads [`alt-text-errors.html`](alt-text-errors.html), runs the real `alt-text-scan` plugin against it, and asserts that every rule in the table above produces a finding. + +For a credential-free, machine-readable demo of both the baseline and the +model-backed finding contract: + +```sh +npm run demo:verify +``` + +The model verdicts in that command are explicitly marked as mocked. They prove +the production rule-to-finding mapping without making network calls. To run the +real GitHub Models judge against only the synthetic advanced page, set +`GITHUB_MODELS_TOKEN` to a PAT with `models:read` and run: + +```sh +npm run demo:live +``` + +If `AZURE_VISION_ENDPOINT` and `AZURE_VISION_KEY` are also set, the live command +automatically uses Azure Vision enrichment. Set `ALT_TEXT_JUDGE_MODE=copilot` +to force GitHub Models only. diff --git a/example/site-with-errors/advanced-alt-text-errors.html b/example/site-with-errors/advanced-alt-text-errors.html new file mode 100644 index 0000000..d6ccff1 --- /dev/null +++ b/example/site-with-errors/advanced-alt-text-errors.html @@ -0,0 +1,28 @@ +--- +layout: page +title: Advanced Alt Text Errors +permalink: /advanced-alt-text-errors/ +--- + +

Advanced alt text errors

+ +
+

Keyword stuffing

+

This synthetic blue test graphic is used to verify the scanner plugin.

+ accessibility testing, web accessibility, best accessibility scanner 2026, WCAG tools, buy accessibility software +
+ +
+

Plausible but contextually wrong

+

The image below is the same blue test graphic and represents a successful test.

+ A red warning triangle marks a failed deployment. +
+ +
+

Correct control

+

This final image shows the same graphic with an accurate alternative.

+ Blue square with the word test in white. +
diff --git a/example/site-with-errors/assets/img/test-image.png b/example/site-with-errors/assets/img/test-image.png new file mode 100644 index 0000000000000000000000000000000000000000..3fe42230f33b46517b5cd945925aab2ce930d5d4 GIT binary patch literal 1628 zcmeAS@N?(olHy`uVBq!ia0vp^6(G#P1|%(0%q{^b#^NA%Cx&(BWL^R}E~ycoX}-P; zT0k}j17mw80}DtA5K93u0|WB{Mh0de%?J`(zyz1ASip>6gA{T+3Vp}GznBR<@v@5|IZ}N`cMJH#uMGOoCSuj9WJaf;`%JBknotDiBWh%A$!m1 zH9W|YbBFQqrx^)MX73sbD}4+YrOOZOuu+@Q&}*F#BPZ;0;Fzp|UO%T=!ec&(>Bm_H zgLdPI53;onMa0kCxMant&zB2M-|}g>%BIN+VT-nJ&GOuvUA=*WJ%06} zyI&)ras^tx+)I}heYw&&*KIf1W7}{w|4Z`icZv14f7f_Fw=9wC`*nMLjb%^y{);o?{0}@me1H1l|7-W@ z^o0KOj6D9o>v7*|<$Wvc+tLeyOV#p9jUW4!Yx$`N{d3%~e<#!KzMjR;Q+T%C_D`&_ znpyGJPepVFb4;*IRKn)ZKR300{U5gH_1B*(E4P&OE~-ryl)9esOaJ}ve-kBh?ppTb zmG9lQX$G^+MvcbV_VQ5=p4>ezmd0L~lyPkTvUiqNjeC!)ZCI$*Wc^y{#tu97?C`dW zA44=>-C23>R1({U?-oig?$y~_Z$9wsjQ!>xKYi}p|M%fy(w>@dMmy_@gJ*X&-`KvD zsaobidfA+#*X?~Dbvp;jWUh9bIXhNA?$RDdQT^~$fm$gxX7BH7y{%fvqZ{A5==E_< zo0??3UD2xw9vl%_7bSUU*C(EOrDv+wo8O(CXTJMp+1A~?RwbVn?G96y&v;XIzuNmt z(x<39-y9a!JCw-Dg}?NRTCs6wZ0hswS?5l2{F=PHihbktgRfrnEjs7B?bRuxJDKKB z{CC+bSpQz~;P(A~mKP4J*e##8=HwC6)rW38Sa-7FsPUyYDVk^AifKKmHmNgS+El6h)o3V7`lm^ov;V(W1m^PbC6WZXG)ijSYB4OIG zAf{^`z88!cxzAjR@H!y1lCw%Jp)pd*n=?Y$)bTuvj)8ZwCS&#zwHGrRSkp3fREL}4 aU*hksQdk$g>=LNtXYh3Ob6Mw<&;$ULKIJR` literal 0 HcmV?d00001 diff --git a/package.json b/package.json index 96f4953..5184614 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,9 @@ "lint": "eslint .", "format": "prettier --write .", "format:check": "prettier --check .", - "grade": "tsx --env-file=.env scripts/grade-alt-text-quality.ts" + "grade": "tsx --env-file=.env scripts/grade-alt-text-quality.ts", + "demo:verify": "tsx scripts/verify-demo.ts", + "demo:live": "tsx scripts/run-live-demo.ts" }, "prettier": "@github/prettier-config", "engines": { diff --git a/scripts/demo-support.ts b/scripts/demo-support.ts new file mode 100644 index 0000000..0844e3c --- /dev/null +++ b/scripts/demo-support.ts @@ -0,0 +1,80 @@ +import {readFile} from 'node:fs/promises' +import {createServer, type Server} from 'node:http' +import {fileURLToPath} from 'node:url' +import {chromium, type Browser, type Page} from 'playwright' + +const fixtureRoot = fileURLToPath(new URL('../example/site-with-errors/', import.meta.url)) + +const routes = new Map([ + ['/baseline', {file: 'alt-text-errors.html', contentType: 'text/html; charset=utf-8'}], + ['/advanced', {file: 'advanced-alt-text-errors.html', contentType: 'text/html; charset=utf-8'}], + ['/assets/img/test-image.png', {file: 'assets/img/test-image.png', contentType: 'image/png'}], +]) + +function stripFrontMatter(contents: Buffer): Buffer { + const text = contents.toString('utf8') + return Buffer.from(text.replace(/^---\n[\s\S]*?\n---\n/, ''), 'utf8') +} + +async function startServer(): Promise<{server: Server; origin: string}> { + const server = createServer(async (request, response) => { + const route = routes.get(request.url ?? '') + if (!route) { + response.writeHead(404).end() + return + } + + try { + const rawContents = await readFile(new URL(route.file, `file://${fixtureRoot}/`)) + const contents = route.contentType.startsWith('text/html') ? stripFrontMatter(rawContents) : rawContents + response.writeHead(200, {'Content-Type': route.contentType}) + response.end(contents) + } catch (error) { + response.writeHead(500, {'Content-Type': 'text/plain; charset=utf-8'}) + response.end(error instanceof Error ? error.message : String(error)) + } + }) + + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', resolve) + }) + + const address = server.address() + if (!address || typeof address === 'string') throw new Error('Demo server did not bind to a TCP port.') + return {server, origin: `http://127.0.0.1:${address.port}`} +} + +export type DemoHarness = { + page: Page + open(route: 'baseline' | 'advanced'): Promise + close(): Promise +} + +export async function createDemoHarness(): Promise { + const {server, origin} = await startServer() + let browser: Browser + try { + browser = await chromium.launch() + } catch (error) { + server.close() + throw error + } + const page = await browser.newPage() + + return { + page, + async open(route) { + const url = `${origin}/${route}` + await page.goto(url) + return url + }, + async close() { + await page.close() + await browser.close() + await new Promise((resolve, reject) => { + server.close(error => (error ? reject(error) : resolve())) + }) + }, + } +} diff --git a/scripts/run-live-demo.ts b/scripts/run-live-demo.ts new file mode 100644 index 0000000..844f493 --- /dev/null +++ b/scripts/run-live-demo.ts @@ -0,0 +1,52 @@ +import {readFile} from 'node:fs/promises' +import {emitFindings} from '../src/findings.js' +import {extractImageContext} from '../src/extract-image-context.js' +import {__setJudge, altTextQuality} from '../src/rules/alt-text-quality.js' +import type {Finding} from '../src/types.js' +import {createDemoHarness} from './demo-support.js' + +if (!process.env['GITHUB_MODELS_TOKEN'] && !process.env['GITHUB_TOKEN']) { + throw new Error('Set GITHUB_MODELS_TOKEN to a PAT with models:read before running the live demo.') +} + +const azureConfigured = Boolean(process.env['AZURE_VISION_ENDPOINT'] && process.env['AZURE_VISION_KEY']) +const requestedMode = process.env['ALT_TEXT_JUDGE_MODE'] +const judgeMode = + requestedMode === 'copilot' || requestedMode === 'azure-augmented' + ? requestedMode + : azureConfigured + ? 'azure-augmented' + : 'copilot' + +const harness = await createDemoHarness() +try { + const url = await harness.open('advanced') + const images = await extractImageContext(harness.page) + __setJudge(null) + const results = await altTextQuality.evaluate({url, images}) + const findings: Finding[] = [] + await emitFindings(altTextQuality, results, url, async finding => { + findings.push(finding) + }) + + const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')) as { + version: string + } + console.log( + JSON.stringify( + { + pluginVersion: packageJson.version, + evidence: 'live GitHub Models evaluation of the repository synthetic fixture', + judgeMode, + azureCredentialsConfigured: azureConfigured, + fixtureUrl: url, + findings, + }, + null, + 2, + ), + ) +} finally { + __setJudge(null) + await harness.close() +} diff --git a/scripts/verify-demo.ts b/scripts/verify-demo.ts new file mode 100644 index 0000000..9e8c8e4 --- /dev/null +++ b/scripts/verify-demo.ts @@ -0,0 +1,91 @@ +import {readFile} from 'node:fs/promises' +import altTextScan from '../index.js' +import {emitFindings} from '../src/findings.js' +import {__setJudge, altTextQuality} from '../src/rules/alt-text-quality.js' +import {extractImageContext} from '../src/extract-image-context.js' +import type {JudgeAltText, JudgeInput, JudgeVerdict} from '../src/judges/types.js' +import type {Finding} from '../src/types.js' +import {createDemoHarness} from './demo-support.js' + +const keywordAlt = + 'accessibility testing, web accessibility, best accessibility scanner 2026, WCAG tools, buy accessibility software' +const wrongAlt = 'A red warning triangle marks a failed deployment.' + +class DemoJudge implements JudgeAltText { + async judge(input: JudgeInput): Promise { + if (input.alt === keywordAlt) { + return { + step: 4, + reasoning: + 'The alt is a search-keyword list with commercial language rather than a description of the blue test graphic.', + verdict: 'needs-fix', + issue: 'keyword-stuffing', + confidence: 1, + suggestion: 'Blue square with the word test in white.', + } + } + if (input.alt === wrongAlt) { + return { + step: 4, + reasoning: + 'The alt describes a red failure warning, but the image is a blue test graphic and the page identifies it as successful.', + verdict: 'needs-fix', + issue: 'inaccurate', + confidence: 1, + suggestion: 'Blue square with the word test in white.', + } + } + return { + step: 4, + reasoning: 'The alt accurately describes the synthetic graphic.', + verdict: 'ok', + issue: '', + confidence: 1, + suggestion: '', + } + } +} + +const harness = await createDemoHarness() +try { + const baselineUrl = await harness.open('baseline') + const deterministicFindings: Finding[] = [] + await altTextScan({ + page: harness.page, + addFinding: async finding => { + deterministicFindings.push(finding) + }, + }) + + const advancedUrl = await harness.open('advanced') + const images = await extractImageContext(harness.page) + __setJudge(new DemoJudge()) + const results = await altTextQuality.evaluate({url: advancedUrl, images}) + const mockedModelFindings: Finding[] = [] + await emitFindings(altTextQuality, results, advancedUrl, async finding => { + mockedModelFindings.push(finding) + }) + + const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8')) as { + version: string + } + console.log( + JSON.stringify( + { + pluginVersion: packageJson.version, + evidence: { + deterministic: 'real plugin scan; no credentials or model calls', + modelBacked: 'mocked judge; real extraction, rule mapping, and scanner Finding shape', + }, + fixtureUrls: {baseline: baselineUrl, advanced: advancedUrl}, + deterministicFindings, + mockedModelFindings, + }, + null, + 2, + ), + ) +} finally { + __setJudge(null) + await harness.close() +} diff --git a/tests/advanced-example.test.ts b/tests/advanced-example.test.ts new file mode 100644 index 0000000..822b044 --- /dev/null +++ b/tests/advanced-example.test.ts @@ -0,0 +1,66 @@ +import {afterAll, beforeAll, describe, expect, it} from 'vitest' +import {emitFindings} from '../src/findings.js' +import {extractImageContext} from '../src/extract-image-context.js' +import type {JudgeAltText, JudgeInput, JudgeVerdict} from '../src/judges/types.js' +import {__setJudge, altTextQuality} from '../src/rules/alt-text-quality.js' +import type {Finding} from '../src/types.js' +import {createDemoHarness, type DemoHarness} from '../scripts/demo-support.js' + +class FixtureJudge implements JudgeAltText { + async judge(input: JudgeInput): Promise { + if (input.alt.includes('best accessibility scanner')) { + return { + step: 4, + reasoning: 'Keyword list instead of an image description.', + verdict: 'needs-fix', + issue: 'keyword-stuffing', + confidence: 1, + suggestion: 'Blue square with the word test in white.', + } + } + if (input.alt.includes('red warning triangle')) { + return { + step: 4, + reasoning: 'The description does not match the image.', + verdict: 'needs-fix', + issue: 'inaccurate', + confidence: 1, + suggestion: 'Blue square with the word test in white.', + } + } + return {step: 4, reasoning: 'Accurate.', verdict: 'ok', issue: '', confidence: 1, suggestion: ''} + } +} + +let harness: DemoHarness + +beforeAll(async () => { + harness = await createDemoHarness() +}) + +afterAll(async () => { + __setJudge(null) + await harness.close() +}) + +describe('advanced example site', () => { + it('demonstrates model-only findings and generated remediation without credentials', async () => { + const url = await harness.open('advanced') + const images = await extractImageContext(harness.page) + expect(images).toHaveLength(3) + + __setJudge(new FixtureJudge()) + const results = await altTextQuality.evaluate({url, images}) + const findings: Finding[] = [] + await emitFindings(altTextQuality, results, url, async finding => { + findings.push(finding) + }) + + expect(findings).toHaveLength(2) + expect(findings.map(finding => finding.problemShort)).toEqual([ + expect.stringContaining('keyword-stuffed'), + expect.stringContaining('inaccurate'), + ]) + expect(findings.every(finding => finding.solutionShort.includes('Blue square with the word test'))).toBe(true) + }) +})