From 99c3fb82f4d2f60b64832e41d4a969e86a714c64 Mon Sep 17 00:00:00 2001 From: Imran Siddique Date: Mon, 14 Sep 2026 16:40:00 -0700 Subject: [PATCH 1/2] docs: page descriptions, titles, structured data and crawler files for search and answer engines A crawl of tests.agentrust-io.com on 2026-09-14 found 21 of 22 pages sharing the site description, JSON-LD with no Organization @id and an inline Organization on every page, no robots.txt directives or sitemap line, and llms.txt sections that predate the five tabs. hooks/seo.py gives each page its own description; overrides/main.html adds the title rule, per-page JSON-LD and breadcrumbs under the hub organization; robots.txt is added and copied by the docs workflow; llms.txt follows the tabs and now lists the limitations and self-verification pages. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013aK3gVWzNdcM3hZ2o2awK2 --- .github/workflows/docs.yml | 4 +- hooks/seo.py | 81 ++++++++++++++++++++++++ index.md | 2 +- mkdocs.yml | 20 ++++-- overrides/main.html | 122 +++++++++++++++++++++++++------------ robots.txt | 38 ++++++++++++ 6 files changed, 222 insertions(+), 45 deletions(-) create mode 100644 hooks/seo.py create mode 100644 robots.txt diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 66e3a60..9071e84 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -14,6 +14,8 @@ on: - "measurement/**" - "PRIVACY.md" - "overrides/**" + - "hooks/**" + - "robots.txt" - "index.md" - "CODE_OF_CONDUCT.md" - "LIMITATIONS.md" @@ -61,7 +63,7 @@ jobs: # measurement/REPORT.md is in the nav as Self-verification. if [ -d measurement ]; then cp -r measurement $BUILD/measurement; fi - for fname in index.md CHANGELOG.md CONTRIBUTING.md CODE_OF_CONDUCT.md LIMITATIONS.md SPONSORS.md PRIVACY.md CNAME; do + for fname in index.md CHANGELOG.md CONTRIBUTING.md CODE_OF_CONDUCT.md LIMITATIONS.md SPONSORS.md PRIVACY.md CNAME robots.txt; do if [ -f "$fname" ]; then cp "$fname" "$BUILD/$fname"; fi done diff --git a/hooks/seo.py b/hooks/seo.py new file mode 100644 index 0000000..47bc852 --- /dev/null +++ b/hooks/seo.py @@ -0,0 +1,81 @@ +"""Give every page its own meta description. + +Material falls back to site_description when a page has none in its front +matter, so every inner page showed search engines and answer engines the same +snippet. This hook takes the first paragraph of prose on the page instead. +Front matter still wins; add a `description:` there to write one by hand. +""" +import re + +LIMIT = 155 +MINIMUM = 50 + +# Lines that are not prose: headings, admonitions, fences, HTML, tables, quotes, +# lists, attribute lists, rules, and the chain label that opens landing pages +# ("[01 \u00b7 Weights: ...](https://agentrust-io.com/#chain)"). +_NOT_PROSE = re.compile(r'^(#|!!!|\?\?\?|<|\||>|[-*+] |\d+\. |\{|:::|---|\*\*\*|\[\d\d \u00b7 )') + + +def _plain(text): + text = re.sub(r'!\[[^\]]*\]\([^)]*\)', '', text) + text = re.sub(r'\[([^\]]+)\]\([^)]*\)', r'\1', text) + text = re.sub(r'\[([^\]]+)\]\[[^\]]*\]', r'\1', text) + text = re.sub(r'\{\s*[:.#][^}]*\}', '', text) + text = re.sub(r'`([^`]*)`', r'\1', text) + text = re.sub(r'(\*\*|__)(.+?)\1', r'\2', text) + text = re.sub(r'(?]+>', '', text) + # House style has no em or en dashes; source text sometimes does. + text = text.replace(' \u2014 ', ', ').replace('\u2014', ', ').replace('\u2013', ' to ') + return re.sub(r'\s+', ' ', text).strip() + + +def first_paragraph(markdown): + fence = None + lines = [] + for raw in markdown.splitlines(): + line = raw.strip() + if fence: + if line.startswith(fence): + fence = None + continue + if line.startswith(('```', '~~~')): + fence = line[:3] + if lines: + break + continue + if not line: + if lines: + break + continue + if raw.startswith((' ', '\t')) and not lines: + continue + if _NOT_PROSE.match(line): + if lines: + break + continue + lines.append(line) + return _plain(' '.join(lines)) + + +def cap(text, limit=LIMIT): + if len(text) <= limit: + return text + cut = text[:limit + 1].rsplit(' ', 1)[0].rstrip(',;:') + end = cut.rfind('. ') + if end >= MINIMUM: + return cut[:end + 1] + # No sentence end in range: cut short enough that the ellipsis fits the limit. + cut = text[:limit - 2].rsplit(' ', 1)[0].rstrip(',;:.') + return cut + '...' + + +def on_page_markdown(markdown, page, config, files): + if page.meta.get('description'): + return markdown + text = first_paragraph(markdown) + if len(text) >= MINIMUM: + page.meta['description'] = cap(text) + elif page.title: + page.meta['description'] = cap(f'{page.title}. {config["site_description"]}') + return markdown diff --git a/index.md b/index.md index 13f1f46..8bde4cc 100644 --- a/index.md +++ b/index.md @@ -3,7 +3,7 @@ hide: - navigation - toc title: TRACE conformance suite -description: Run this suite against a TRACE record to see which conformance level it reaches, and produce a report anyone can reproduce from the record and the suite version. +description: Run this suite against a TRACE record to see which conformance level it reaches, with a report anyone can reproduce from the record and suite version. --- [04 · Evidence: can a third party verify all of it offline, years later?](https://agentrust-io.com/#chain) diff --git a/mkdocs.yml b/mkdocs.yml index 714553b..54c047b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,6 +9,8 @@ copyright: "© 2026 AgenTrust contributors. Apache 2.0" exclude_docs: | .github/ + hooks/ + overrides/ node_modules/ src/ tests/ @@ -59,6 +61,10 @@ theme: icon: repo: fontawesome/brands/github +# Gives each page its own meta description; see hooks/seo.py. +hooks: + - hooks/seo.py + plugins: - search: lang: en @@ -70,13 +76,14 @@ plugins: portable, signed runtime evidence about AI agent runs. Use it to see which conformance level a TRACE record reaches and to produce a report anyone can reproduce. It checks the shape of attestation fields, not - the attestation itself. + the attestation itself. It is part of AgenTrust, open specifications + for verifiable AI: https://agentrust-io.com sections: - Getting started: + Get started: - index.md - docs/quickstart.md + Specification: - docs/levels.md - Test modules: - docs/modules.md - docs/modules/tr-env.md - docs/modules/tr-sig.md @@ -86,10 +93,15 @@ plugins: - docs/modules/tr-txn.md - docs/modules/tr-anc.md - docs/modules/tr-sca.md - Reference: - docs/error-codes.md + Guides: - docs/tutorials/writing-conformance-tests.md - docs/tutorials/ci-integration.md + - measurement/REPORT.md + Project: + - LIMITATIONS.md + - CHANGELOG.md + - CONTRIBUTING.md - minify: minify_html: true - mkdocstrings: diff --git a/overrides/main.html b/overrides/main.html index 718f634..82d963a 100644 --- a/overrides/main.html +++ b/overrides/main.html @@ -3,78 +3,122 @@ {# SEO and AEO head additions for the TRACE conformance suite. - 1. The homepage is overridden. Material falls back to site_name when a - page has no front-matter title, which left the homepage titled "TRACE - Tests". Front matter is not an option: docs_dir is the repository root, so - README.md is also the GitHub landing page and YAML would render as noise - there. + Title: Material appends " - TRACE Tests" to every page title and uses the + front-matter title on the home page. The home page now uses that title alone, + and a title that already names the site gets no suffix. Open Graph and + Twitter use the same title string. An earlier comment here described a + homepage title override that did not exist; this block is that override. - 2. Open Graph and Twitter meta, so links produce a card instead of a bare URL, - plus JSON-LD so answer engines can model what this suite is. + JSON-LD: one AgenTrust Organization node under the hub's @id, the WebSite, + and a node for the page itself (WebPage on the home page, TechArticle + elsewhere) carrying its own title, description and canonical URL. Inner pages + add a BreadcrumbList; the home page adds the SoftwareApplication node for the + agentrust-trace-tests package. Page descriptions come from hooks/seo.py when + the front matter has none. Asset paths derive from config.site_url. Do not hardcode them: assets are served under /docs/assets/ here because docs_dir is the repository root and - CI copies docs/ into the build directory. That differs between sibling - repositories, and a hardcoded path is how cMCP shipped a broken card. + CI copies docs/ into the build directory. #} +{% block htmltitle %} + {%- set t = page.meta.title if page and page.meta and page.meta.title else (page.title | striptags if page and page.title and not page.is_homepage else "") -%} + {%- if not t %} + <title>{{ config.site_name }} + {%- elif (page and page.is_homepage) or config.site_name in t %} + {{ t }} + {%- else %} + {{ t }} - {{ config.site_name }} + {%- endif %} +{% endblock %} + {% block extrahead %} {{ super() }} - {# page.title is the nav label ("Home"); page.meta.title is the front matter. #} - {% set page_name = (page.meta.title if page and page.meta and page.meta.title else (page.title if page else None)) %} - {% set page_desc = page.meta.description if page and page.meta and page.meta.description else config.site_description %} - {% set page_url = page.canonical_url if page and page.canonical_url else config.site_url %} - {% set og_image = config.site_url ~ 'docs/assets/og.png' %} - {% set social_title = 'Verify your TRACE implementation' %} + {%- set t = page.meta.title if page and page.meta and page.meta.title else (page.title | striptags if page and page.title and not page.is_homepage else "") -%} + {%- set is_home = page and page.is_homepage -%} + {%- set page_title = config.site_name if not t else (t if is_home or config.site_name in t else t ~ " - " ~ config.site_name) -%} + {%- set page_desc = page.meta.description if page and page.meta and page.meta.description else config.site_description -%} + {%- set page_url = page.canonical_url if page and page.canonical_url else config.site_url %} - - - + + + - + - - - - + + + diff --git a/robots.txt b/robots.txt new file mode 100644 index 0000000..734a195 --- /dev/null +++ b/robots.txt @@ -0,0 +1,38 @@ +# tests.agentrust-io.com: search and answer-engine crawlers are welcome. + +User-agent: * +Allow: / +# Cloudflare email-obfuscation endpoint: not a page +Disallow: /cdn-cgi/ + +User-agent: GPTBot +Allow: / + +User-agent: OAI-SearchBot +Allow: / + +User-agent: ChatGPT-User +Allow: / + +User-agent: ClaudeBot +Allow: / + +User-agent: Claude-SearchBot +Allow: / + +User-agent: Claude-User +Allow: / + +User-agent: PerplexityBot +Allow: / + +User-agent: Perplexity-User +Allow: / + +User-agent: Google-Extended +Allow: / + +User-agent: CCBot +Allow: / + +Sitemap: https://tests.agentrust-io.com/sitemap.xml From a41af06ef9f3157692ccde732af90256608fa55d Mon Sep 17 00:00:00 2001 From: Imran Siddique Date: Mon, 14 Sep 2026 16:47:07 -0700 Subject: [PATCH 2/2] docs: update the description hook to the shared kit v2 Indented continuation lines of a skipped list item are no longer read as prose, so CHANGELOG's description is no longer the wrapped second line of its first bullet, and double quotes in a derived description become apostrophes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013aK3gVWzNdcM3hZ2o2awK2 --- hooks/seo.py | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/hooks/seo.py b/hooks/seo.py index 47bc852..79941ef 100644 --- a/hooks/seo.py +++ b/hooks/seo.py @@ -10,10 +10,19 @@ LIMIT = 155 MINIMUM = 50 -# Lines that are not prose: headings, admonitions, fences, HTML, tables, quotes, -# lists, attribute lists, rules, and the chain label that opens landing pages -# ("[01 \u00b7 Weights: ...](https://agentrust-io.com/#chain)"). -_NOT_PROSE = re.compile(r'^(#|!!!|\?\?\?|<|\||>|[-*+] |\d+\. |\{|:::|---|\*\*\*|\[\d\d \u00b7 )') +# Lines that are not prose: headings, admonitions, HTML, tables, quotes, lists, +# attribute lists, rules, snippet includes, and the chain label that opens +# landing pages ("[01 \u00b7 Weights: ...](https://agentrust-io.com/#chain)"). +_NOT_PROSE = re.compile(r'^(#|!!!|\?\?\?|<|\||>|[-*+] |\d+\. |\{|:::|---|\*\*\*|--8<--|\[\d\d \u00b7 )') + +# Front-of-page metadata such as "**Status**: Accepted" or "Last updated: 2026-08-01". +_METADATA = re.compile( + r'^(\*\*[^*]+\*\*\s*:|\*\*[^*]+:\*\*|' + '(Status|Date|Last updated|Updated|Stability|Document status|Applies to|Written|' + 'Authors?|Contact|Owner|Organisation|Organization|Version|Scope|Target|' + r'Spec section|Related issues|Supersedes|Superseded by)\s*:)', + re.IGNORECASE, +) def _plain(text): @@ -26,13 +35,17 @@ def _plain(text): text = re.sub(r'(?]+>', '', text) # House style has no em or en dashes; source text sometimes does. - text = text.replace(' \u2014 ', ', ').replace('\u2014', ', ').replace('\u2013', ' to ') - return re.sub(r'\s+', ' ', text).strip() + text = re.sub(r'\s*\u2014\s*', ', ', text).replace('\u2013', ' to ') + # Material writes the description into content="..." without escaping it. + text = text.replace('"', "'").replace('\u201c', "'").replace('\u201d', "'") + text = re.sub(r'\s+', ' ', text).strip() + return re.sub(r'\s+,', ',', text) def first_paragraph(markdown): fence = None lines = [] + skipping = False for raw in markdown.splitlines(): line = raw.strip() if fence: @@ -47,12 +60,18 @@ def first_paragraph(markdown): if not line: if lines: break + skipping = False continue - if raw.startswith((' ', '\t')) and not lines: + # Indented lines before any prose are admonition bodies; after a skipped + # list item or metadata line they are its wrapped continuation. + if raw[:1] in (' ', '\t') and (not lines or skipping): continue - if _NOT_PROSE.match(line): + if _NOT_PROSE.match(line) or _METADATA.match(line): if lines: break + skipping = True + continue + if skipping and not lines: continue lines.append(line) return _plain(' '.join(lines)) @@ -72,10 +91,12 @@ def cap(text, limit=LIMIT): def on_page_markdown(markdown, page, config, files): if page.meta.get('description'): + # Hand-written descriptions reach the same unescaped attribute. + page.meta['description'] = _plain(str(page.meta['description'])) return markdown text = first_paragraph(markdown) if len(text) >= MINIMUM: page.meta['description'] = cap(text) elif page.title: - page.meta['description'] = cap(f'{page.title}. {config["site_description"]}') + page.meta['description'] = cap(_plain(f'{page.title}. {config["site_description"]}')) return markdown