docs: page descriptions, titles, structured data and crawler files for search and answer engines

.github/workflows/docs.yml

@@ -14,6 +14,8 @@ on:
       - "measurement/**"
       - "PRIVACY.md"
       - "overrides/**"
+      - "hooks/**"
+      - "robots.txt"
       - "index.md"
       - "CODE_OF_CONDUCT.md"
       - "LIMITATIONS.md"
@@ -61,7 +63,7 @@ jobs:
           # measurement/REPORT.md is in the nav as Self-verification.
           if [ -d measurement ]; then cp -r measurement $BUILD/measurement; fi
 
-          for fname in index.md CHANGELOG.md CONTRIBUTING.md CODE_OF_CONDUCT.md LIMITATIONS.md SPONSORS.md PRIVACY.md CNAME; do
+          for fname in index.md CHANGELOG.md CONTRIBUTING.md CODE_OF_CONDUCT.md LIMITATIONS.md SPONSORS.md PRIVACY.md CNAME robots.txt; do
             if [ -f "$fname" ]; then cp "$fname" "$BUILD/$fname"; fi
           done
 

hooks/seo.py

@@ -0,0 +1,102 @@
+"""Give every page its own meta description.
+
+Material falls back to site_description when a page has none in its front
+matter, so every inner page showed search engines and answer engines the same
+snippet. This hook takes the first paragraph of prose on the page instead.
+Front matter still wins; add a `description:` there to write one by hand.
+"""
+import re
+
+LIMIT = 155
+MINIMUM = 50
+
+# Lines that are not prose: headings, admonitions, HTML, tables, quotes, lists,
+# attribute lists, rules, snippet includes, and the chain label that opens
+# landing pages ("[01 \u00b7 Weights: ...](https://agentrust-io.com/#chain)").
+_NOT_PROSE = re.compile(r'^(#|!!!|\?\?\?|<|\||>|[-*+] |\d+\. |\{|:::|---|\*\*\*|--8<--|\[\d\d \u00b7 )')
+
+# Front-of-page metadata such as "**Status**: Accepted" or "Last updated: 2026-08-01".
+_METADATA = re.compile(
+    r'^(\*\*[^*]+\*\*\s*:|\*\*[^*]+:\*\*|'
+    '(Status|Date|Last updated|Updated|Stability|Document status|Applies to|Written|'
+    'Authors?|Contact|Owner|Organisation|Organization|Version|Scope|Target|'
+    r'Spec section|Related issues|Supersedes|Superseded by)\s*:)',
+    re.IGNORECASE,
+)
+
+
+def _plain(text):
+    text = re.sub(r'!\[[^\]]*\]\([^)]*\)', '', text)
+    text = re.sub(r'\[([^\]]+)\]\([^)]*\)', r'\1', text)
+    text = re.sub(r'\[([^\]]+)\]\[[^\]]*\]', r'\1', text)
+    text = re.sub(r'\{\s*[:.#][^}]*\}', '', text)
+    text = re.sub(r'`([^`]*)`', r'\1', text)
+    text = re.sub(r'(\*\*|__)(.+?)\1', r'\2', text)
+    text = re.sub(r'(?<![\w*])\*(?!\s)(.+?)(?<!\s)\*(?![\w*])', r'\1', text)
+    text = re.sub(r'<[^>]+>', '', text)
+    # House style has no em or en dashes; source text sometimes does.
+    text = re.sub(r'\s*\u2014\s*', ', ', text).replace('\u2013', ' to ')
+    # Material writes the description into content="..." without escaping it.
+    text = text.replace('"', "'").replace('\u201c', "'").replace('\u201d', "'")
+    text = re.sub(r'\s+', ' ', text).strip()
+    return re.sub(r'\s+,', ',', text)
+
+
+def first_paragraph(markdown):
+    fence = None
+    lines = []
+    skipping = False
+    for raw in markdown.splitlines():
+        line = raw.strip()
+        if fence:
+            if line.startswith(fence):
+                fence = None
+            continue
+        if line.startswith(('```', '~~~')):
+            fence = line[:3]
+            if lines:
+                break
+            continue
+        if not line:
+            if lines:
+                break
+            skipping = False
+            continue
+        # Indented lines before any prose are admonition bodies; after a skipped
+        # list item or metadata line they are its wrapped continuation.
+        if raw[:1] in (' ', '\t') and (not lines or skipping):
+            continue
+        if _NOT_PROSE.match(line) or _METADATA.match(line):
+            if lines:
+                break
+            skipping = True
+            continue
+        if skipping and not lines:
+            continue
+        lines.append(line)
+    return _plain(' '.join(lines))
+
+
+def cap(text, limit=LIMIT):
+    if len(text) <= limit:
+        return text
+    cut = text[:limit + 1].rsplit(' ', 1)[0].rstrip(',;:')
+    end = cut.rfind('. ')
+    if end >= MINIMUM:
+        return cut[:end + 1]
+    # No sentence end in range: cut short enough that the ellipsis fits the limit.
+    cut = text[:limit - 2].rsplit(' ', 1)[0].rstrip(',;:.')
+    return cut + '...'
+
+
+def on_page_markdown(markdown, page, config, files):
+    if page.meta.get('description'):
+        # Hand-written descriptions reach the same unescaped attribute.
+        page.meta['description'] = _plain(str(page.meta['description']))
+        return markdown
+    text = first_paragraph(markdown)
+    if len(text) >= MINIMUM:
+        page.meta['description'] = cap(text)
+    elif page.title:
+        page.meta['description'] = cap(_plain(f'{page.title}. {config["site_description"]}'))
+    return markdown

index.md

@@ -3,7 +3,7 @@ hide:
   - navigation
   - toc
 title: TRACE conformance suite
-description: Run this suite against a TRACE record to see which conformance level it reaches, and produce a report anyone can reproduce from the record and the suite version.
+description: Run this suite against a TRACE record to see which conformance level it reaches, with a report anyone can reproduce from the record and suite version.
 ---
 
 [04 · Evidence: can a third party verify all of it offline, years later?](https://agentrust-io.com/#chain)

mkdocs.yml

@@ -9,6 +9,8 @@ copyright: "© 2026 AgenTrust contributors. Apache 2.0"
 
 exclude_docs: |
   .github/
+  hooks/
+  overrides/
   node_modules/
   src/
   tests/
@@ -59,6 +61,10 @@ theme:
   icon:
     repo: fontawesome/brands/github
 
+# Gives each page its own meta description; see hooks/seo.py.
+hooks:
+  - hooks/seo.py
+
 plugins:
   - search:
       lang: en
@@ -70,13 +76,14 @@ plugins:
         portable, signed runtime evidence about AI agent runs. Use it to see
         which conformance level a TRACE record reaches and to produce a report
         anyone can reproduce. It checks the shape of attestation fields, not
-        the attestation itself.
+        the attestation itself. It is part of AgenTrust, open specifications
+        for verifiable AI: https://agentrust-io.com
       sections:
-        Getting started:
+        Get started:
           - index.md
           - docs/quickstart.md
+        Specification:
           - docs/levels.md
-        Test modules:
           - docs/modules.md
           - docs/modules/tr-env.md
           - docs/modules/tr-sig.md
@@ -86,10 +93,15 @@ plugins:
           - docs/modules/tr-txn.md
           - docs/modules/tr-anc.md
           - docs/modules/tr-sca.md
-        Reference:
           - docs/error-codes.md
+        Guides:
           - docs/tutorials/writing-conformance-tests.md
           - docs/tutorials/ci-integration.md
+          - measurement/REPORT.md
+        Project:
+          - LIMITATIONS.md
+          - CHANGELOG.md
+          - CONTRIBUTING.md
   - minify:
       minify_html: true
   - mkdocstrings:

overrides/main.html

@@ -3,78 +3,122 @@
 {#
   SEO and AEO head additions for the TRACE conformance suite.
 
-  1. The homepage <title> is overridden. Material falls back to site_name when a
-     page has no front-matter title, which left the homepage titled "TRACE
-     Tests". Front matter is not an option: docs_dir is the repository root, so
-     README.md is also the GitHub landing page and YAML would render as noise
-     there.
+  Title: Material appends " - TRACE Tests" to every page title and uses the
+  front-matter title on the home page. The home page now uses that title alone,
+  and a title that already names the site gets no suffix. Open Graph and
+  Twitter use the same title string. An earlier comment here described a
+  homepage title override that did not exist; this block is that override.
 
-  2. Open Graph and Twitter meta, so links produce a card instead of a bare URL,
-     plus JSON-LD so answer engines can model what this suite is.
+  JSON-LD: one AgenTrust Organization node under the hub's @id, the WebSite,
+  and a node for the page itself (WebPage on the home page, TechArticle
+  elsewhere) carrying its own title, description and canonical URL. Inner pages
+  add a BreadcrumbList; the home page adds the SoftwareApplication node for the
+  agentrust-trace-tests package. Page descriptions come from hooks/seo.py when
+  the front matter has none.
 
   Asset paths derive from config.site_url. Do not hardcode them: assets are
   served under /docs/assets/ here because docs_dir is the repository root and
-  CI copies docs/ into the build directory. That differs between sibling
-  repositories, and a hardcoded path is how cMCP shipped a broken card.
+  CI copies docs/ into the build directory.
 #}
 
+{% block htmltitle %}
+  {%- set t = page.meta.title if page and page.meta and page.meta.title else (page.title | striptags if page and page.title and not page.is_homepage else "") -%}
+  {%- if not t %}
+    <title>{{ config.site_name }}</title>
+  {%- elif (page and page.is_homepage) or config.site_name in t %}
+    <title>{{ t }}</title>
+  {%- else %}
+    <title>{{ t }} - {{ config.site_name }}</title>
+  {%- endif %}
+{% endblock %}
+
 {% block extrahead %}
   {{ super() }}
-  {# page.title is the nav label ("Home"); page.meta.title is the front matter. #}
-  {% set page_name = (page.meta.title if page and page.meta and page.meta.title else (page.title if page else None)) %}
-  {% set page_desc = page.meta.description if page and page.meta and page.meta.description else config.site_description %}
-  {% set page_url = page.canonical_url if page and page.canonical_url else config.site_url %}
-  {% set og_image = config.site_url ~ 'docs/assets/og.png' %}
-  {% set social_title = 'Verify your TRACE implementation' %}
+  {%- set t = page.meta.title if page and page.meta and page.meta.title else (page.title | striptags if page and page.title and not page.is_homepage else "") -%}
+  {%- set is_home = page and page.is_homepage -%}
+  {%- set page_title = config.site_name if not t else (t if is_home or config.site_name in t else t ~ " - " ~ config.site_name) -%}
+  {%- set page_desc = page.meta.description if page and page.meta and page.meta.description else config.site_description -%}
+  {%- set page_url = page.canonical_url if page and page.canonical_url else config.site_url %}
 
   <meta property="og:type" content="website">
-  <meta property="og:site_name" content="{{ config.site_name }}">
-  <meta property="og:title" content="{% if page_name %}{{ page_name }} - {{ config.site_name }}{% else %}{{ config.site_name }}: {{ social_title }}{% endif %}">
-  <meta property="og:description" content="{{ page_desc }}">
+  <meta property="og:site_name" content="AgenTrust">
+  <meta property="og:title" content="{{ page_title | e }}">
+  <meta property="og:description" content="{{ page_desc | e }}">
   <meta property="og:url" content="{{ page_url }}">
-  <meta property="og:image" content="{{ og_image }}">
+  <meta property="og:image" content="{{ config.site_url }}docs/assets/og.png">
   <meta property="og:image:width" content="1200">
   <meta property="og:image:height" content="630">
   <meta property="og:image:alt" content="TRACE Tests: the AgenTrust conformance suite">
   <meta property="og:locale" content="en_US">
-
   <meta name="twitter:card" content="summary_large_image">
-  <meta name="twitter:title" content="{% if page_name %}{{ page_name }} - {{ config.site_name }}{% else %}{{ config.site_name }}: {{ social_title }}{% endif %}">
-  <meta name="twitter:description" content="{{ page_desc }}">
-  <meta name="twitter:image" content="{{ og_image }}">
+  <meta name="twitter:title" content="{{ page_title | e }}">
+  <meta name="twitter:description" content="{{ page_desc | e }}">
+  <meta name="twitter:image" content="{{ config.site_url }}docs/assets/og.png">
 
   <script type="application/ld+json">
   {
     "@context": "https://schema.org",
     "@graph": [
-      {
-        "@type": "SoftwareApplication",
-        "name": "TRACE Tests",
-        "applicationCategory": "DeveloperApplication",
-        "description": "Conformance tests and an integration harness for TRACE records. It reports which conformance level a record reaches and checks the shape of attestation fields, not the attestation itself.",
-        "url": "{{ config.site_url }}",
-        "offers": { "@type": "Offer", "price": "0", "priceCurrency": "USD" },
-        "author": { "@type": "Organization", "name": "AgenTrust", "url": "https://agentrust-io.com" },
-        "isPartOf": { "@type": "WebSite", "name": "{{ config.site_name }}", "url": "{{ config.site_url }}" }
-      },
       {
         "@type": "Organization",
+        "@id": "https://agentrust-io.com/#organization",
         "name": "AgenTrust",
-        "url": "https://agentrust-io.com",
+        "url": "https://agentrust-io.com/",
         "sameAs": [
           "https://github.com/agentrust-io",
-          "https://agentrust-io.com/demos/",
           "https://trace.agentrust-io.com",
-          "https://cmcp.agentrust-io.com",
           "https://manifest.agentrust-io.com",
-          "https://ca2a.agentrust-io.com"
+          "https://cmcp.agentrust-io.com",
+          "https://ca2a.agentrust-io.com",
+          "https://wcm.agentrust-io.com",
+          "https://tests.agentrust-io.com",
+          "https://governance.agentrust-io.com"
         ]
       },
       {
         "@type": "WebSite",
-        "name": "{{ config.site_name }}",
-        "url": "{{ config.site_url }}"
+        "@id": {{ (config.site_url ~ "#website") | tojson }},
+        "name": {{ config.site_name | tojson }},
+        "url": {{ config.site_url | tojson }},
+        "description": {{ config.site_description | tojson }},
+        "publisher": { "@id": "https://agentrust-io.com/#organization" }
+      },
+      {
+        "@type": {{ ("WebPage" if is_home else "TechArticle") | tojson }},
+        "@id": {{ (page_url ~ "#page") | tojson }},
+        "url": {{ page_url | tojson }},
+        "name": {{ page_title | tojson }},
+        "headline": {{ page_title | tojson }},
+        "description": {{ page_desc | tojson }},
+        "inLanguage": "en",
+        "isPartOf": { "@id": {{ (config.site_url ~ "#website") | tojson }} },
+        "publisher": { "@id": "https://agentrust-io.com/#organization" }
+      }
+      {%- if is_home %},
+      {
+        "@type": "SoftwareApplication",
+        "name": "TRACE Tests",
+        "alternateName": "agentrust-trace-tests",
+        "description": "Conformance tests and an integration harness for TRACE records. It reports which conformance level a record reaches and checks the shape of attestation fields, not the attestation itself.",
+        "applicationCategory": "DeveloperApplication",
+        "operatingSystem": "Cross-platform",
+        "url": {{ config.site_url | tojson }},
+        "downloadUrl": "https://pypi.org/project/agentrust-trace-tests/",
+        "codeRepository": {{ config.repo_url | tojson }},
+        "license": "https://www.apache.org/licenses/LICENSE-2.0",
+        "author": { "@id": "https://agentrust-io.com/#organization" },
+        "publisher": { "@id": "https://agentrust-io.com/#organization" },
+        "offers": { "@type": "Offer", "price": "0", "priceCurrency": "USD" }
+      }
+      {%- elif page and page.canonical_url %},
+      {
+        "@type": "BreadcrumbList",
+        "itemListElement": [
+          { "@type": "ListItem", "position": 1, "name": {{ config.site_name | tojson }}, "item": {{ config.site_url | tojson }} },
+          { "@type": "ListItem", "position": 2, "name": {{ (t or page_title) | tojson }}, "item": {{ page_url | tojson }} }
+        ]
       }
+      {%- endif %}
     ]
   }
   </script>

robots.txt

@@ -0,0 +1,38 @@
+# tests.agentrust-io.com: search and answer-engine crawlers are welcome.
+
+User-agent: *
+Allow: /
+# Cloudflare email-obfuscation endpoint: not a page
+Disallow: /cdn-cgi/
+
+User-agent: GPTBot
+Allow: /
+
+User-agent: OAI-SearchBot
+Allow: /
+
+User-agent: ChatGPT-User
+Allow: /
+
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: Claude-SearchBot
+Allow: /
+
+User-agent: Claude-User
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+User-agent: Perplexity-User
+Allow: /
+
+User-agent: Google-Extended
+Allow: /
+
+User-agent: CCBot
+Allow: /
+
+Sitemap: https://tests.agentrust-io.com/sitemap.xml