From 4b4219b794498fde696e2bce19fc2a690bc792ca Mon Sep 17 00:00:00 2001 From: Chaitanya Sharma Date: Thu, 17 Sep 2026 20:07:16 +0530 Subject: [PATCH 1/4] fix(static-md): keep indentation inside fenced code blocks toPlainMarkdown removed every leading space and tab from every line of every fenced code block, not only the fence's own indentation. Nested YAML keys and Python bodies came out flush left, so the Markdown copies linked from llms.txt carried code that no longer means what the page shows. For example hyperexecute-yaml-version0.2.md served 'framework:' followed by an unindented 'name:'. Remove at most the fence's own indentation from each line, as CommonMark does for an indented fence. On stage at 08fce1fc, generated files with an indented code line go from 0 to 536, and 537 of the 1,408 generated files change. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/generate-static-md.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/generate-static-md.js b/scripts/generate-static-md.js index dee06b8be..99386530e 100644 --- a/scripts/generate-static-md.js +++ b/scripts/generate-static-md.js @@ -75,8 +75,12 @@ function toPlainMarkdown(body) { // 1. Shield fenced code blocks (with any leading indent) so later transforms // never touch their content. const codeBlocks = []; - body = body.replace(/^[ \t]*```[\s\S]*?```/gm, (block) => { - codeBlocks.push(block.replace(/^[ \t]+/gm, '')); // de-indent nested fences + body = body.replace(/^([ \t]*)```[\s\S]*?```/gm, (block, fenceIndent) => { + // De-indent nested fences the CommonMark way: remove at most the fence's + // own indentation from each line, so indentation inside the code (YAML, + // Python) is kept. + const outdent = new RegExp(`^[ \\t]{0,${fenceIndent.length}}`, 'gm'); + codeBlocks.push(block.replace(outdent, '')); return `\u0000CODE${codeBlocks.length - 1}\u0000`; }); From edb968536ff230e48e494a0562687062ad6f3394 Mon Sep 17 00:00:00 2001 From: Chaitanya Sharma Date: Thu, 17 Sep 2026 20:07:17 +0530 Subject: [PATCH 2/4] ci(static-md): add a fence-fidelity check for generated Markdown scripts/check-static-md-fences.js runs after generate-static-md.js and compares every fenced code block in docs/ with its copy in static/docs/: the same non-blank lines, in order, with the same relative indentation (only the fence's own indentation may be removed). It fails on any changed line and on a source fence that is never closed or that another opening fence interrupts, and warns when a code block is missing from the generated copy. Node built-ins only. Wired as npm run check-static-md-fences and as a pull request workflow on docs and generator changes. On stage at 08fce1fc with the previous commit's fix: 5,143 blocks checked, 0 changed lines, 7 unclosed or interrupted fences in 3 source docs that need fixing before this check can pass, and 13 code blocks missing from 6 generated files (a separate content-dropping defect in the generator). Without the fix the same check reports 20,024 changed lines in 535 docs. Co-Authored-By: Claude Opus 5 (1M context) --- .../workflows/static-md-fence-fidelity.yml | 44 ++++ package.json | 1 + scripts/check-static-md-fences.js | 188 ++++++++++++++++++ 3 files changed, 233 insertions(+) create mode 100644 .github/workflows/static-md-fence-fidelity.yml create mode 100644 scripts/check-static-md-fences.js diff --git a/.github/workflows/static-md-fence-fidelity.yml b/.github/workflows/static-md-fence-fidelity.yml new file mode 100644 index 000000000..cd8216f56 --- /dev/null +++ b/.github/workflows/static-md-fence-fidelity.yml @@ -0,0 +1,44 @@ +# Fails a pull request when the plain-Markdown copies written by +# scripts/generate-static-md.js (served as .md and linked from llms.txt) +# lose or change code: every fenced code block must keep its lines and their +# relative indentation. Needs no npm install; both scripts use Node built-ins only. +name: Static Markdown fence fidelity + +on: + pull_request: + branches: [testmuCom, stage] + paths: + - 'docs/**' + - 'scripts/generate-static-md.js' + - 'scripts/check-static-md-fences.js' + - '.github/workflows/static-md-fence-fidelity.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: static-md-fence-fidelity-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + docs + scripts + package.json + + - uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Generate static Markdown + run: node scripts/generate-static-md.js + + - name: Check fenced code survives unchanged + run: npm run check-static-md-fences diff --git a/package.json b/package.json index 56d8c3acc..f0fb51dba 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "start": "docusaurus start", "prebuild": "node scripts/build-api-data.js && node scripts/generate-api-pages.js && node scripts/generate-static-md.js && node scripts/generate-llms-txt.js && node scripts/generate-skill-index.js", "generate-static-md": "node scripts/generate-static-md.js", + "check-static-md-fences": "node scripts/check-static-md-fences.js", "generate-llms-txt": "node scripts/generate-llms-txt.js", "generate-skill-index": "node scripts/generate-skill-index.js", "sitemap-exclusions": "node scripts/sitemap-exclusions.js", diff --git a/scripts/check-static-md-fences.js b/scripts/check-static-md-fences.js new file mode 100644 index 000000000..78c8c4266 --- /dev/null +++ b/scripts/check-static-md-fences.js @@ -0,0 +1,188 @@ +/** + * Fence-fidelity check for the Markdown copies written by generate-static-md.js. + * + * Every fenced code block in a source doc must come out of the generator with + * the same code: the same non-blank lines, in the same order, with the same + * relative indentation. Only the fence's own indentation may be removed (the + * CommonMark rule for a fence nested in a list or JSX). YAML and Python change + * meaning when inner indentation is lost, and AI agents read these files + * through llms.txt. + * + * Run after the generator: + * node scripts/generate-static-md.js && node scripts/check-static-md-fences.js + * + * Errors (exit 1): + * - a code line whose text or indentation differs from the source; + * - a source fence that is never closed, or that another opening fence + * interrupts. Every later fence in that file then pairs the wrong way + * round, on the site and in the generated copy alike. + * Warnings (exit 1 only with --strict): + * - a source code block missing from the generated copy (for example when a + * tag-stripping step removes a whole region of the page); + * - a slug produced by more than one doc, which is then not checked. + */ + +const fs = require('fs'); +const path = require('path'); + +const DOCS_DIR = path.join(__dirname, '..', 'docs'); +const OUT_DIR = path.join(__dirname, '..', 'static', 'docs'); +const STRICT = process.argv.includes('--strict'); + +/** Same shallow frontmatter read as generate-static-md.js, plus where the body starts. */ +function parseFrontmatter(raw) { + const match = raw.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/); + if (!match) return { data: {}, body: raw, bodyLine: 0 }; + const data = {}; + for (const line of match[1].split(/\r?\n/)) { + const kv = line.match(/^([A-Za-z0-9_-]+):\s*(.*)$/); + if (!kv) continue; + let value = kv[2].trim(); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + data[kv[1]] = value; + } + const bodyLine = (match[0].match(/\n/g) || []).length; + return { data, body: raw.slice(match[0].length), bodyLine }; +} + +/** Same slug rule as generate-static-md.js. */ +function resolveSlug(data, fileName) { + const fileBase = fileName.replace(/\.mdx?$/, ''); + const base = data.slug || data.id || fileBase; + const name = base.replace(/^\//, '').replace(/\/$/, '').split('/').pop(); + return name || fileBase; +} + +/** + * Fenced blocks of a Markdown text, line based. Each block keeps the fence's + * indentation, its 1-based start line and its content lines. `firstLine` is + * the number of lines that precede `text` in its file. + */ +function fences(text, firstLine = 0) { + const lines = text.split(/\r?\n/); + const blocks = []; + const unclosed = []; + let open = null; + lines.forEach((line, i) => { + const number = firstLine + i + 1; + if (!open) { + // An info string cannot contain a backtick, so "```status``` | x" is inline code. + const m = line.match(/^([ \t]*)(`{3,})[^`]*$/); + if (m) open = { indent: m[1].length, marker: m[2], start: number, lines: [] }; + return; + } + // Lenient close: "``` -->" (a fence inside an HTML comment) still closes. + const close = line.match(/^[ \t]*(`{3,})(?![`\w])/); + if (close && close[1].length >= open.marker.length) { + blocks.push(open); + open = null; + return; + } + const inner = line.match(/^[ \t]*(`{3,})[ \t]*[A-Za-z][\w+#.-]*[ \t]*$/); + if (inner && inner[1].length >= open.marker.length) { + unclosed.push({ start: open.start, next: number }); + } + open.lines.push({ text: line, number }); + }); + if (open) unclosed.push({ start: open.start, next: null }); + return { blocks, unclosed }; +} + +/** Non-blank code lines with trailing whitespace removed (the generator trims both). */ +const codeLines = (block) => + block.lines + .map((l) => ({ ...l, text: l.text.replace(/[ \t]+$/, '') })) + .filter((l) => l.text.trim() !== ''); + +/** A source line as the rendered site shows it: at most the fence indent removed. */ +const outdent = (text, width) => text.replace(new RegExp(`^[ \\t]{0,${width}}`), ''); + +const signature = (lines) => lines.map((l) => l.text.replace(/\s+/g, '')).join('\n'); + +function main() { + const errors = []; + const warnings = []; + let blocksChecked = 0; + + // Two docs with one slug overwrite each other's output in directory order, + // which is not stable across file systems, so those slugs are not checked. + const bySlug = new Map(); + const duplicated = new Set(); + const sources = fs + .readdirSync(DOCS_DIR) + .filter((f) => /\.mdx?$/.test(f)) + .sort(); + for (const file of sources) { + const raw = fs.readFileSync(path.join(DOCS_DIR, file), 'utf8').replace(/^\uFEFF/, ''); + const { data, body, bodyLine } = parseFrontmatter(raw); + if (String(data.draft).toLowerCase() === 'true') continue; + const slug = resolveSlug(data, file); + if (bySlug.has(slug)) duplicated.add(slug); + bySlug.set(slug, { file, body, bodyLine }); + } + for (const slug of duplicated) { + warnings.push(`slug "${slug}" is produced by more than one doc; not checked`); + bySlug.delete(slug); + } + + for (const [slug, { file, body, bodyLine }] of bySlug) { + const outPath = path.join(OUT_DIR, `${slug}.md`); + if (!fs.existsSync(outPath)) { + errors.push(`docs/${file}: no generated file static/docs/${slug}.md`); + continue; + } + + const source = fences(body, bodyLine); + if (source.unclosed.length) { + // Comparing lines here would only repeat this one defect many times. + for (const u of source.unclosed) { + errors.push( + `docs/${file} line ${u.start}: code fence is not closed` + + (u.next ? ` before the fence at line ${u.next}` : ' before the end of the file') + ); + } + continue; + } + + const outBlocks = fences(fs.readFileSync(outPath, 'utf8')).blocks.map((b) => ({ + lines: codeLines(b), + used: false, + })); + + for (const src of source.blocks) { + const expected = codeLines(src).map((l) => ({ ...l, text: outdent(l.text, src.indent) })); + if (!expected.length) continue; + const sig = signature(expected); + const match = outBlocks.find((b) => !b.used && signature(b.lines) === sig); + if (!match) { + warnings.push(`docs/${file} line ${src.start}: code block missing from static/docs/${slug}.md`); + continue; + } + match.used = true; + blocksChecked++; + expected.forEach((line, i) => { + const got = match.lines[i]; + if (got.text !== line.text) { + errors.push( + `docs/${file} line ${line.number} -> static/docs/${slug}.md line ${got.number}: ` + + `expected ${JSON.stringify(line.text)}, got ${JSON.stringify(got.text)}` + ); + } + }); + } + } + + for (const w of warnings) console.warn(`warning: ${w}`); + for (const e of errors) console.error(`error: ${e}`); + console.log( + `Checked ${blocksChecked} code block(s) in ${bySlug.size} doc(s): ${errors.length} error(s), ${warnings.length} warning(s).` + ); + if (errors.length || (STRICT && warnings.length)) process.exit(1); +} + +main(); From f55c6b4e420b1de13861e5df22a583c23baf50b8 Mon Sep 17 00:00:00 2001 From: Chaitanya Sharma Date: Thu, 17 Sep 2026 20:07:17 +0530 Subject: [PATCH 3/4] docs: close unclosed and interrupted code fences in 3 pages On stage the fence-fidelity check finds 7 code fences that are never closed or that another opening fence interrupts. On the site, every later fence in those pages pairs the wrong way round, so prose renders as code and code as prose. Each fix follows what the page was written to show: - kane-cli-testmd-composition.md: the two test.md examples that contain a nested yaml step block now use four-backtick outer fences, so the inner three-backtick fences stay inside the example. Four VerifiedTag lines that the badge script had inserted inside those examples and inside the path tree are removed; the tags before each example stay. - playwright-sdk.md: close the last bash block, which is the end of the page. - smartui-build-merging.md: close the bash blocks of strategies 1 and 2 before the next strategy heading. smartui-appium-hooks.md and smartui-cli-env-variables.md, which need the same kind of fix on testmuCom, are already closed on stage. Co-Authored-By: Claude Opus 5 (1M context) --- docs/kane-cli-testmd-composition.md | 20 ++++---------------- docs/playwright-sdk.md | 1 + docs/smartui-build-merging.md | 2 ++ 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/docs/kane-cli-testmd-composition.md b/docs/kane-cli-testmd-composition.md index bf8fffcae..8230aa2f1 100644 --- a/docs/kane-cli-testmd-composition.md +++ b/docs/kane-cli-testmd-composition.md @@ -291,7 +291,7 @@ Rules: -```markdown +````markdown ## OK @import ./helpers/login.md @@ -302,9 +302,6 @@ optional: true @import ./helpers/skip-tour.md ## NOT OK — extra config - - - ```yaml timeout: 60 ``` @@ -313,10 +310,7 @@ timeout: 60 ## NOT OK — body mixes prose and import Click somewhere first. @import ./helpers/login.md - - - -``` +```` ## How paths resolve @@ -329,9 +323,6 @@ tests/ helpers/ login.md # contains: @import ./submit-button.md submit-button.md - - - ``` When `checkout_test.md` imports `../../helpers/login.md`, the path is relative to `tests/e2e/`, so it resolves to `helpers/login.md`. When `login.md` imports `./submit-button.md`, the path is relative to `helpers/`, so it resolves to `helpers/submit-button.md`. @@ -382,16 +373,13 @@ A root-level `@import` step can be marked optional in the same way a prose step -```markdown +````markdown ## Skip the tour if it shows up ```yaml optional: true ``` @import ./helpers/dismiss-product-tour.md - - - -``` +```` If the helper fails, the run continues to the next step. The `Result.md` entry is suffixed with `(optional)`. diff --git a/docs/playwright-sdk.md b/docs/playwright-sdk.md index 06405bae7..8dd3393b5 100644 --- a/docs/playwright-sdk.md +++ b/docs/playwright-sdk.md @@ -382,3 +382,4 @@ You can pass any standard Playwright CLI options directly to this command. For i ```bash npx playwright-node-sdk playwright test tests/my-test.spec.js +``` diff --git a/docs/smartui-build-merging.md b/docs/smartui-build-merging.md index 1aa72ee88..e22fb1993 100644 --- a/docs/smartui-build-merging.md +++ b/docs/smartui-build-merging.md @@ -187,6 +187,7 @@ npx smartui merge build --source build-123 --target build-456 ```bash # 1. Merge staging build to production npx smartui merge build --source staging-build-123 --target prod-build-456 +``` ### 2. Feature Build Strategy @@ -195,6 +196,7 @@ npx smartui merge build --source staging-build-123 --target prod-build-456 ```bash # 1. Merge feature build into main build npx smartui merge build --source feature-build-789 --target main-build-101 +``` ### 3. Hotfix Build Strategy From a334c724371c3ddcfe459f8d37fc2f26675f9602 Mon Sep 17 00:00:00 2001 From: Chaitanya Sharma Date: Thu, 17 Sep 2026 20:08:01 +0530 Subject: [PATCH 4/4] ci: notify docs-retrieval-service after prod deploy Adds docs-retrieval-index-notify.yml, triggered by workflow_run on 'Deployment (Prod - testmucom New Bucket)' when it succeeds, or by hand with a deployed sha. It sends an HMAC-SHA256 signed POST /v1/index with the deployed commit sha (TE-28101, RFC section 4.3), retries on 409, 429, 5xx and connection failures with backoff, and fails on 400, 401, 403, 404 and 413. While the DOCS_RETRIEVAL_URL variable is unset it logs a notice and succeeds; once it is set, a missing DOCS_RETRIEVAL_DISPATCH_TOKEN secret is an error. It is a separate file, so the existing documentation-notify.yml (product-context notifier) is not changed. workflow_run only fires from the default branch, so on stage this workflow is dormant until it reaches testmuCom. Co-Authored-By: Claude Opus 5 (1M context) --- .../workflows/docs-retrieval-index-notify.yml | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 .github/workflows/docs-retrieval-index-notify.yml diff --git a/.github/workflows/docs-retrieval-index-notify.yml b/.github/workflows/docs-retrieval-index-notify.yml new file mode 100644 index 000000000..fb84ab417 --- /dev/null +++ b/.github/workflows/docs-retrieval-index-notify.yml @@ -0,0 +1,118 @@ +# Runs after every successful production deploy of testmuCom and tells +# docs-retrieval-service (TE-28101) which commit is now live: an HMAC-signed +# POST /v1/index with the deployed commit sha. The service downloads that +# commit, diffs it against what it has indexed and updates the search index in +# the background. +# +# Why workflow_run and not push: a push fires before the site is built and +# uploaded, so an indexer triggered by it would read pages that are not live +# yet. workflow_run waits for the deploy to finish, and the `if` below skips +# failed or cancelled deploys. +# +# workflow_run only fires for a workflow file on the default branch +# (testmuCom). On any other branch this file is dormant. +# +# This is its own file on purpose: documentation-notify.yml (the product-context +# notifier) is a separate workflow and is not changed by this one. +# +# One-time setup (Settings, Secrets and variables, Actions): +# Secret DOCS_RETRIEVAL_DISPATCH_TOKEN same value as INDEX_DISPATCH_HMAC on the service +# Variable DOCS_RETRIEVAL_URL service base URL, no trailing slash +# (while it is unset, the job logs a notice and succeeds) +# +# Manual run: Actions, "Notify docs retrieval index after prod deploy", Run +# workflow, with the 40-character sha of a commit that is already deployed. +name: Notify docs retrieval index after prod deploy + +on: + workflow_run: + workflows: ['Deployment (Prod - testmucom New Bucket)'] + types: [completed] + branches: [testmuCom] + workflow_dispatch: + inputs: + sha: + description: 'Deployed commit sha (40 hex characters)' + required: true + +permissions: + contents: read + +concurrency: + group: docs-retrieval-index-${{ github.event.workflow_run.head_sha || inputs.sha }} + cancel-in-progress: false + +jobs: + docs-retrieval-index: + name: 'docs-retrieval-service: index deployed commit' + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Signed POST /v1/index + env: + SHA: ${{ github.event.workflow_run.head_sha || inputs.sha }} + HMAC_SECRET: ${{ secrets.DOCS_RETRIEVAL_DISPATCH_TOKEN }} + BASE_URL: ${{ vars.DOCS_RETRIEVAL_URL }} + run: | + set -euo pipefail + # Until docs-retrieval-service is deployed and DOCS_RETRIEVAL_URL is + # set, this job does nothing and succeeds, so prod deploys stay green. + if [ -z "${BASE_URL}" ]; then + echo "::notice::DOCS_RETRIEVAL_URL variable is not set, so the docs-retrieval index is not notified. Nothing to do." + exit 0 + fi + if [ -z "${HMAC_SECRET}" ]; then + echo "::error::DOCS_RETRIEVAL_URL is set but the DOCS_RETRIEVAL_DISPATCH_TOKEN secret is not." + exit 1 + fi + if ! printf '%s' "${SHA}" | grep -Eq '^[0-9a-f]{40}$'; then + echo "::error::sha must be 40 lowercase hex characters, got '${SHA}'." + exit 1 + fi + + attempt=0 + max_attempts=6 + delay=15 + while :; do + attempt=$((attempt + 1)) + # Sign the exact bytes that are sent. The timestamp is fresh on every + # attempt because the service rejects a ts more than 5 minutes old. + printf '{"sha":"%s","ts":%s,"trigger":"deploy"}' "${SHA}" "$(date +%s)" > body.json + sig=$(openssl dgst -sha256 -hmac "${HMAC_SECRET}" -r body.json | cut -d' ' -f1) + status=$(curl -sS -o response.json -w '%{http_code}' \ + --max-time 30 \ + -X POST "${BASE_URL}/v1/index" \ + -H 'Content-Type: application/json' \ + -H "X-Signature-256: sha256=${sig}" \ + --data-binary @body.json || echo 000) + echo "attempt ${attempt}: HTTP ${status}" + cat response.json 2>/dev/null || true + echo + + case "${status}" in + 200|202) + echo "Index run accepted for ${SHA}." + exit 0 + ;; + 400|401|403|404|413) + echo "::error::docs-retrieval-service refused the request (HTTP ${status}); retrying will not help." + exit 1 + ;; + 409|429|5*|000) + # 409: another index run is in progress. The nightly reconcile + # also catches up, but retry so this deploy is indexed promptly. + if [ "${attempt}" -ge "${max_attempts}" ]; then + echo "::error::Giving up after ${attempt} attempts (last HTTP ${status})." + exit 1 + fi + echo "Retrying in ${delay}s." + sleep "${delay}" + delay=$((delay * 2)) + ;; + *) + echo "::error::Unexpected HTTP ${status}." + exit 1 + ;; + esac + done