diff --git a/.github/workflows/docs-retrieval-index-notify.yml b/.github/workflows/docs-retrieval-index-notify.yml new file mode 100644 index 000000000..fb84ab417 --- /dev/null +++ b/.github/workflows/docs-retrieval-index-notify.yml @@ -0,0 +1,118 @@ +# Runs after every successful production deploy of testmuCom and tells +# docs-retrieval-service (TE-28101) which commit is now live: an HMAC-signed +# POST /v1/index with the deployed commit sha. The service downloads that +# commit, diffs it against what it has indexed and updates the search index in +# the background. +# +# Why workflow_run and not push: a push fires before the site is built and +# uploaded, so an indexer triggered by it would read pages that are not live +# yet. workflow_run waits for the deploy to finish, and the `if` below skips +# failed or cancelled deploys. +# +# workflow_run only fires for a workflow file on the default branch +# (testmuCom). On any other branch this file is dormant. +# +# This is its own file on purpose: documentation-notify.yml (the product-context +# notifier) is a separate workflow and is not changed by this one. +# +# One-time setup (Settings, Secrets and variables, Actions): +# Secret DOCS_RETRIEVAL_DISPATCH_TOKEN same value as INDEX_DISPATCH_HMAC on the service +# Variable DOCS_RETRIEVAL_URL service base URL, no trailing slash +# (while it is unset, the job logs a notice and succeeds) +# +# Manual run: Actions, "Notify docs retrieval index after prod deploy", Run +# workflow, with the 40-character sha of a commit that is already deployed. +name: Notify docs retrieval index after prod deploy + +on: + workflow_run: + workflows: ['Deployment (Prod - testmucom New Bucket)'] + types: [completed] + branches: [testmuCom] + workflow_dispatch: + inputs: + sha: + description: 'Deployed commit sha (40 hex characters)' + required: true + +permissions: + contents: read + +concurrency: + group: docs-retrieval-index-${{ github.event.workflow_run.head_sha || inputs.sha }} + cancel-in-progress: false + +jobs: + docs-retrieval-index: + name: 'docs-retrieval-service: index deployed commit' + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Signed POST /v1/index + env: + SHA: ${{ github.event.workflow_run.head_sha || inputs.sha }} + HMAC_SECRET: ${{ secrets.DOCS_RETRIEVAL_DISPATCH_TOKEN }} + BASE_URL: ${{ vars.DOCS_RETRIEVAL_URL }} + run: | + set -euo pipefail + # Until docs-retrieval-service is deployed and DOCS_RETRIEVAL_URL is + # set, this job does nothing and succeeds, so prod deploys stay green. + if [ -z "${BASE_URL}" ]; then + echo "::notice::DOCS_RETRIEVAL_URL variable is not set, so the docs-retrieval index is not notified. Nothing to do." + exit 0 + fi + if [ -z "${HMAC_SECRET}" ]; then + echo "::error::DOCS_RETRIEVAL_URL is set but the DOCS_RETRIEVAL_DISPATCH_TOKEN secret is not." + exit 1 + fi + if ! printf '%s' "${SHA}" | grep -Eq '^[0-9a-f]{40}$'; then + echo "::error::sha must be 40 lowercase hex characters, got '${SHA}'." + exit 1 + fi + + attempt=0 + max_attempts=6 + delay=15 + while :; do + attempt=$((attempt + 1)) + # Sign the exact bytes that are sent. The timestamp is fresh on every + # attempt because the service rejects a ts more than 5 minutes old. + printf '{"sha":"%s","ts":%s,"trigger":"deploy"}' "${SHA}" "$(date +%s)" > body.json + sig=$(openssl dgst -sha256 -hmac "${HMAC_SECRET}" -r body.json | cut -d' ' -f1) + status=$(curl -sS -o response.json -w '%{http_code}' \ + --max-time 30 \ + -X POST "${BASE_URL}/v1/index" \ + -H 'Content-Type: application/json' \ + -H "X-Signature-256: sha256=${sig}" \ + --data-binary @body.json || echo 000) + echo "attempt ${attempt}: HTTP ${status}" + cat response.json 2>/dev/null || true + echo + + case "${status}" in + 200|202) + echo "Index run accepted for ${SHA}." + exit 0 + ;; + 400|401|403|404|413) + echo "::error::docs-retrieval-service refused the request (HTTP ${status}); retrying will not help." + exit 1 + ;; + 409|429|5*|000) + # 409: another index run is in progress. The nightly reconcile + # also catches up, but retry so this deploy is indexed promptly. + if [ "${attempt}" -ge "${max_attempts}" ]; then + echo "::error::Giving up after ${attempt} attempts (last HTTP ${status})." + exit 1 + fi + echo "Retrying in ${delay}s." + sleep "${delay}" + delay=$((delay * 2)) + ;; + *) + echo "::error::Unexpected HTTP ${status}." + exit 1 + ;; + esac + done diff --git a/.github/workflows/static-md-fence-fidelity.yml b/.github/workflows/static-md-fence-fidelity.yml new file mode 100644 index 000000000..cd8216f56 --- /dev/null +++ b/.github/workflows/static-md-fence-fidelity.yml @@ -0,0 +1,44 @@ +# Fails a pull request when the plain-Markdown copies written by +# scripts/generate-static-md.js (served as .md and linked from llms.txt) +# lose or change code: every fenced code block must keep its lines and their +# relative indentation. Needs no npm install; both scripts use Node built-ins only. +name: Static Markdown fence fidelity + +on: + pull_request: + branches: [testmuCom, stage] + paths: + - 'docs/**' + - 'scripts/generate-static-md.js' + - 'scripts/check-static-md-fences.js' + - '.github/workflows/static-md-fence-fidelity.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: static-md-fence-fidelity-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: | + docs + scripts + package.json + + - uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Generate static Markdown + run: node scripts/generate-static-md.js + + - name: Check fenced code survives unchanged + run: npm run check-static-md-fences diff --git a/docs/kane-cli-testmd-composition.md b/docs/kane-cli-testmd-composition.md index a45b4c192..568270aa9 100644 --- a/docs/kane-cli-testmd-composition.md +++ b/docs/kane-cli-testmd-composition.md @@ -290,7 +290,7 @@ Rules: -```markdown +````markdown ## OK @import ./helpers/login.md @@ -301,9 +301,6 @@ optional: true @import ./helpers/skip-tour.md ## NOT OK — extra config - - - ```yaml timeout: 60 ``` @@ -312,10 +309,7 @@ timeout: 60 ## NOT OK — body mixes prose and import Click somewhere first. @import ./helpers/login.md - - - -``` +```` ## How paths resolve @@ -328,9 +322,6 @@ tests/ helpers/ login.md # contains: @import ./submit-button.md submit-button.md - - - ``` When `checkout_test.md` imports `../../helpers/login.md`, the path is relative to `tests/e2e/`, so it resolves to `helpers/login.md`. When `login.md` imports `./submit-button.md`, the path is relative to `helpers/`, so it resolves to `helpers/submit-button.md`. @@ -381,16 +372,13 @@ A root-level `@import` step can be marked optional in the same way a prose step -```markdown +````markdown ## Skip the tour if it shows up ```yaml optional: true ``` @import ./helpers/dismiss-product-tour.md - - - -``` +```` If the helper fails, the run continues to the next step. The `Result.md` entry is suffixed with `(optional)`. diff --git a/docs/playwright-sdk.md b/docs/playwright-sdk.md index c55d6d663..dc14d8bd8 100644 --- a/docs/playwright-sdk.md +++ b/docs/playwright-sdk.md @@ -382,3 +382,4 @@ You can pass any standard Playwright CLI options directly to this command. For i ```bash npx playwright-node-sdk playwright test tests/my-test.spec.js +``` diff --git a/docs/smartui-appium-hooks.md b/docs/smartui-appium-hooks.md index 735c0fb97..91ba724c5 100644 --- a/docs/smartui-appium-hooks.md +++ b/docs/smartui-appium-hooks.md @@ -819,6 +819,9 @@ config = { 'pageCount': 15 # Enter the number of pages for the Full Page screenshot (Minimum 1, Maximum 20) } driver.execute("smartui.takeScreenshot", config) +``` + + ```ruby diff --git a/docs/smartui-build-merging.md b/docs/smartui-build-merging.md index 1aa72ee88..e22fb1993 100644 --- a/docs/smartui-build-merging.md +++ b/docs/smartui-build-merging.md @@ -187,6 +187,7 @@ npx smartui merge build --source build-123 --target build-456 ```bash # 1. Merge staging build to production npx smartui merge build --source staging-build-123 --target prod-build-456 +``` ### 2. Feature Build Strategy @@ -195,6 +196,7 @@ npx smartui merge build --source staging-build-123 --target prod-build-456 ```bash # 1. Merge feature build into main build npx smartui merge build --source feature-build-789 --target main-build-101 +``` ### 3. Hotfix Build Strategy diff --git a/docs/smartui-cli-env-variables.md b/docs/smartui-cli-env-variables.md index ed365b9e5..9998b1dd2 100644 --- a/docs/smartui-cli-env-variables.md +++ b/docs/smartui-cli-env-variables.md @@ -452,6 +452,9 @@ In case you are accessing your network using corporate proxies, set the proxies ```bash export HTTP_PROXY="http://:@:/" +``` + + ```bash diff --git a/package.json b/package.json index e163e28e4..42aa1a3e5 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "start": "docusaurus start", "prebuild": "node scripts/build-api-data.js && node scripts/generate-api-pages.js && node scripts/generate-static-md.js && node scripts/generate-llms-txt.js && node scripts/generate-skill-index.js", "generate-static-md": "node scripts/generate-static-md.js", + "check-static-md-fences": "node scripts/check-static-md-fences.js", "generate-llms-txt": "node scripts/generate-llms-txt.js", "generate-skill-index": "node scripts/generate-skill-index.js", "sitemap-exclusions": "node scripts/sitemap-exclusions.js", diff --git a/scripts/check-static-md-fences.js b/scripts/check-static-md-fences.js new file mode 100644 index 000000000..78c8c4266 --- /dev/null +++ b/scripts/check-static-md-fences.js @@ -0,0 +1,188 @@ +/** + * Fence-fidelity check for the Markdown copies written by generate-static-md.js. + * + * Every fenced code block in a source doc must come out of the generator with + * the same code: the same non-blank lines, in the same order, with the same + * relative indentation. Only the fence's own indentation may be removed (the + * CommonMark rule for a fence nested in a list or JSX). YAML and Python change + * meaning when inner indentation is lost, and AI agents read these files + * through llms.txt. + * + * Run after the generator: + * node scripts/generate-static-md.js && node scripts/check-static-md-fences.js + * + * Errors (exit 1): + * - a code line whose text or indentation differs from the source; + * - a source fence that is never closed, or that another opening fence + * interrupts. Every later fence in that file then pairs the wrong way + * round, on the site and in the generated copy alike. + * Warnings (exit 1 only with --strict): + * - a source code block missing from the generated copy (for example when a + * tag-stripping step removes a whole region of the page); + * - a slug produced by more than one doc, which is then not checked. + */ + +const fs = require('fs'); +const path = require('path'); + +const DOCS_DIR = path.join(__dirname, '..', 'docs'); +const OUT_DIR = path.join(__dirname, '..', 'static', 'docs'); +const STRICT = process.argv.includes('--strict'); + +/** Same shallow frontmatter read as generate-static-md.js, plus where the body starts. */ +function parseFrontmatter(raw) { + const match = raw.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/); + if (!match) return { data: {}, body: raw, bodyLine: 0 }; + const data = {}; + for (const line of match[1].split(/\r?\n/)) { + const kv = line.match(/^([A-Za-z0-9_-]+):\s*(.*)$/); + if (!kv) continue; + let value = kv[2].trim(); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + data[kv[1]] = value; + } + const bodyLine = (match[0].match(/\n/g) || []).length; + return { data, body: raw.slice(match[0].length), bodyLine }; +} + +/** Same slug rule as generate-static-md.js. */ +function resolveSlug(data, fileName) { + const fileBase = fileName.replace(/\.mdx?$/, ''); + const base = data.slug || data.id || fileBase; + const name = base.replace(/^\//, '').replace(/\/$/, '').split('/').pop(); + return name || fileBase; +} + +/** + * Fenced blocks of a Markdown text, line based. Each block keeps the fence's + * indentation, its 1-based start line and its content lines. `firstLine` is + * the number of lines that precede `text` in its file. + */ +function fences(text, firstLine = 0) { + const lines = text.split(/\r?\n/); + const blocks = []; + const unclosed = []; + let open = null; + lines.forEach((line, i) => { + const number = firstLine + i + 1; + if (!open) { + // An info string cannot contain a backtick, so "```status``` | x" is inline code. + const m = line.match(/^([ \t]*)(`{3,})[^`]*$/); + if (m) open = { indent: m[1].length, marker: m[2], start: number, lines: [] }; + return; + } + // Lenient close: "``` -->" (a fence inside an HTML comment) still closes. + const close = line.match(/^[ \t]*(`{3,})(?![`\w])/); + if (close && close[1].length >= open.marker.length) { + blocks.push(open); + open = null; + return; + } + const inner = line.match(/^[ \t]*(`{3,})[ \t]*[A-Za-z][\w+#.-]*[ \t]*$/); + if (inner && inner[1].length >= open.marker.length) { + unclosed.push({ start: open.start, next: number }); + } + open.lines.push({ text: line, number }); + }); + if (open) unclosed.push({ start: open.start, next: null }); + return { blocks, unclosed }; +} + +/** Non-blank code lines with trailing whitespace removed (the generator trims both). */ +const codeLines = (block) => + block.lines + .map((l) => ({ ...l, text: l.text.replace(/[ \t]+$/, '') })) + .filter((l) => l.text.trim() !== ''); + +/** A source line as the rendered site shows it: at most the fence indent removed. */ +const outdent = (text, width) => text.replace(new RegExp(`^[ \\t]{0,${width}}`), ''); + +const signature = (lines) => lines.map((l) => l.text.replace(/\s+/g, '')).join('\n'); + +function main() { + const errors = []; + const warnings = []; + let blocksChecked = 0; + + // Two docs with one slug overwrite each other's output in directory order, + // which is not stable across file systems, so those slugs are not checked. + const bySlug = new Map(); + const duplicated = new Set(); + const sources = fs + .readdirSync(DOCS_DIR) + .filter((f) => /\.mdx?$/.test(f)) + .sort(); + for (const file of sources) { + const raw = fs.readFileSync(path.join(DOCS_DIR, file), 'utf8').replace(/^\uFEFF/, ''); + const { data, body, bodyLine } = parseFrontmatter(raw); + if (String(data.draft).toLowerCase() === 'true') continue; + const slug = resolveSlug(data, file); + if (bySlug.has(slug)) duplicated.add(slug); + bySlug.set(slug, { file, body, bodyLine }); + } + for (const slug of duplicated) { + warnings.push(`slug "${slug}" is produced by more than one doc; not checked`); + bySlug.delete(slug); + } + + for (const [slug, { file, body, bodyLine }] of bySlug) { + const outPath = path.join(OUT_DIR, `${slug}.md`); + if (!fs.existsSync(outPath)) { + errors.push(`docs/${file}: no generated file static/docs/${slug}.md`); + continue; + } + + const source = fences(body, bodyLine); + if (source.unclosed.length) { + // Comparing lines here would only repeat this one defect many times. + for (const u of source.unclosed) { + errors.push( + `docs/${file} line ${u.start}: code fence is not closed` + + (u.next ? ` before the fence at line ${u.next}` : ' before the end of the file') + ); + } + continue; + } + + const outBlocks = fences(fs.readFileSync(outPath, 'utf8')).blocks.map((b) => ({ + lines: codeLines(b), + used: false, + })); + + for (const src of source.blocks) { + const expected = codeLines(src).map((l) => ({ ...l, text: outdent(l.text, src.indent) })); + if (!expected.length) continue; + const sig = signature(expected); + const match = outBlocks.find((b) => !b.used && signature(b.lines) === sig); + if (!match) { + warnings.push(`docs/${file} line ${src.start}: code block missing from static/docs/${slug}.md`); + continue; + } + match.used = true; + blocksChecked++; + expected.forEach((line, i) => { + const got = match.lines[i]; + if (got.text !== line.text) { + errors.push( + `docs/${file} line ${line.number} -> static/docs/${slug}.md line ${got.number}: ` + + `expected ${JSON.stringify(line.text)}, got ${JSON.stringify(got.text)}` + ); + } + }); + } + } + + for (const w of warnings) console.warn(`warning: ${w}`); + for (const e of errors) console.error(`error: ${e}`); + console.log( + `Checked ${blocksChecked} code block(s) in ${bySlug.size} doc(s): ${errors.length} error(s), ${warnings.length} warning(s).` + ); + if (errors.length || (STRICT && warnings.length)) process.exit(1); +} + +main(); diff --git a/scripts/generate-static-md.js b/scripts/generate-static-md.js index dee06b8be..99386530e 100644 --- a/scripts/generate-static-md.js +++ b/scripts/generate-static-md.js @@ -75,8 +75,12 @@ function toPlainMarkdown(body) { // 1. Shield fenced code blocks (with any leading indent) so later transforms // never touch their content. const codeBlocks = []; - body = body.replace(/^[ \t]*```[\s\S]*?```/gm, (block) => { - codeBlocks.push(block.replace(/^[ \t]+/gm, '')); // de-indent nested fences + body = body.replace(/^([ \t]*)```[\s\S]*?```/gm, (block, fenceIndent) => { + // De-indent nested fences the CommonMark way: remove at most the fence's + // own indentation from each line, so indentation inside the code (YAML, + // Python) is kept. + const outdent = new RegExp(`^[ \\t]{0,${fenceIndent.length}}`, 'gm'); + codeBlocks.push(block.replace(outdent, '')); return `\u0000CODE${codeBlocks.length - 1}\u0000`; });