diff --git a/docs/tasks/P002-sitemap-robots-integrity.md b/docs/tasks/P002-sitemap-robots-integrity.md new file mode 100644 index 0000000..1209c62 --- /dev/null +++ b/docs/tasks/P002-sitemap-robots-integrity.md @@ -0,0 +1,32 @@ +# P002 — Sitemap and robots crawl-discovery integrity + +## Goal +Keep sitemap.xml and robots.txt consistent with published GitHub Pages URLs and prevent misleading per-page modification timestamps. + +## Diagnosis (2026-10-26) +- Search Console showed "Site haritası okunamadı" with 0 discovered pages. +- Repository main contained a valid-looking XML sitemap with 49 URL entries, matching all 49 published index.html paths in the Git tree. +- robots.txt advertised the correct HTTPS sitemap and allowed crawling. +- Both offline documentation generators rewrote every sitemap `lastmod` to the generator execution date, regardless of which pages changed. This is not a verifiable per-page modification date. +- Search Console's fetch failure cannot be conclusively reproduced from the repository; public HTTP responses and Google's fetch diagnostics must be checked separately. + +## Scope +- Preserve all 49 canonical URLs and the existing crawler access policy. +- Omit optional `lastmod` unless a future generator can derive an accurate per-URL significant-content modification date. +- Update both offline docs generators so they do not recreate inaccurate `lastmod`. +- Extend the existing dependency-free `scripts/verify-site.mjs` CI verifier to check sitemap XML envelope/entry structure, complete and unique canonical coverage, HTML canonical/noindex consistency, and the robots sitemap declaration. + +## Acceptance criteria and verification +- Exactly one sitemap URL per public HTML page (root + both documentation trees). +- No stale URLs, duplicates, malformed sitemap entries or cross-origin URLs. +- robots.txt continues to allow general crawling and advertises exactly one canonical sitemap. +- `node scripts/verify-site.mjs` and `git diff --check` pass on the PR head. +- The required GitHub Actions `docs-integrity` check passes before squash merge to protected `main`. +- Verify PR merged and `main` contains the change. Preserve local uncommitted work. + +## Evidence / operational follow-up +- Baseline repository tree inspection: 49 public HTML pages and 49 matching sitemap locations, with 49 entries dated 2026-10-02. +- Root cause of the Search Console fetch failure remains unconfirmed until the production endpoint responds with HTTP 200/XML from an external network and Google Search Console's live fetch diagnosis is available. +- After deployment, confirm `https://cynrath.github.io/sitemap.xml` returns HTTP 200 with XML, then resubmit `sitemap.xml` to Google Search Console and verify the report updates. +- Google guidance: https://developers.google.com/search/docs/crawling-indexing/sitemaps/build-sitemap +- Google troubleshooting: https://support.google.com/webmasters/answer/7451001 diff --git a/scripts/sync-ackit-docs.mjs b/scripts/sync-ackit-docs.mjs index f1b6806..3294ac3 100644 --- a/scripts/sync-ackit-docs.mjs +++ b/scripts/sync-ackit-docs.mjs @@ -200,13 +200,13 @@ async function main() { // Merge sitemap: keep existing bridge URLs verbatim (owned by // scripts/sync-ackit-spec-kit-bridge-docs.mjs) so generator order never // drops product pages. Sorted+unique for deterministic output. - const today = new Date().toISOString().slice(0,10); + // Omit lastmod without a verifiable per-page modification timestamp. const existingRaw = await readOptional(path.join(siteRoot,'sitemap.xml')); const existingLocs = existingRaw ? [...existingRaw.matchAll(/([^<]+)<\/loc>/g)].map(m => m[1]) : []; const bridgeLocs = existingLocs.filter(u => u === `${SITE}/ackit-spec-kit-bridge/` || u.startsWith(`${SITE}/ackit-spec-kit-bridge/`)); const urls = [...new Set([`${SITE}/`, ...list.map(p => `${SITE}/agent-context-kit/${p.slug ? `${p.slug}/` : ''}`), ...bridgeLocs])].sort(); let sitemap = '\n\n'; - for (const url of urls) sitemap += ` \n ${url}\n ${today}\n \n`; + for (const url of urls) sitemap += ` \n ${url}\n \n`; sitemap += '\n'; await write(path.join(siteRoot,'sitemap.xml'), sitemap); diff --git a/scripts/sync-ackit-spec-kit-bridge-docs.mjs b/scripts/sync-ackit-spec-kit-bridge-docs.mjs index ce9b9ad..eeceb84 100644 --- a/scripts/sync-ackit-spec-kit-bridge-docs.mjs +++ b/scripts/sync-ackit-spec-kit-bridge-docs.mjs @@ -189,13 +189,13 @@ async function main() { // Merge sitemap: keep existing non-bridge URLs verbatim (root, ACKit), // replace the bridge set with the freshly generated one. Sorted+unique // so output is deterministic regardless of generator order. - const today = new Date().toISOString().slice(0,10); + // Omit lastmod without a verifiable per-page modification timestamp. const existing = readSitemapLocs(await readOptional(path.join(siteRoot,'sitemap.xml'))); const keep = existing.filter(u => u !== `${SITE}/ackit-spec-kit-bridge/` && !u.startsWith(`${SITE}/ackit-spec-kit-bridge/`)); const bridgeUrls = list.map(p => `${SITE}/ackit-spec-kit-bridge/${p.slug ? `${p.slug}/` : ''}`); const urls = [...new Set([...keep, ...bridgeUrls])].sort(); let sitemap = '\n\n'; - for (const url of urls) sitemap += ` \n ${url}\n ${today}\n \n`; + for (const url of urls) sitemap += ` \n ${url}\n \n`; sitemap += '\n'; await write(path.join(siteRoot,'sitemap.xml'), sitemap); diff --git a/scripts/verify-site.mjs b/scripts/verify-site.mjs index dd708ed..d19464c 100644 --- a/scripts/verify-site.mjs +++ b/scripts/verify-site.mjs @@ -484,6 +484,71 @@ async function main() { fail("ACKit generator does not preserve bridge sitemap URLs"); } + + // 15. Crawl discovery: sitemap structure and canonical URL coverage, plus robots. + const seoXml = await fsp.readFile(path.join(siteRoot, "sitemap.xml"), "utf8"); + const envelope = seoXml.match( + /^<\?xml version="1\.0" encoding="UTF-8"\?>\s*\s*([\s\S]*?)\s*<\/urlset>\s*$/, + ); + if (!envelope) { + fail("SEO: sitemap XML declaration or urlset structure is invalid"); + } else { + const entryPattern = /\s*([^<]+)<\/loc>\s*(?:(\d{4}-\d{2}-\d{2})<\/lastmod>\s*)?<\/url>/g; + const entries = [...envelope[1].matchAll(entryPattern)]; + if (envelope[1].replace(entryPattern, "").trim() !== "") { + fail("SEO: sitemap contains malformed or unsupported XML entries"); + } + const found = new Set(); + const expected = new Set([ + SITE + "/", + ...[...pages, ...bridgePages].map((page) => SITE + "/" + posix(page).replace(/index\.html$/, "")), + ]); + for (const [, loc, lastmod] of entries) { + if (found.has(loc)) fail("SEO: duplicate sitemap URL " + loc); + found.add(loc); + try { + const parsed = new URL(loc); + if (parsed.origin !== SITE || parsed.href !== loc || !parsed.pathname.endsWith("/")) { + fail("SEO: invalid or non-canonical sitemap URL " + loc); + } + } catch { + fail("SEO: malformed sitemap URL " + loc); + } + if (lastmod && (!Number.isFinite(Date.parse(lastmod)) || lastmod > new Date().toISOString().slice(0, 10))) { + fail("SEO: invalid or future lastmod on " + loc); + } + if (!expected.has(loc)) fail("SEO: unexpected or stale sitemap URL " + loc); + } + for (const loc of expected) { + if (!found.has(loc)) fail("SEO: published HTML page missing from sitemap " + loc); + } + for (const page of [path.join(siteRoot, "index.html"), ...pages, ...bridgePages]) { + const html = await fsp.readFile(page, "utf8"); + const canonical = html.match(//); + const expectedCanonical = page === path.join(siteRoot, "index.html") + ? SITE + "/" + : SITE + "/" + posix(page).replace(/index\.html$/, ""); + if (!canonical || canonical[1] !== expectedCanonical) { + fail("SEO: canonical mismatch on " + posix(page) + " (expected " + expectedCanonical + ")"); + } + if (/ 0) { for (const failure of failures) process.stdout.write(`FAIL ${failure}\n`); process.stdout.write(`docs-integrity: ${failures.length} failure(s)\n`); diff --git a/sitemap.xml b/sitemap.xml index bf9ee3f..7c8876a 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -2,198 +2,149 @@ https://cynrath.github.io/ - 2026-10-02 https://cynrath.github.io/ackit-spec-kit-bridge/ - 2026-10-02 https://cynrath.github.io/ackit-spec-kit-bridge/architecture/ - 2026-10-02 https://cynrath.github.io/ackit-spec-kit-bridge/cli/ - 2026-10-02 https://cynrath.github.io/ackit-spec-kit-bridge/compatibility/ - 2026-10-02 https://cynrath.github.io/ackit-spec-kit-bridge/configuration/ - 2026-10-02 https://cynrath.github.io/ackit-spec-kit-bridge/getting-started/ - 2026-10-02 https://cynrath.github.io/ackit-spec-kit-bridge/lifecycle/ - 2026-10-02 https://cynrath.github.io/ackit-spec-kit-bridge/profiles/ - 2026-10-02 https://cynrath.github.io/ackit-spec-kit-bridge/release/ - 2026-10-02 https://cynrath.github.io/ackit-spec-kit-bridge/security/ - 2026-10-02 https://cynrath.github.io/ackit-spec-kit-bridge/spec-kit-extension/ - 2026-10-02 https://cynrath.github.io/ackit-spec-kit-bridge/troubleshooting/ - 2026-10-02 https://cynrath.github.io/ackit-spec-kit-bridge/trust-flow/ - 2026-10-02 https://cynrath.github.io/ackit-spec-kit-bridge/verification/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/agents-md-validator/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/ai-agent-readiness/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/benchmarks/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/checkpoints/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/cli/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/context-optimization/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/dashboard/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/diagnostics/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/drift/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/evidence/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/getting-started/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/github-action/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/instruction-graph/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/intent/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/mcp-repository-tools/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/mcp/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/migration/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/optimize/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/policy/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/profiles/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/provider-surfaces/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/readiness/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/roles/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/rule-packs/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/sdk/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/security/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/skills/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/status/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/sync/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/trust-flow/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/verification/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/vscode/ - 2026-10-02 https://cynrath.github.io/agent-context-kit/workflows/ - 2026-10-02