From cab3df783cdf9c73cf91e5994d3b5d6bf8cfad28 Mon Sep 17 00:00:00 2001 From: Ben Sabic Date: Fri, 29 May 2026 16:50:02 +1000 Subject: [PATCH 1/6] feat(docs): suffix page and OG titles with "| Chat SDK" Add a title template in the root layout so every page's and og:title render as "<Page> | Chat SDK", with explicit og:title strings on pages that set their own openGraph. The home page stays "Chat SDK". Also registers a <link rel="llms-txt"> on every page. --- apps/docs/app/[lang]/(home)/page.tsx | 5 ++++- apps/docs/app/[lang]/(home)/resources/page.tsx | 3 +++ .../docs/app/[lang]/adapters/(listing)/page.tsx | 3 +++ apps/docs/app/[lang]/layout.tsx | 17 +++++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/apps/docs/app/[lang]/(home)/page.tsx b/apps/docs/app/[lang]/(home)/page.tsx index 2e01fce5..644d8aaa 100644 --- a/apps/docs/app/[lang]/(home)/page.tsx +++ b/apps/docs/app/[lang]/(home)/page.tsx @@ -28,8 +28,11 @@ const heroDescription = "A unified TypeScript SDK for building chat bots with type-safe handlers, JSX cards, and multi-platform support—powered by Vercel"; export const metadata: Metadata = { - title: metadataTitle, + title: { absolute: metadataTitle }, description: heroDescription, + openGraph: { + title: { absolute: metadataTitle }, + }, twitter: { card: "summary_large_image", }, diff --git a/apps/docs/app/[lang]/(home)/resources/page.tsx b/apps/docs/app/[lang]/(home)/resources/page.tsx index de3034e2..ad380cff 100644 --- a/apps/docs/app/[lang]/(home)/resources/page.tsx +++ b/apps/docs/app/[lang]/(home)/resources/page.tsx @@ -17,6 +17,9 @@ export const metadata: Metadata = { "ai agent", "vercel", ], + openGraph: { + title: "Resources", + }, twitter: { card: "summary_large_image", }, diff --git a/apps/docs/app/[lang]/adapters/(listing)/page.tsx b/apps/docs/app/[lang]/adapters/(listing)/page.tsx index ee8f88ee..25caf1a8 100644 --- a/apps/docs/app/[lang]/adapters/(listing)/page.tsx +++ b/apps/docs/app/[lang]/adapters/(listing)/page.tsx @@ -6,6 +6,9 @@ export const metadata: Metadata = { title: "Adapters", description: "Browse official and community adapters for Chat SDK. Connect your bot to Slack, Teams, Discord, and more.", + openGraph: { + title: "Adapters", + }, twitter: { card: "summary_large_image", }, diff --git a/apps/docs/app/[lang]/layout.tsx b/apps/docs/app/[lang]/layout.tsx index 83147660..5b5e065e 100644 --- a/apps/docs/app/[lang]/layout.tsx +++ b/apps/docs/app/[lang]/layout.tsx @@ -1,4 +1,5 @@ import "../global.css"; +import type { Metadata } from "next"; import { Footer } from "@/components/geistdocs/footer"; import { Navbar } from "@/components/geistdocs/navbar"; import { GeistdocsProvider } from "@/components/geistdocs/provider"; @@ -6,6 +7,19 @@ import { basePath } from "@/geistdocs"; import { mono, sans } from "@/lib/geistdocs/fonts"; import { cn } from "@/lib/utils"; +export const metadata: Metadata = { + title: { + template: "%s | Chat SDK", + default: "Chat SDK", + }, + openGraph: { + title: { + template: "%s | Chat SDK", + default: "Chat SDK", + }, + }, +}; + const Layout = async ({ children, params }: LayoutProps<"/[lang]">) => { const { lang } = await params; @@ -15,6 +29,9 @@ const Layout = async ({ children, params }: LayoutProps<"/[lang]">) => { lang={lang} suppressHydrationWarning > + <head> + <link href="/llms.txt" rel="llms-txt" /> + </head> <body> <GeistdocsProvider basePath={basePath} lang={lang}> <Navbar /> From c88b82597b019a84799612a84054802f5a2cd8c0 Mon Sep 17 00:00:00 2001 From: Ben Sabic <bensabic@users.noreply.github.com> Date: Fri, 29 May 2026 16:50:34 +1000 Subject: [PATCH 2/6] feat(docs): serve markdown versions of docs & adapter pages Surface a plain-markdown URL for every docs and adapter page via an sr-only, aria-hidden link (for AI/LLM crawlers) plus a text/markdown alternate. - Add an /adapters/<group>/<slug>.md endpoint (new adapters.mdx route + proxy rewrite) that serves the adapter's README or MDX body. - Extract shared README resolution into lib/geistdocs/adapter-readme.ts, deduped from the community and vendor-official pages, and add getAdapterLLMText. - Fix the docs markdown alternate to join nested slugs with "/". --- .../[lang]/adapters.mdx/[[...slug]]/route.ts | 43 ++++++ .../(detail)/community/[slug]/page.tsx | 127 ++++-------------- .../(detail)/official/[slug]/page.tsx | 17 +++ .../(detail)/vendor-official/[slug]/page.tsx | 113 ++++------------ .../docs/app/[lang]/docs/[[...slug]]/page.tsx | 9 +- apps/docs/lib/geistdocs/adapter-readme.ts | 121 +++++++++++++++++ apps/docs/lib/geistdocs/adapters-source.ts | 26 ++++ apps/docs/proxy.ts | 33 ++++- 8 files changed, 292 insertions(+), 197 deletions(-) create mode 100644 apps/docs/app/[lang]/adapters.mdx/[[...slug]]/route.ts create mode 100644 apps/docs/lib/geistdocs/adapter-readme.ts diff --git a/apps/docs/app/[lang]/adapters.mdx/[[...slug]]/route.ts b/apps/docs/app/[lang]/adapters.mdx/[[...slug]]/route.ts new file mode 100644 index 00000000..0780e45f --- /dev/null +++ b/apps/docs/app/[lang]/adapters.mdx/[[...slug]]/route.ts @@ -0,0 +1,43 @@ +import { notFound } from "next/navigation"; +import { getAdapter, getReadme } from "@/lib/geistdocs/adapter-readme"; +import { + adaptersSource, + getAdapterLLMText, +} from "@/lib/geistdocs/adapters-source"; + +export const revalidate = false; + +export async function GET( + _req: Request, + { params }: RouteContext<"/[lang]/adapters.mdx/[[...slug]]"> +) { + const { slug, lang } = await params; + const page = adaptersSource.getPage(slug, lang); + + if (!page) { + notFound(); + } + + // Official adapters render their MDX body directly. Community and + // vendor-official adapters render an upstream README (unless they opt into an + // MDX body), so serve that same content in the markdown version. + const type = slug?.[0]; + let body: string | undefined; + if (type !== "official") { + const data = page.data as unknown as { mdxBody?: boolean; slug: string }; + if (data.mdxBody !== true) { + const adapter = getAdapter(data.slug); + if (adapter) { + body = await getReadme(adapter); + } + } + } + + return new Response(await getAdapterLLMText(page, body), { + headers: { + "Content-Type": "text/markdown", + }, + }); +} + +export const generateStaticParams = () => adaptersSource.generateParams(); diff --git a/apps/docs/app/[lang]/adapters/(detail)/community/[slug]/page.tsx b/apps/docs/app/[lang]/adapters/(detail)/community/[slug]/page.tsx index 41196dfb..69bfe2f1 100644 --- a/apps/docs/app/[lang]/adapters/(detail)/community/[slug]/page.tsx +++ b/apps/docs/app/[lang]/adapters/(detail)/community/[slug]/page.tsx @@ -1,16 +1,19 @@ -import { readFile } from "node:fs/promises"; -import { join } from "node:path"; import { createRelativeLink } from "fumadocs-ui/mdx"; import type { Metadata } from "next"; import { notFound } from "next/navigation"; import type { ComponentProps, FC } from "react"; -import adaptersJson from "@/adapters.json"; import { AdapterHero } from "@/components/geistdocs/adapter-hero"; import { DocsBody, DocsPage } from "@/components/geistdocs/docs-page"; import { FeatureSupport } from "@/components/geistdocs/feature-support"; import { getMDXComponents } from "@/components/geistdocs/mdx-components"; import { Upsell } from "@/components/geistdocs/upsell"; import type { AdapterFeatureValue } from "@/lib/adapter-features"; +import { + type Adapter, + getAdapter, + getIssuesUrl, + getReadme, +} from "@/lib/geistdocs/adapter-readme"; import { adaptersSource } from "@/lib/geistdocs/adapters-source"; import { ReadmeContent } from "../../../components/readme-content"; @@ -33,105 +36,6 @@ const wrapWithNofollow = (BaseLink: FC<MdxLinkProps>): FC<MdxLinkProps> => { return NofollowExternalLink; }; -const LOCAL_PACKAGE_PATTERN = /github\.com\/vercel\/chat\/tree\/[^/]+\/(.+)/; -const GITHUB_SUBPATH_PATTERN = - /github\.com\/([^/]+)\/([^/]+)\/tree\/([^/]+)\/(.+)/; -const GITHUB_REPO_REF_PATTERN = - /^https:\/\/github\.com\/([^/]+)\/([^/]+)\/tree\/([^/]+)\/?$/; -const GITHUB_REPO_PATTERN = /github\.com\/([^/]+)\/([^/]+)/; -const GITHUB_REPO_ROOT_PATTERN = /^(https:\/\/github\.com\/[^/]+\/[^/]+)/; - -const UNPINNED_REF_PATTERN = /^(main|master|head|dev|develop|trunk|default)$/i; - -const MAX_README_BYTES = 500_000; - -type Adapter = (typeof adaptersJson)[number]; - -const getAdapter = (slug: string): Adapter | undefined => - adaptersJson.find((a) => a.slug === slug); - -const getIssuesUrl = (readmeUrl: string | undefined): string | undefined => { - if (!readmeUrl) { - return; - } - const match = readmeUrl.match(GITHUB_REPO_ROOT_PATTERN); - return match ? `${match[1]}/issues` : undefined; -}; - -const warnUnpinned = (adapter: Adapter, ref: string | undefined) => { - if (ref && !UNPINNED_REF_PATTERN.test(ref)) { - return; - } - console.warn( - `[adapters] Community adapter "${adapter.name}" uses an unpinned README ref "${ - ref ?? "<default branch>" - }". Pin to a commit SHA or tag in adapters.json to freeze content at review time.` - ); -}; - -const truncate = (content: string): string => - content.length <= MAX_README_BYTES - ? content - : `${content.slice(0, MAX_README_BYTES)}\n\n> _README truncated — view the full version on GitHub._`; - -const fetchGitHubReadme = async (url: string): Promise<string | undefined> => { - const response = await fetch(url, { - headers: { Accept: "application/vnd.github.raw+json" }, - next: { revalidate: 3600 }, - }); - if (response.ok) { - return response.text(); - } -}; - -const getReadme = async (adapter: Adapter): Promise<string | undefined> => { - if (!adapter.readme) { - return; - } - const repoUrl = adapter.readme; - - const localMatch = repoUrl.match(LOCAL_PACKAGE_PATTERN); - if (localMatch) { - const [, pkgPath] = localMatch; - const filePath = join(process.cwd(), "..", "..", pkgPath, "README.md"); - try { - return truncate(await readFile(filePath, "utf-8")); - } catch { - return; - } - } - - const subpathMatch = repoUrl.match(GITHUB_SUBPATH_PATTERN); - if (subpathMatch) { - const [, owner, repo, ref, path] = subpathMatch; - warnUnpinned(adapter, ref); - const content = await fetchGitHubReadme( - `https://api.github.com/repos/${owner}/${repo}/readme/${path}?ref=${ref}` - ); - return content ? truncate(content) : undefined; - } - - const repoRefMatch = repoUrl.match(GITHUB_REPO_REF_PATTERN); - if (repoRefMatch) { - const [, owner, repo, ref] = repoRefMatch; - warnUnpinned(adapter, ref); - const content = await fetchGitHubReadme( - `https://api.github.com/repos/${owner}/${repo}/readme?ref=${ref}` - ); - return content ? truncate(content) : undefined; - } - - const repoMatch = repoUrl.match(GITHUB_REPO_PATTERN); - if (repoMatch) { - const [, owner, repo] = repoMatch; - warnUnpinned(adapter, undefined); - const content = await fetchGitHubReadme( - `https://api.github.com/repos/${owner}/${repo}/readme` - ); - return content ? truncate(content) : undefined; - } -}; - const CommunityNotice = ({ adapter }: { adapter: Adapter }) => { const issuesUrl = getIssuesUrl(adapter.readme); @@ -194,10 +98,11 @@ const Page = async ({ params }: { params: Promise<PageParams> }) => { const data = page.data as unknown as AdapterFrontmatter; const adapter = getAdapter(slug); + const markdownPath = `/adapters/community/${slug}.md`; const useMdxBody = data.mdxBody === true; let readme: string | undefined; if (!useMdxBody && adapter) { - readme = await getReadme(adapter); + readme = await getReadme(adapter, { warnOnUnpinnedRef: true }); } const MDX = page.data.body; const BoundFeatureSupport = renderBoundFeatureSupport( @@ -253,6 +158,16 @@ const Page = async ({ params }: { params: Promise<PageParams> }) => { toc={page.data.toc} > <DocsBody> + {/* biome-ignore lint/a11y/useAnchorContent: intentionally aria-hidden hint surfacing the markdown URL for AI/LLM crawlers, not for screen readers */} + <a + aria-hidden="true" + className="sr-only" + href={markdownPath} + tabIndex={-1} + > + AI/LLM: This adapter page is available in plain markdown format at + {markdownPath} + </a> <AdapterHero community logo={data.logo} @@ -289,11 +204,17 @@ export const generateMetadata = async ({ title: page.data.title, description: page.data.description, openGraph: { + title: page.data.title, images: `/${lang}/adapters/community/${slug}/og`, }, twitter: { card: "summary_large_image", }, + alternates: { + types: { + "text/markdown": `/adapters/community/${slug}.md`, + }, + }, }; }; diff --git a/apps/docs/app/[lang]/adapters/(detail)/official/[slug]/page.tsx b/apps/docs/app/[lang]/adapters/(detail)/official/[slug]/page.tsx index a194ef17..2aa1d3a6 100644 --- a/apps/docs/app/[lang]/adapters/(detail)/official/[slug]/page.tsx +++ b/apps/docs/app/[lang]/adapters/(detail)/official/[slug]/page.tsx @@ -45,6 +45,7 @@ const Page = async ({ params }: { params: Promise<PageParams> }) => { const data = page.data as unknown as AdapterFrontmatter; const MDX = page.data.body; + const markdownPath = `/adapters/official/${slug}.md`; const BoundFeatureSupport = renderBoundFeatureSupport( data.features, data.type @@ -64,6 +65,16 @@ const Page = async ({ params }: { params: Promise<PageParams> }) => { toc={page.data.toc} > <DocsBody> + {/* biome-ignore lint/a11y/useAnchorContent: intentionally aria-hidden hint surfacing the markdown URL for AI/LLM crawlers, not for screen readers */} + <a + aria-hidden="true" + className="sr-only" + href={markdownPath} + tabIndex={-1} + > + AI/LLM: This adapter page is available in plain markdown format at + {markdownPath} + </a> <AdapterHero beta={data.beta} logo={data.logo} @@ -104,11 +115,17 @@ export const generateMetadata = async ({ title: page.data.title, description: page.data.description, openGraph: { + title: page.data.title, images: `/${lang}/adapters/official/${slug}/og`, }, twitter: { card: "summary_large_image", }, + alternates: { + types: { + "text/markdown": `/adapters/official/${slug}.md`, + }, + }, }; }; diff --git a/apps/docs/app/[lang]/adapters/(detail)/vendor-official/[slug]/page.tsx b/apps/docs/app/[lang]/adapters/(detail)/vendor-official/[slug]/page.tsx index 460e872c..26143624 100644 --- a/apps/docs/app/[lang]/adapters/(detail)/vendor-official/[slug]/page.tsx +++ b/apps/docs/app/[lang]/adapters/(detail)/vendor-official/[slug]/page.tsx @@ -1,104 +1,22 @@ -import { readFile } from "node:fs/promises"; -import { join } from "node:path"; import { createRelativeLink } from "fumadocs-ui/mdx"; import type { Metadata } from "next"; import { notFound } from "next/navigation"; -import adaptersJson from "@/adapters.json"; import { AdapterHero } from "@/components/geistdocs/adapter-hero"; import { DocsBody, DocsPage } from "@/components/geistdocs/docs-page"; import { FeatureSupport } from "@/components/geistdocs/feature-support"; import { getMDXComponents } from "@/components/geistdocs/mdx-components"; import { Upsell } from "@/components/geistdocs/upsell"; import type { AdapterFeatureValue } from "@/lib/adapter-features"; +import { + type Adapter, + getAdapter, + getAuthor, + getIssuesUrl, + getReadme, +} from "@/lib/geistdocs/adapter-readme"; import { adaptersSource } from "@/lib/geistdocs/adapters-source"; import { ReadmeContent } from "../../../components/readme-content"; -const LOCAL_PACKAGE_PATTERN = /github\.com\/vercel\/chat\/tree\/[^/]+\/(.+)/; -const GITHUB_SUBPATH_PATTERN = - /github\.com\/([^/]+)\/([^/]+)\/tree\/([^/]+)\/(.+)/; -const GITHUB_REPO_REF_PATTERN = - /^https:\/\/github\.com\/([^/]+)\/([^/]+)\/tree\/([^/]+)\/?$/; -const GITHUB_REPO_PATTERN = /github\.com\/([^/]+)\/([^/]+)/; -const GITHUB_REPO_ROOT_PATTERN = /^(https:\/\/github\.com\/[^/]+\/[^/]+)/; - -const MAX_README_BYTES = 500_000; - -type Adapter = (typeof adaptersJson)[number]; - -const getAdapter = (slug: string): Adapter | undefined => - adaptersJson.find((a) => a.slug === slug); - -const getAuthor = (adapter: Adapter): string | undefined => - "author" in adapter ? adapter.author : undefined; - -const getIssuesUrl = (readmeUrl: string | undefined): string | undefined => { - if (!readmeUrl) { - return; - } - const match = readmeUrl.match(GITHUB_REPO_ROOT_PATTERN); - return match ? `${match[1]}/issues` : undefined; -}; - -const truncate = (content: string): string => - content.length <= MAX_README_BYTES - ? content - : `${content.slice(0, MAX_README_BYTES)}\n\n> _README truncated — view the full version on GitHub._`; - -const fetchGitHubReadme = async (url: string): Promise<string | undefined> => { - const response = await fetch(url, { - headers: { Accept: "application/vnd.github.raw+json" }, - next: { revalidate: 3600 }, - }); - if (response.ok) { - return response.text(); - } -}; - -const getReadme = async (adapter: Adapter): Promise<string | undefined> => { - if (!adapter.readme) { - return; - } - const repoUrl = adapter.readme; - - const localMatch = repoUrl.match(LOCAL_PACKAGE_PATTERN); - if (localMatch) { - const [, pkgPath] = localMatch; - const filePath = join(process.cwd(), "..", "..", pkgPath, "README.md"); - try { - return truncate(await readFile(filePath, "utf-8")); - } catch { - return; - } - } - - const subpathMatch = repoUrl.match(GITHUB_SUBPATH_PATTERN); - if (subpathMatch) { - const [, owner, repo, ref, path] = subpathMatch; - const content = await fetchGitHubReadme( - `https://api.github.com/repos/${owner}/${repo}/readme/${path}?ref=${ref}` - ); - return content ? truncate(content) : undefined; - } - - const repoRefMatch = repoUrl.match(GITHUB_REPO_REF_PATTERN); - if (repoRefMatch) { - const [, owner, repo, ref] = repoRefMatch; - const content = await fetchGitHubReadme( - `https://api.github.com/repos/${owner}/${repo}/readme?ref=${ref}` - ); - return content ? truncate(content) : undefined; - } - - const repoMatch = repoUrl.match(GITHUB_REPO_PATTERN); - if (repoMatch) { - const [, owner, repo] = repoMatch; - const content = await fetchGitHubReadme( - `https://api.github.com/repos/${owner}/${repo}/readme` - ); - return content ? truncate(content) : undefined; - } -}; - const VendorOfficialNotice = ({ adapter }: { adapter: Adapter }) => { const issuesUrl = getIssuesUrl(adapter.readme); const author = getAuthor(adapter); @@ -172,6 +90,7 @@ const Page = async ({ params }: { params: Promise<PageParams> }) => { const data = page.data as unknown as AdapterFrontmatter; const adapter = getAdapter(slug); + const markdownPath = `/adapters/vendor-official/${slug}.md`; const useMdxBody = data.mdxBody === true; let readme: string | undefined; if (!useMdxBody && adapter) { @@ -231,6 +150,16 @@ const Page = async ({ params }: { params: Promise<PageParams> }) => { toc={page.data.toc} > <DocsBody> + {/* biome-ignore lint/a11y/useAnchorContent: intentionally aria-hidden hint surfacing the markdown URL for AI/LLM crawlers, not for screen readers */} + <a + aria-hidden="true" + className="sr-only" + href={markdownPath} + tabIndex={-1} + > + AI/LLM: This adapter page is available in plain markdown format at + {markdownPath} + </a> <AdapterHero logo={data.logo} name={data.title} @@ -267,11 +196,17 @@ export const generateMetadata = async ({ title: page.data.title, description: page.data.description, openGraph: { + title: page.data.title, images: `/${lang}/adapters/vendor-official/${slug}/og`, }, twitter: { card: "summary_large_image", }, + alternates: { + types: { + "text/markdown": `/adapters/vendor-official/${slug}.md`, + }, + }, }; }; diff --git a/apps/docs/app/[lang]/docs/[[...slug]]/page.tsx b/apps/docs/app/[lang]/docs/[[...slug]]/page.tsx index 6cc77f69..ce0c2aa0 100644 --- a/apps/docs/app/[lang]/docs/[[...slug]]/page.tsx +++ b/apps/docs/app/[lang]/docs/[[...slug]]/page.tsx @@ -28,6 +28,8 @@ const Page = async ({ params }: PageProps<"/[lang]/docs/[[...slug]]">) => { const markdown = await getLLMText(page); const MDX = page.data.body; + const slugPath = slug?.join("/"); + const markdownPath = slugPath ? `/docs/${slugPath}.md` : "/docs.md"; return ( <DocsPage @@ -50,6 +52,10 @@ const Page = async ({ params }: PageProps<"/[lang]/docs/[[...slug]]">) => { toc={page.data.toc} > <MobileDocsBar toc={page.data.toc} /> + <a aria-hidden="true" className="sr-only" href={markdownPath} tabIndex={-1}> + AI/LLM: This documentation page is available in plain markdown format at + {markdownPath} + </a> <DocsTitle>{page.data.title}</DocsTitle> <DocsDescription>{page.data.description}</DocsDescription> <DocsBody> @@ -81,6 +87,7 @@ export const generateMetadata = async ({ title: page.data.title, description: page.data.description, openGraph: { + title: page.data.title, images: getPageImage(page).url, }, twitter: { @@ -88,7 +95,7 @@ export const generateMetadata = async ({ }, alternates: { types: { - "text/markdown": slug ? `/docs/${slug}.md` : "/docs.md", + "text/markdown": slug?.length ? `/docs/${slug.join("/")}.md` : "/docs.md", }, }, }; diff --git a/apps/docs/lib/geistdocs/adapter-readme.ts b/apps/docs/lib/geistdocs/adapter-readme.ts new file mode 100644 index 00000000..0396b08f --- /dev/null +++ b/apps/docs/lib/geistdocs/adapter-readme.ts @@ -0,0 +1,121 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import adaptersJson from "@/adapters.json"; + +const LOCAL_PACKAGE_PATTERN = /github\.com\/vercel\/chat\/tree\/[^/]+\/(.+)/; +const GITHUB_SUBPATH_PATTERN = + /github\.com\/([^/]+)\/([^/]+)\/tree\/([^/]+)\/(.+)/; +const GITHUB_REPO_REF_PATTERN = + /^https:\/\/github\.com\/([^/]+)\/([^/]+)\/tree\/([^/]+)\/?$/; +const GITHUB_REPO_PATTERN = /github\.com\/([^/]+)\/([^/]+)/; +const GITHUB_REPO_ROOT_PATTERN = /^(https:\/\/github\.com\/[^/]+\/[^/]+)/; +const UNPINNED_REF_PATTERN = /^(main|master|head|dev|develop|trunk|default)$/i; + +const MAX_README_BYTES = 500_000; + +export type Adapter = (typeof adaptersJson)[number]; + +export const getAdapter = (slug: string): Adapter | undefined => + adaptersJson.find((a) => a.slug === slug); + +export const getAuthor = (adapter: Adapter): string | undefined => + "author" in adapter ? adapter.author : undefined; + +export const getIssuesUrl = ( + readmeUrl: string | undefined +): string | undefined => { + if (!readmeUrl) { + return; + } + const match = readmeUrl.match(GITHUB_REPO_ROOT_PATTERN); + return match ? `${match[1]}/issues` : undefined; +}; + +const warnUnpinned = (adapter: Adapter, ref: string | undefined) => { + if (ref && !UNPINNED_REF_PATTERN.test(ref)) { + return; + } + console.warn( + `[adapters] Community adapter "${adapter.name}" uses an unpinned README ref "${ + ref ?? "<default branch>" + }". Pin to a commit SHA or tag in adapters.json to freeze content at review time.` + ); +}; + +const truncate = (content: string): string => + content.length <= MAX_README_BYTES + ? content + : `${content.slice(0, MAX_README_BYTES)}\n\n> _README truncated — view the full version on GitHub._`; + +const fetchGitHubReadme = async (url: string): Promise<string | undefined> => { + const response = await fetch(url, { + headers: { Accept: "application/vnd.github.raw+json" }, + next: { revalidate: 3600 }, + }); + if (response.ok) { + return response.text(); + } +}; + +interface GetReadmeOptions { + /** Emit a build-time warning when the README ref is not pinned to a SHA/tag. */ + warnOnUnpinnedRef?: boolean; +} + +export const getReadme = async ( + adapter: Adapter, + options: GetReadmeOptions = {} +): Promise<string | undefined> => { + if (!adapter.readme) { + return; + } + const repoUrl = adapter.readme; + const warn = options.warnOnUnpinnedRef ?? false; + + const localMatch = repoUrl.match(LOCAL_PACKAGE_PATTERN); + if (localMatch) { + const [, pkgPath] = localMatch; + const filePath = join(process.cwd(), "..", "..", pkgPath, "README.md"); + try { + return truncate(await readFile(filePath, "utf-8")); + } catch { + return; + } + } + + const subpathMatch = repoUrl.match(GITHUB_SUBPATH_PATTERN); + if (subpathMatch) { + const [, owner, repo, ref, path] = subpathMatch; + if (warn) { + warnUnpinned(adapter, ref); + } + const content = await fetchGitHubReadme( + `https://api.github.com/repos/${owner}/${repo}/readme/${path}?ref=${ref}` + ); + return content ? truncate(content) : undefined; + } + + const repoRefMatch = repoUrl.match(GITHUB_REPO_REF_PATTERN); + if (repoRefMatch) { + const [, owner, repo, ref] = repoRefMatch; + if (warn) { + warnUnpinned(adapter, ref); + } + const content = await fetchGitHubReadme( + `https://api.github.com/repos/${owner}/${repo}/readme?ref=${ref}` + ); + return content ? truncate(content) : undefined; + } + + const repoMatch = repoUrl.match(GITHUB_REPO_PATTERN); + if (repoMatch) { + const [, owner, repo] = repoMatch; + if (warn) { + warnUnpinned(adapter, undefined); + } + const content = await fetchGitHubReadme( + `https://api.github.com/repos/${owner}/${repo}/readme` + ); + return content ? truncate(content) : undefined; + } +}; diff --git a/apps/docs/lib/geistdocs/adapters-source.ts b/apps/docs/lib/geistdocs/adapters-source.ts index ec51700f..dfd108d2 100644 --- a/apps/docs/lib/geistdocs/adapters-source.ts +++ b/apps/docs/lib/geistdocs/adapters-source.ts @@ -21,3 +21,29 @@ export const getAdapterPageImage = (page: AdapterPage) => { : `/og/${segments.join("/")}`, }; }; + +export const getAdapterLLMText = async (page: AdapterPage, body?: string) => { + const content = body ?? (await page.data.getText("processed")); + const { title, description } = page.data; + const { packageName, tagline } = page.data as unknown as { + packageName?: string; + tagline?: string; + }; + + const frontmatter = [ + "---", + `title: ${title}`, + description && `description: ${description}`, + tagline && `tagline: ${tagline}`, + packageName && `package: ${packageName}`, + "---", + ] + .filter(Boolean) + .join("\n"); + + return `${frontmatter} + +# ${title} + +${content}`; +}; diff --git a/apps/docs/proxy.ts b/apps/docs/proxy.ts index bc192262..2fe9adcc 100644 --- a/apps/docs/proxy.ts +++ b/apps/docs/proxy.ts @@ -13,6 +13,11 @@ const { rewrite: rewriteLLM } = rewritePath( `/${i18n.defaultLanguage}/llms.mdx/*path` ); +const { rewrite: rewriteAdaptersLLM } = rewritePath( + "/adapters/*path", + `/${i18n.defaultLanguage}/adapters.mdx/*path` +); + const MDX_EXTENSION_PATTERN = /\.mdx?$/; const internationalizer = createI18nMiddleware(i18n); @@ -20,11 +25,11 @@ const internationalizer = createI18nMiddleware(i18n); const proxy = (request: NextRequest, context: NextFetchEvent) => { const pathname = request.nextUrl.pathname; - // Track llms.txt requests - if (pathname === "/llms.txt") { + // Track llms.txt / llms-full.txt requests + if (pathname === "/llms.txt" || pathname === "/llms-full.txt") { context.waitUntil( trackMdRequest({ - path: "/llms.txt", + path: pathname, userAgent: request.headers.get("user-agent"), referer: request.headers.get("referer"), acceptHeader: request.headers.get("accept"), @@ -57,9 +62,29 @@ const proxy = (request: NextRequest, context: NextFetchEvent) => { } } + // Handle .md/.mdx URL requests for adapter pages before i18n runs + if ( + pathname.startsWith("/adapters/") && + (pathname.endsWith(".md") || pathname.endsWith(".mdx")) + ) { + const stripped = pathname.replace(MDX_EXTENSION_PATTERN, ""); + const result = rewriteAdaptersLLM(stripped); + if (result) { + context.waitUntil( + trackMdRequest({ + path: pathname, + userAgent: request.headers.get("user-agent"), + referer: request.headers.get("referer"), + acceptHeader: request.headers.get("accept"), + }) + ); + return NextResponse.rewrite(new URL(result, request.nextUrl)); + } + } + // Handle Accept header content negotiation and track the request if (isMarkdownPreferred(request)) { - const result = rewriteLLM(pathname); + const result = rewriteLLM(pathname) ?? rewriteAdaptersLLM(pathname); if (result) { context.waitUntil( trackMdRequest({ From 89da56df7825fe16962ca06a9ca0a1cc97c90860 Mon Sep 17 00:00:00 2001 From: Ben Sabic <bensabic@users.noreply.github.com> Date: Fri, 29 May 2026 16:50:50 +1000 Subject: [PATCH 3/6] feat(docs): make llms.txt a sitemap-style index, add llms-full.txt Rewrite llms.txt as a curated index that links to the markdown version of every docs and adapter page, grouped by section. Move the previous full-text concatenation of every page to llms-full.txt. --- apps/docs/app/[lang]/llms-full.txt/route.ts | 19 +++ apps/docs/app/[lang]/llms.txt/route.ts | 131 +++++++++++++++++++- 2 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 apps/docs/app/[lang]/llms-full.txt/route.ts diff --git a/apps/docs/app/[lang]/llms-full.txt/route.ts b/apps/docs/app/[lang]/llms-full.txt/route.ts new file mode 100644 index 00000000..6b729442 --- /dev/null +++ b/apps/docs/app/[lang]/llms-full.txt/route.ts @@ -0,0 +1,19 @@ +import type { NextRequest } from "next/server"; +import { getLLMText, source } from "@/lib/geistdocs/source"; + +export const revalidate = false; + +export const GET = async ( + _req: NextRequest, + { params }: { params: Promise<{ lang: string }> } +) => { + const { lang } = await params; + const scan = source.getPages(lang).map(getLLMText); + const scanned = await Promise.all(scan); + + return new Response(scanned.join("\n\n"), { + headers: { + "Content-Type": "text/markdown; charset=utf-8", + }, + }); +}; diff --git a/apps/docs/app/[lang]/llms.txt/route.ts b/apps/docs/app/[lang]/llms.txt/route.ts index 9601f34e..0b32da99 100644 --- a/apps/docs/app/[lang]/llms.txt/route.ts +++ b/apps/docs/app/[lang]/llms.txt/route.ts @@ -1,17 +1,140 @@ +import type { Item, Node, Root } from "fumadocs-core/page-tree"; import type { NextRequest } from "next/server"; -import { getLLMText, source } from "@/lib/geistdocs/source"; +import type { ReactNode } from "react"; +import { title } from "@/geistdocs"; +import { adaptersSource } from "@/lib/geistdocs/adapters-source"; +import { source } from "@/lib/geistdocs/source"; export const revalidate = false; +const baseUrl = "https://chat-sdk.dev"; + +const DESCRIPTION = + "A unified TypeScript SDK for building chat bots and agents across Slack, Microsoft Teams, Google Chat, Discord, Telegram, WhatsApp, and more — with type-safe handlers, JSX cards, and AI streaming."; + +const DETAILS = + "This index lists the documentation pages in plain markdown. Each link points to the markdown version of a page. For the full text of every page concatenated into a single file, see the llms-full.txt link under Optional."; + +const getName = (name: ReactNode): string => + typeof name === "string" ? name : ""; + +const renderItem = ( + item: Item, + descriptionByUrl: Map<string, string | undefined> +): string => { + const url = `${baseUrl}${item.url}.md`; + const description = descriptionByUrl.get(item.url); + const label = getName(item.name); + return description + ? `- [${label}](${url}): ${description}` + : `- [${label}](${url})`; +}; + +type Section = { name: string; items: Item[] }; + +const collectItems = (node: Node, out: Item[]) => { + if (node.type === "page") { + out.push(node); + return; + } + if (node.type === "folder") { + if (node.index) { + out.push(node.index); + } + for (const child of node.children) { + collectItems(child, out); + } + } +}; + +/** + * Build sections from a tree whose top-level children are separators (docs): + * each separator starts a new section, pages/folders fall under it. + */ +const sectionsFromSeparators = (root: Root): Section[] => { + const sections: Section[] = []; + let current: Section = { + name: getName(root.name) || "Documentation", + items: [], + }; + sections.push(current); + + for (const node of root.children) { + if (node.type === "separator") { + current = { name: getName(node.name), items: [] }; + sections.push(current); + } else { + collectItems(node, current.items); + } + } + + return sections; +}; + +/** + * Build sections from a tree whose top-level children are folders (adapters): + * each folder becomes a section, flattening its nested separators/pages. + */ +const sectionsFromFolders = (root: Root): Section[] => { + const sections: Section[] = []; + + for (const node of root.children) { + if (node.type !== "folder") { + continue; + } + const items: Item[] = []; + if (node.index) { + items.push(node.index); + } + for (const child of node.children) { + collectItems(child, items); + } + sections.push({ name: getName(node.name), items }); + } + + return sections; +}; + export const GET = async ( _req: NextRequest, { params }: RouteContext<"/[lang]/llms.txt"> ) => { const { lang } = await params; - const scan = source.getPages(lang).map(getLLMText); - const scanned = await Promise.all(scan); + const descriptionByUrl = new Map<string, string | undefined>([ + ...source.getPages(lang).map( + (page) => [page.url, page.data.description] as const + ), + ...adaptersSource.getPages(lang).map( + (page) => [page.url, page.data.description] as const + ), + ]); + + const sections = [ + ...sectionsFromSeparators(source.pageTree[lang]), + ...sectionsFromFolders(adaptersSource.pageTree[lang]), + ]; + + const lines: string[] = [`# ${title}`, "", `> ${DESCRIPTION}`, "", DETAILS]; + + for (const section of sections) { + if (section.items.length === 0) { + continue; + } + lines.push("", `## ${section.name}`, ""); + for (const item of section.items) { + lines.push(renderItem(item, descriptionByUrl)); + } + } + + lines.push( + "", + "## Optional", + "", + `- [Full documentation](${baseUrl}/llms-full.txt): Every documentation page concatenated into a single file.`, + `- [Documentation sitemap](${baseUrl}/sitemap.md): Semantic index of every documentation page.` + ); - return new Response(scanned.join("\n\n"), { + return new Response(`${lines.join("\n")}\n`, { headers: { "Content-Type": "text/markdown; charset=utf-8", }, From f5b0901cf42275a00b2e4b30db1ea6e753c12d9a Mon Sep 17 00:00:00 2001 From: Ben Sabic <bensabic@users.noreply.github.com> Date: Fri, 29 May 2026 16:51:05 +1000 Subject: [PATCH 4/6] test(docs): validate docs frontmatter and adapter markdown URL scheme Add content-validation tests guarding the invariants the llms.txt index and the markdown routes rely on: every docs page has a title and description, and adapter content groups map to well-formed /adapters/<group>/<slug>.md URLs. --- .../integration-tests/src/docs-llms.test.ts | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 packages/integration-tests/src/docs-llms.test.ts diff --git a/packages/integration-tests/src/docs-llms.test.ts b/packages/integration-tests/src/docs-llms.test.ts new file mode 100644 index 00000000..4d22bcae --- /dev/null +++ b/packages/integration-tests/src/docs-llms.test.ts @@ -0,0 +1,92 @@ +import { readFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { DOCS_CONTENT_DIR, findDocsMdxFiles } from "./documentation-test-utils"; + +const DOCS_DIR = join(DOCS_CONTENT_DIR, "docs"); +const ADAPTERS_DIR = join(DOCS_CONTENT_DIR, "adapters"); + +// The markdown route handler (`/adapters.mdx/[[...slug]]`), the proxy `.md` +// rewrite, the per-page `sr-only` markdown links + `text/markdown` alternates, +// and the `llms.txt` section builder all assume these exact group directories. +const ADAPTER_GROUPS = ["official", "community", "vendor-official"] as const; + +const FRONTMATTER_BLOCK = /^---\r?\n([\s\S]*?)\r?\n---/; +const FIELD_LINE = /^([a-zA-Z][a-zA-Z0-9_]*):\s*(.*)$/; +const NEWLINE = /\r?\n/; +const QUOTES = /^["']|["']$/g; +const MDX_EXTENSION = /\.mdx?$/; +const ADAPTER_MARKDOWN_URL = + /^\/adapters\/(official|community|vendor-official)\/[a-z0-9]+(?:-[a-z0-9]+)*\.md$/; + +const parseFrontmatter = (content: string): Record<string, string> => { + const fields: Record<string, string> = {}; + const match = content.match(FRONTMATTER_BLOCK); + if (!match) { + return fields; + } + for (const line of match[1].split(NEWLINE)) { + const fieldMatch = line.match(FIELD_LINE); + if (fieldMatch) { + const [, key, value] = fieldMatch; + fields[key] = value.trim().replace(QUOTES, ""); + } + } + return fields; +}; + +describe("Docs pages power the llms.txt index", () => { + const docs = findDocsMdxFiles(DOCS_DIR); + + it("discovers documentation pages", () => { + expect(docs.length).toBeGreaterThan(0); + }); + + for (const doc of docs) { + describe(doc.name, () => { + const fields = parseFrontmatter(readFileSync(doc.path, "utf-8")); + + it("has a title (used for <title>, OG title, and llms.txt label)", () => { + expect( + fields.title, + `${doc.name}: missing frontmatter "title"` + ).toBeTruthy(); + }); + + it("has a description (used for metadata and the llms.txt entry)", () => { + expect( + fields.description, + `${doc.name}: missing frontmatter "description"` + ).toBeTruthy(); + }); + }); + } +}); + +describe("Adapter markdown URL scheme", () => { + it("content groups match the route segments exactly", () => { + const present = ADAPTER_GROUPS.filter( + (group) => findDocsMdxFiles(join(ADAPTERS_DIR, group)).length > 0 + ); + expect([...present].sort()).toEqual([...ADAPTER_GROUPS].sort()); + }); + + for (const group of ADAPTER_GROUPS) { + describe(group, () => { + const files = findDocsMdxFiles(join(ADAPTERS_DIR, group)); + + it("contains at least one adapter", () => { + expect(files.length).toBeGreaterThan(0); + }); + + for (const file of files) { + const slug = basename(file.path).replace(MDX_EXTENSION, ""); + + it(`${slug} maps to a well-formed markdown URL`, () => { + const markdownPath = `/adapters/${group}/${slug}.md`; + expect(markdownPath).toMatch(ADAPTER_MARKDOWN_URL); + }); + } + }); + } +}); From e44aee9315d302d23206aac0dbd16b431cb91d3e Mon Sep 17 00:00:00 2001 From: Ben Sabic <bensabic@users.noreply.github.com> Date: Fri, 29 May 2026 16:58:10 +1000 Subject: [PATCH 5/6] ci: exclude integration-tests from the changeset requirement The integration-tests package is private and already in the changeset `ignore` list, so changes to it should not require a changeset. Skip it in the "packages changed" detection like packages/chat/resources. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a16a1115..e41b8606 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: BASE_REF: ${{ github.base_ref }} run: | git fetch origin "$BASE_REF" --depth=1 - if git diff --name-only "origin/$BASE_REF"...HEAD | grep '^packages/' | grep -qEv '\.md$|^packages/chat/resources/'; then + if git diff --name-only "origin/$BASE_REF"...HEAD | grep '^packages/' | grep -qEv '\.md$|^packages/chat/resources/|^packages/integration-tests/'; then echo "changed=true" >> "$GITHUB_OUTPUT" else echo "changed=false" >> "$GITHUB_OUTPUT" From 7b5882f404e9d85bb1ab9a7588dffc7bc707ff79 Mon Sep 17 00:00:00 2001 From: Ben Sabic <bensabic@users.noreply.github.com> Date: Mon, 1 Jun 2026 23:28:19 +1000 Subject: [PATCH 6/6] fix(docs): honor Accept: text/markdown on adapter pages `rewritePath().rewrite` returns `string | false`, so the `??` fallback left adapter pages serving HTML under `Accept: text/markdown` (a `false` docs-rewrite result never fell through to the adapters rewriter). Use `||` so the adapters rewriter is tried. Add a regression guard. --- apps/docs/proxy.ts | 2 +- .../integration-tests/src/docs-llms.test.ts | 25 ++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/apps/docs/proxy.ts b/apps/docs/proxy.ts index 2fe9adcc..6977881b 100644 --- a/apps/docs/proxy.ts +++ b/apps/docs/proxy.ts @@ -84,7 +84,7 @@ const proxy = (request: NextRequest, context: NextFetchEvent) => { // Handle Accept header content negotiation and track the request if (isMarkdownPreferred(request)) { - const result = rewriteLLM(pathname) ?? rewriteAdaptersLLM(pathname); + const result = rewriteLLM(pathname) || rewriteAdaptersLLM(pathname); if (result) { context.waitUntil( trackMdRequest({ diff --git a/packages/integration-tests/src/docs-llms.test.ts b/packages/integration-tests/src/docs-llms.test.ts index 4d22bcae..a1bc3f68 100644 --- a/packages/integration-tests/src/docs-llms.test.ts +++ b/packages/integration-tests/src/docs-llms.test.ts @@ -1,10 +1,15 @@ import { readFileSync } from "node:fs"; import { basename, join } from "node:path"; import { describe, expect, it } from "vitest"; -import { DOCS_CONTENT_DIR, findDocsMdxFiles } from "./documentation-test-utils"; +import { + DOCS_CONTENT_DIR, + findDocsMdxFiles, + REPO_ROOT, +} from "./documentation-test-utils"; const DOCS_DIR = join(DOCS_CONTENT_DIR, "docs"); const ADAPTERS_DIR = join(DOCS_CONTENT_DIR, "adapters"); +const PROXY_PATH = join(REPO_ROOT, "apps/docs/proxy.ts"); // The markdown route handler (`/adapters.mdx/[[...slug]]`), the proxy `.md` // rewrite, the per-page `sr-only` markdown links + `text/markdown` alternates, @@ -90,3 +95,21 @@ describe("Adapter markdown URL scheme", () => { }); } }); + +describe("Adapter Accept-header markdown negotiation", () => { + const proxy = readFileSync(PROXY_PATH, "utf-8"); + + // `rewritePath().rewrite` returns `string | false`. With `??`, a `false` + // result from the docs rewriter does NOT fall through to the adapters + // rewriter, so `Accept: text/markdown` on an adapter page (e.g. + // /adapters/official/slack) would keep serving HTML. It must use `||`. + it("falls through to the adapters rewriter with `||`", () => { + expect(proxy).toContain( + "rewriteLLM(pathname) || rewriteAdaptersLLM(pathname)" + ); + }); + + it("does not regress to `??` for the rewriter fallback", () => { + expect(proxy).not.toContain("rewriteLLM(pathname) ?? rewriteAdaptersLLM"); + }); +});