diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a16a1115..e41b8606 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: BASE_REF: ${{ github.base_ref }} run: | git fetch origin "$BASE_REF" --depth=1 - if git diff --name-only "origin/$BASE_REF"...HEAD | grep '^packages/' | grep -qEv '\.md$|^packages/chat/resources/'; then + if git diff --name-only "origin/$BASE_REF"...HEAD | grep '^packages/' | grep -qEv '\.md$|^packages/chat/resources/|^packages/integration-tests/'; then echo "changed=true" >> "$GITHUB_OUTPUT" else echo "changed=false" >> "$GITHUB_OUTPUT" diff --git a/apps/docs/app/[lang]/(home)/page.tsx b/apps/docs/app/[lang]/(home)/page.tsx index 2e01fce5..644d8aaa 100644 --- a/apps/docs/app/[lang]/(home)/page.tsx +++ b/apps/docs/app/[lang]/(home)/page.tsx @@ -28,8 +28,11 @@ const heroDescription = "A unified TypeScript SDK for building chat bots with type-safe handlers, JSX cards, and multi-platform support—powered by Vercel"; export const metadata: Metadata = { - title: metadataTitle, + title: { absolute: metadataTitle }, description: heroDescription, + openGraph: { + title: { absolute: metadataTitle }, + }, twitter: { card: "summary_large_image", }, diff --git a/apps/docs/app/[lang]/(home)/resources/page.tsx b/apps/docs/app/[lang]/(home)/resources/page.tsx index de3034e2..ad380cff 100644 --- a/apps/docs/app/[lang]/(home)/resources/page.tsx +++ b/apps/docs/app/[lang]/(home)/resources/page.tsx @@ -17,6 +17,9 @@ export const metadata: Metadata = { "ai agent", "vercel", ], + openGraph: { + title: "Resources", + }, twitter: { card: "summary_large_image", }, diff --git a/apps/docs/app/[lang]/adapters.mdx/[[...slug]]/route.ts b/apps/docs/app/[lang]/adapters.mdx/[[...slug]]/route.ts new file mode 100644 index 00000000..0780e45f --- /dev/null +++ b/apps/docs/app/[lang]/adapters.mdx/[[...slug]]/route.ts @@ -0,0 +1,43 @@ +import { notFound } from "next/navigation"; +import { getAdapter, getReadme } from "@/lib/geistdocs/adapter-readme"; +import { + adaptersSource, + getAdapterLLMText, +} from "@/lib/geistdocs/adapters-source"; + +export const revalidate = false; + +export async function GET( + _req: Request, + { params }: RouteContext<"/[lang]/adapters.mdx/[[...slug]]"> +) { + const { slug, lang } = await params; + const page = adaptersSource.getPage(slug, lang); + + if (!page) { + notFound(); + } + + // Official adapters render their MDX body directly. Community and + // vendor-official adapters render an upstream README (unless they opt into an + // MDX body), so serve that same content in the markdown version. + const type = slug?.[0]; + let body: string | undefined; + if (type !== "official") { + const data = page.data as unknown as { mdxBody?: boolean; slug: string }; + if (data.mdxBody !== true) { + const adapter = getAdapter(data.slug); + if (adapter) { + body = await getReadme(adapter); + } + } + } + + return new Response(await getAdapterLLMText(page, body), { + headers: { + "Content-Type": "text/markdown", + }, + }); +} + +export const generateStaticParams = () => adaptersSource.generateParams(); diff --git a/apps/docs/app/[lang]/adapters/(detail)/community/[slug]/page.tsx b/apps/docs/app/[lang]/adapters/(detail)/community/[slug]/page.tsx index 41196dfb..69bfe2f1 100644 --- a/apps/docs/app/[lang]/adapters/(detail)/community/[slug]/page.tsx +++ b/apps/docs/app/[lang]/adapters/(detail)/community/[slug]/page.tsx @@ -1,16 +1,19 @@ -import { readFile } from "node:fs/promises"; -import { join } from "node:path"; import { createRelativeLink } from "fumadocs-ui/mdx"; import type { Metadata } from "next"; import { notFound } from "next/navigation"; import type { ComponentProps, FC } from "react"; -import adaptersJson from "@/adapters.json"; import { AdapterHero } from "@/components/geistdocs/adapter-hero"; import { DocsBody, DocsPage } from "@/components/geistdocs/docs-page"; import { FeatureSupport } from "@/components/geistdocs/feature-support"; import { getMDXComponents } from "@/components/geistdocs/mdx-components"; import { Upsell } from "@/components/geistdocs/upsell"; import type { AdapterFeatureValue } from "@/lib/adapter-features"; +import { + type Adapter, + getAdapter, + getIssuesUrl, + getReadme, +} from "@/lib/geistdocs/adapter-readme"; import { adaptersSource } from "@/lib/geistdocs/adapters-source"; import { ReadmeContent } from "../../../components/readme-content"; @@ -33,105 +36,6 @@ const wrapWithNofollow = (BaseLink: FC): FC => { return NofollowExternalLink; }; -const LOCAL_PACKAGE_PATTERN = /github\.com\/vercel\/chat\/tree\/[^/]+\/(.+)/; -const GITHUB_SUBPATH_PATTERN = - /github\.com\/([^/]+)\/([^/]+)\/tree\/([^/]+)\/(.+)/; -const GITHUB_REPO_REF_PATTERN = - /^https:\/\/github\.com\/([^/]+)\/([^/]+)\/tree\/([^/]+)\/?$/; -const GITHUB_REPO_PATTERN = /github\.com\/([^/]+)\/([^/]+)/; -const GITHUB_REPO_ROOT_PATTERN = /^(https:\/\/github\.com\/[^/]+\/[^/]+)/; - -const UNPINNED_REF_PATTERN = /^(main|master|head|dev|develop|trunk|default)$/i; - -const MAX_README_BYTES = 500_000; - -type Adapter = (typeof adaptersJson)[number]; - -const getAdapter = (slug: string): Adapter | undefined => - adaptersJson.find((a) => a.slug === slug); - -const getIssuesUrl = (readmeUrl: string | undefined): string | undefined => { - if (!readmeUrl) { - return; - } - const match = readmeUrl.match(GITHUB_REPO_ROOT_PATTERN); - return match ? `${match[1]}/issues` : undefined; -}; - -const warnUnpinned = (adapter: Adapter, ref: string | undefined) => { - if (ref && !UNPINNED_REF_PATTERN.test(ref)) { - return; - } - console.warn( - `[adapters] Community adapter "${adapter.name}" uses an unpinned README ref "${ - ref ?? "" - }". Pin to a commit SHA or tag in adapters.json to freeze content at review time.` - ); -}; - -const truncate = (content: string): string => - content.length <= MAX_README_BYTES - ? content - : `${content.slice(0, MAX_README_BYTES)}\n\n> _README truncated — view the full version on GitHub._`; - -const fetchGitHubReadme = async (url: string): Promise => { - const response = await fetch(url, { - headers: { Accept: "application/vnd.github.raw+json" }, - next: { revalidate: 3600 }, - }); - if (response.ok) { - return response.text(); - } -}; - -const getReadme = async (adapter: Adapter): Promise => { - if (!adapter.readme) { - return; - } - const repoUrl = adapter.readme; - - const localMatch = repoUrl.match(LOCAL_PACKAGE_PATTERN); - if (localMatch) { - const [, pkgPath] = localMatch; - const filePath = join(process.cwd(), "..", "..", pkgPath, "README.md"); - try { - return truncate(await readFile(filePath, "utf-8")); - } catch { - return; - } - } - - const subpathMatch = repoUrl.match(GITHUB_SUBPATH_PATTERN); - if (subpathMatch) { - const [, owner, repo, ref, path] = subpathMatch; - warnUnpinned(adapter, ref); - const content = await fetchGitHubReadme( - `https://api.github.com/repos/${owner}/${repo}/readme/${path}?ref=${ref}` - ); - return content ? truncate(content) : undefined; - } - - const repoRefMatch = repoUrl.match(GITHUB_REPO_REF_PATTERN); - if (repoRefMatch) { - const [, owner, repo, ref] = repoRefMatch; - warnUnpinned(adapter, ref); - const content = await fetchGitHubReadme( - `https://api.github.com/repos/${owner}/${repo}/readme?ref=${ref}` - ); - return content ? truncate(content) : undefined; - } - - const repoMatch = repoUrl.match(GITHUB_REPO_PATTERN); - if (repoMatch) { - const [, owner, repo] = repoMatch; - warnUnpinned(adapter, undefined); - const content = await fetchGitHubReadme( - `https://api.github.com/repos/${owner}/${repo}/readme` - ); - return content ? truncate(content) : undefined; - } -}; - const CommunityNotice = ({ adapter }: { adapter: Adapter }) => { const issuesUrl = getIssuesUrl(adapter.readme); @@ -194,10 +98,11 @@ const Page = async ({ params }: { params: Promise }) => { const data = page.data as unknown as AdapterFrontmatter; const adapter = getAdapter(slug); + const markdownPath = `/adapters/community/${slug}.md`; const useMdxBody = data.mdxBody === true; let readme: string | undefined; if (!useMdxBody && adapter) { - readme = await getReadme(adapter); + readme = await getReadme(adapter, { warnOnUnpinnedRef: true }); } const MDX = page.data.body; const BoundFeatureSupport = renderBoundFeatureSupport( @@ -253,6 +158,16 @@ const Page = async ({ params }: { params: Promise }) => { toc={page.data.toc} > + {/* biome-ignore lint/a11y/useAnchorContent: intentionally aria-hidden hint surfacing the markdown URL for AI/LLM crawlers, not for screen readers */} + }) => { const data = page.data as unknown as AdapterFrontmatter; const MDX = page.data.body; + const markdownPath = `/adapters/official/${slug}.md`; const BoundFeatureSupport = renderBoundFeatureSupport( data.features, data.type @@ -64,6 +65,16 @@ const Page = async ({ params }: { params: Promise }) => { toc={page.data.toc} > + {/* biome-ignore lint/a11y/useAnchorContent: intentionally aria-hidden hint surfacing the markdown URL for AI/LLM crawlers, not for screen readers */} + - adaptersJson.find((a) => a.slug === slug); - -const getAuthor = (adapter: Adapter): string | undefined => - "author" in adapter ? adapter.author : undefined; - -const getIssuesUrl = (readmeUrl: string | undefined): string | undefined => { - if (!readmeUrl) { - return; - } - const match = readmeUrl.match(GITHUB_REPO_ROOT_PATTERN); - return match ? `${match[1]}/issues` : undefined; -}; - -const truncate = (content: string): string => - content.length <= MAX_README_BYTES - ? content - : `${content.slice(0, MAX_README_BYTES)}\n\n> _README truncated — view the full version on GitHub._`; - -const fetchGitHubReadme = async (url: string): Promise => { - const response = await fetch(url, { - headers: { Accept: "application/vnd.github.raw+json" }, - next: { revalidate: 3600 }, - }); - if (response.ok) { - return response.text(); - } -}; - -const getReadme = async (adapter: Adapter): Promise => { - if (!adapter.readme) { - return; - } - const repoUrl = adapter.readme; - - const localMatch = repoUrl.match(LOCAL_PACKAGE_PATTERN); - if (localMatch) { - const [, pkgPath] = localMatch; - const filePath = join(process.cwd(), "..", "..", pkgPath, "README.md"); - try { - return truncate(await readFile(filePath, "utf-8")); - } catch { - return; - } - } - - const subpathMatch = repoUrl.match(GITHUB_SUBPATH_PATTERN); - if (subpathMatch) { - const [, owner, repo, ref, path] = subpathMatch; - const content = await fetchGitHubReadme( - `https://api.github.com/repos/${owner}/${repo}/readme/${path}?ref=${ref}` - ); - return content ? truncate(content) : undefined; - } - - const repoRefMatch = repoUrl.match(GITHUB_REPO_REF_PATTERN); - if (repoRefMatch) { - const [, owner, repo, ref] = repoRefMatch; - const content = await fetchGitHubReadme( - `https://api.github.com/repos/${owner}/${repo}/readme?ref=${ref}` - ); - return content ? truncate(content) : undefined; - } - - const repoMatch = repoUrl.match(GITHUB_REPO_PATTERN); - if (repoMatch) { - const [, owner, repo] = repoMatch; - const content = await fetchGitHubReadme( - `https://api.github.com/repos/${owner}/${repo}/readme` - ); - return content ? truncate(content) : undefined; - } -}; - const VendorOfficialNotice = ({ adapter }: { adapter: Adapter }) => { const issuesUrl = getIssuesUrl(adapter.readme); const author = getAuthor(adapter); @@ -172,6 +90,7 @@ const Page = async ({ params }: { params: Promise }) => { const data = page.data as unknown as AdapterFrontmatter; const adapter = getAdapter(slug); + const markdownPath = `/adapters/vendor-official/${slug}.md`; const useMdxBody = data.mdxBody === true; let readme: string | undefined; if (!useMdxBody && adapter) { @@ -231,6 +150,16 @@ const Page = async ({ params }: { params: Promise }) => { toc={page.data.toc} > + {/* biome-ignore lint/a11y/useAnchorContent: intentionally aria-hidden hint surfacing the markdown URL for AI/LLM crawlers, not for screen readers */} + ) => { const markdown = await getLLMText(page); const MDX = page.data.body; + const slugPath = slug?.join("/"); + const markdownPath = slugPath ? `/docs/${slugPath}.md` : "/docs.md"; return ( ) => { toc={page.data.toc} > + {page.data.title} {page.data.description} @@ -81,6 +87,7 @@ export const generateMetadata = async ({ title: page.data.title, description: page.data.description, openGraph: { + title: page.data.title, images: getPageImage(page).url, }, twitter: { @@ -88,7 +95,7 @@ export const generateMetadata = async ({ }, alternates: { types: { - "text/markdown": slug ? `/docs/${slug}.md` : "/docs.md", + "text/markdown": slug?.length ? `/docs/${slug.join("/")}.md` : "/docs.md", }, }, }; diff --git a/apps/docs/app/[lang]/layout.tsx b/apps/docs/app/[lang]/layout.tsx index 83147660..5b5e065e 100644 --- a/apps/docs/app/[lang]/layout.tsx +++ b/apps/docs/app/[lang]/layout.tsx @@ -1,4 +1,5 @@ import "../global.css"; +import type { Metadata } from "next"; import { Footer } from "@/components/geistdocs/footer"; import { Navbar } from "@/components/geistdocs/navbar"; import { GeistdocsProvider } from "@/components/geistdocs/provider"; @@ -6,6 +7,19 @@ import { basePath } from "@/geistdocs"; import { mono, sans } from "@/lib/geistdocs/fonts"; import { cn } from "@/lib/utils"; +export const metadata: Metadata = { + title: { + template: "%s | Chat SDK", + default: "Chat SDK", + }, + openGraph: { + title: { + template: "%s | Chat SDK", + default: "Chat SDK", + }, + }, +}; + const Layout = async ({ children, params }: LayoutProps<"/[lang]">) => { const { lang } = await params; @@ -15,6 +29,9 @@ const Layout = async ({ children, params }: LayoutProps<"/[lang]">) => { lang={lang} suppressHydrationWarning > + + + diff --git a/apps/docs/app/[lang]/llms-full.txt/route.ts b/apps/docs/app/[lang]/llms-full.txt/route.ts new file mode 100644 index 00000000..6b729442 --- /dev/null +++ b/apps/docs/app/[lang]/llms-full.txt/route.ts @@ -0,0 +1,19 @@ +import type { NextRequest } from "next/server"; +import { getLLMText, source } from "@/lib/geistdocs/source"; + +export const revalidate = false; + +export const GET = async ( + _req: NextRequest, + { params }: { params: Promise<{ lang: string }> } +) => { + const { lang } = await params; + const scan = source.getPages(lang).map(getLLMText); + const scanned = await Promise.all(scan); + + return new Response(scanned.join("\n\n"), { + headers: { + "Content-Type": "text/markdown; charset=utf-8", + }, + }); +}; diff --git a/apps/docs/app/[lang]/llms.txt/route.ts b/apps/docs/app/[lang]/llms.txt/route.ts index 9601f34e..0b32da99 100644 --- a/apps/docs/app/[lang]/llms.txt/route.ts +++ b/apps/docs/app/[lang]/llms.txt/route.ts @@ -1,17 +1,140 @@ +import type { Item, Node, Root } from "fumadocs-core/page-tree"; import type { NextRequest } from "next/server"; -import { getLLMText, source } from "@/lib/geistdocs/source"; +import type { ReactNode } from "react"; +import { title } from "@/geistdocs"; +import { adaptersSource } from "@/lib/geistdocs/adapters-source"; +import { source } from "@/lib/geistdocs/source"; export const revalidate = false; +const baseUrl = "https://chat-sdk.dev"; + +const DESCRIPTION = + "A unified TypeScript SDK for building chat bots and agents across Slack, Microsoft Teams, Google Chat, Discord, Telegram, WhatsApp, and more — with type-safe handlers, JSX cards, and AI streaming."; + +const DETAILS = + "This index lists the documentation pages in plain markdown. Each link points to the markdown version of a page. For the full text of every page concatenated into a single file, see the llms-full.txt link under Optional."; + +const getName = (name: ReactNode): string => + typeof name === "string" ? name : ""; + +const renderItem = ( + item: Item, + descriptionByUrl: Map +): string => { + const url = `${baseUrl}${item.url}.md`; + const description = descriptionByUrl.get(item.url); + const label = getName(item.name); + return description + ? `- [${label}](${url}): ${description}` + : `- [${label}](${url})`; +}; + +type Section = { name: string; items: Item[] }; + +const collectItems = (node: Node, out: Item[]) => { + if (node.type === "page") { + out.push(node); + return; + } + if (node.type === "folder") { + if (node.index) { + out.push(node.index); + } + for (const child of node.children) { + collectItems(child, out); + } + } +}; + +/** + * Build sections from a tree whose top-level children are separators (docs): + * each separator starts a new section, pages/folders fall under it. + */ +const sectionsFromSeparators = (root: Root): Section[] => { + const sections: Section[] = []; + let current: Section = { + name: getName(root.name) || "Documentation", + items: [], + }; + sections.push(current); + + for (const node of root.children) { + if (node.type === "separator") { + current = { name: getName(node.name), items: [] }; + sections.push(current); + } else { + collectItems(node, current.items); + } + } + + return sections; +}; + +/** + * Build sections from a tree whose top-level children are folders (adapters): + * each folder becomes a section, flattening its nested separators/pages. + */ +const sectionsFromFolders = (root: Root): Section[] => { + const sections: Section[] = []; + + for (const node of root.children) { + if (node.type !== "folder") { + continue; + } + const items: Item[] = []; + if (node.index) { + items.push(node.index); + } + for (const child of node.children) { + collectItems(child, items); + } + sections.push({ name: getName(node.name), items }); + } + + return sections; +}; + export const GET = async ( _req: NextRequest, { params }: RouteContext<"/[lang]/llms.txt"> ) => { const { lang } = await params; - const scan = source.getPages(lang).map(getLLMText); - const scanned = await Promise.all(scan); + const descriptionByUrl = new Map([ + ...source.getPages(lang).map( + (page) => [page.url, page.data.description] as const + ), + ...adaptersSource.getPages(lang).map( + (page) => [page.url, page.data.description] as const + ), + ]); + + const sections = [ + ...sectionsFromSeparators(source.pageTree[lang]), + ...sectionsFromFolders(adaptersSource.pageTree[lang]), + ]; + + const lines: string[] = [`# ${title}`, "", `> ${DESCRIPTION}`, "", DETAILS]; + + for (const section of sections) { + if (section.items.length === 0) { + continue; + } + lines.push("", `## ${section.name}`, ""); + for (const item of section.items) { + lines.push(renderItem(item, descriptionByUrl)); + } + } + + lines.push( + "", + "## Optional", + "", + `- [Full documentation](${baseUrl}/llms-full.txt): Every documentation page concatenated into a single file.`, + `- [Documentation sitemap](${baseUrl}/sitemap.md): Semantic index of every documentation page.` + ); - return new Response(scanned.join("\n\n"), { + return new Response(`${lines.join("\n")}\n`, { headers: { "Content-Type": "text/markdown; charset=utf-8", }, diff --git a/apps/docs/lib/geistdocs/adapter-readme.ts b/apps/docs/lib/geistdocs/adapter-readme.ts new file mode 100644 index 00000000..0396b08f --- /dev/null +++ b/apps/docs/lib/geistdocs/adapter-readme.ts @@ -0,0 +1,121 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import adaptersJson from "@/adapters.json"; + +const LOCAL_PACKAGE_PATTERN = /github\.com\/vercel\/chat\/tree\/[^/]+\/(.+)/; +const GITHUB_SUBPATH_PATTERN = + /github\.com\/([^/]+)\/([^/]+)\/tree\/([^/]+)\/(.+)/; +const GITHUB_REPO_REF_PATTERN = + /^https:\/\/github\.com\/([^/]+)\/([^/]+)\/tree\/([^/]+)\/?$/; +const GITHUB_REPO_PATTERN = /github\.com\/([^/]+)\/([^/]+)/; +const GITHUB_REPO_ROOT_PATTERN = /^(https:\/\/github\.com\/[^/]+\/[^/]+)/; +const UNPINNED_REF_PATTERN = /^(main|master|head|dev|develop|trunk|default)$/i; + +const MAX_README_BYTES = 500_000; + +export type Adapter = (typeof adaptersJson)[number]; + +export const getAdapter = (slug: string): Adapter | undefined => + adaptersJson.find((a) => a.slug === slug); + +export const getAuthor = (adapter: Adapter): string | undefined => + "author" in adapter ? adapter.author : undefined; + +export const getIssuesUrl = ( + readmeUrl: string | undefined +): string | undefined => { + if (!readmeUrl) { + return; + } + const match = readmeUrl.match(GITHUB_REPO_ROOT_PATTERN); + return match ? `${match[1]}/issues` : undefined; +}; + +const warnUnpinned = (adapter: Adapter, ref: string | undefined) => { + if (ref && !UNPINNED_REF_PATTERN.test(ref)) { + return; + } + console.warn( + `[adapters] Community adapter "${adapter.name}" uses an unpinned README ref "${ + ref ?? "" + }". Pin to a commit SHA or tag in adapters.json to freeze content at review time.` + ); +}; + +const truncate = (content: string): string => + content.length <= MAX_README_BYTES + ? content + : `${content.slice(0, MAX_README_BYTES)}\n\n> _README truncated — view the full version on GitHub._`; + +const fetchGitHubReadme = async (url: string): Promise => { + const response = await fetch(url, { + headers: { Accept: "application/vnd.github.raw+json" }, + next: { revalidate: 3600 }, + }); + if (response.ok) { + return response.text(); + } +}; + +interface GetReadmeOptions { + /** Emit a build-time warning when the README ref is not pinned to a SHA/tag. */ + warnOnUnpinnedRef?: boolean; +} + +export const getReadme = async ( + adapter: Adapter, + options: GetReadmeOptions = {} +): Promise => { + if (!adapter.readme) { + return; + } + const repoUrl = adapter.readme; + const warn = options.warnOnUnpinnedRef ?? false; + + const localMatch = repoUrl.match(LOCAL_PACKAGE_PATTERN); + if (localMatch) { + const [, pkgPath] = localMatch; + const filePath = join(process.cwd(), "..", "..", pkgPath, "README.md"); + try { + return truncate(await readFile(filePath, "utf-8")); + } catch { + return; + } + } + + const subpathMatch = repoUrl.match(GITHUB_SUBPATH_PATTERN); + if (subpathMatch) { + const [, owner, repo, ref, path] = subpathMatch; + if (warn) { + warnUnpinned(adapter, ref); + } + const content = await fetchGitHubReadme( + `https://api.github.com/repos/${owner}/${repo}/readme/${path}?ref=${ref}` + ); + return content ? truncate(content) : undefined; + } + + const repoRefMatch = repoUrl.match(GITHUB_REPO_REF_PATTERN); + if (repoRefMatch) { + const [, owner, repo, ref] = repoRefMatch; + if (warn) { + warnUnpinned(adapter, ref); + } + const content = await fetchGitHubReadme( + `https://api.github.com/repos/${owner}/${repo}/readme?ref=${ref}` + ); + return content ? truncate(content) : undefined; + } + + const repoMatch = repoUrl.match(GITHUB_REPO_PATTERN); + if (repoMatch) { + const [, owner, repo] = repoMatch; + if (warn) { + warnUnpinned(adapter, undefined); + } + const content = await fetchGitHubReadme( + `https://api.github.com/repos/${owner}/${repo}/readme` + ); + return content ? truncate(content) : undefined; + } +}; diff --git a/apps/docs/lib/geistdocs/adapters-source.ts b/apps/docs/lib/geistdocs/adapters-source.ts index ec51700f..dfd108d2 100644 --- a/apps/docs/lib/geistdocs/adapters-source.ts +++ b/apps/docs/lib/geistdocs/adapters-source.ts @@ -21,3 +21,29 @@ export const getAdapterPageImage = (page: AdapterPage) => { : `/og/${segments.join("/")}`, }; }; + +export const getAdapterLLMText = async (page: AdapterPage, body?: string) => { + const content = body ?? (await page.data.getText("processed")); + const { title, description } = page.data; + const { packageName, tagline } = page.data as unknown as { + packageName?: string; + tagline?: string; + }; + + const frontmatter = [ + "---", + `title: ${title}`, + description && `description: ${description}`, + tagline && `tagline: ${tagline}`, + packageName && `package: ${packageName}`, + "---", + ] + .filter(Boolean) + .join("\n"); + + return `${frontmatter} + +# ${title} + +${content}`; +}; diff --git a/apps/docs/proxy.ts b/apps/docs/proxy.ts index bc192262..6977881b 100644 --- a/apps/docs/proxy.ts +++ b/apps/docs/proxy.ts @@ -13,6 +13,11 @@ const { rewrite: rewriteLLM } = rewritePath( `/${i18n.defaultLanguage}/llms.mdx/*path` ); +const { rewrite: rewriteAdaptersLLM } = rewritePath( + "/adapters/*path", + `/${i18n.defaultLanguage}/adapters.mdx/*path` +); + const MDX_EXTENSION_PATTERN = /\.mdx?$/; const internationalizer = createI18nMiddleware(i18n); @@ -20,11 +25,11 @@ const internationalizer = createI18nMiddleware(i18n); const proxy = (request: NextRequest, context: NextFetchEvent) => { const pathname = request.nextUrl.pathname; - // Track llms.txt requests - if (pathname === "/llms.txt") { + // Track llms.txt / llms-full.txt requests + if (pathname === "/llms.txt" || pathname === "/llms-full.txt") { context.waitUntil( trackMdRequest({ - path: "/llms.txt", + path: pathname, userAgent: request.headers.get("user-agent"), referer: request.headers.get("referer"), acceptHeader: request.headers.get("accept"), @@ -57,9 +62,29 @@ const proxy = (request: NextRequest, context: NextFetchEvent) => { } } + // Handle .md/.mdx URL requests for adapter pages before i18n runs + if ( + pathname.startsWith("/adapters/") && + (pathname.endsWith(".md") || pathname.endsWith(".mdx")) + ) { + const stripped = pathname.replace(MDX_EXTENSION_PATTERN, ""); + const result = rewriteAdaptersLLM(stripped); + if (result) { + context.waitUntil( + trackMdRequest({ + path: pathname, + userAgent: request.headers.get("user-agent"), + referer: request.headers.get("referer"), + acceptHeader: request.headers.get("accept"), + }) + ); + return NextResponse.rewrite(new URL(result, request.nextUrl)); + } + } + // Handle Accept header content negotiation and track the request if (isMarkdownPreferred(request)) { - const result = rewriteLLM(pathname); + const result = rewriteLLM(pathname) || rewriteAdaptersLLM(pathname); if (result) { context.waitUntil( trackMdRequest({ diff --git a/packages/integration-tests/src/docs-llms.test.ts b/packages/integration-tests/src/docs-llms.test.ts new file mode 100644 index 00000000..a1bc3f68 --- /dev/null +++ b/packages/integration-tests/src/docs-llms.test.ts @@ -0,0 +1,115 @@ +import { readFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + DOCS_CONTENT_DIR, + findDocsMdxFiles, + REPO_ROOT, +} from "./documentation-test-utils"; + +const DOCS_DIR = join(DOCS_CONTENT_DIR, "docs"); +const ADAPTERS_DIR = join(DOCS_CONTENT_DIR, "adapters"); +const PROXY_PATH = join(REPO_ROOT, "apps/docs/proxy.ts"); + +// The markdown route handler (`/adapters.mdx/[[...slug]]`), the proxy `.md` +// rewrite, the per-page `sr-only` markdown links + `text/markdown` alternates, +// and the `llms.txt` section builder all assume these exact group directories. +const ADAPTER_GROUPS = ["official", "community", "vendor-official"] as const; + +const FRONTMATTER_BLOCK = /^---\r?\n([\s\S]*?)\r?\n---/; +const FIELD_LINE = /^([a-zA-Z][a-zA-Z0-9_]*):\s*(.*)$/; +const NEWLINE = /\r?\n/; +const QUOTES = /^["']|["']$/g; +const MDX_EXTENSION = /\.mdx?$/; +const ADAPTER_MARKDOWN_URL = + /^\/adapters\/(official|community|vendor-official)\/[a-z0-9]+(?:-[a-z0-9]+)*\.md$/; + +const parseFrontmatter = (content: string): Record => { + const fields: Record = {}; + const match = content.match(FRONTMATTER_BLOCK); + if (!match) { + return fields; + } + for (const line of match[1].split(NEWLINE)) { + const fieldMatch = line.match(FIELD_LINE); + if (fieldMatch) { + const [, key, value] = fieldMatch; + fields[key] = value.trim().replace(QUOTES, ""); + } + } + return fields; +}; + +describe("Docs pages power the llms.txt index", () => { + const docs = findDocsMdxFiles(DOCS_DIR); + + it("discovers documentation pages", () => { + expect(docs.length).toBeGreaterThan(0); + }); + + for (const doc of docs) { + describe(doc.name, () => { + const fields = parseFrontmatter(readFileSync(doc.path, "utf-8")); + + it("has a title (used for , OG title, and llms.txt label)", () => { + expect( + fields.title, + `${doc.name}: missing frontmatter "title"` + ).toBeTruthy(); + }); + + it("has a description (used for metadata and the llms.txt entry)", () => { + expect( + fields.description, + `${doc.name}: missing frontmatter "description"` + ).toBeTruthy(); + }); + }); + } +}); + +describe("Adapter markdown URL scheme", () => { + it("content groups match the route segments exactly", () => { + const present = ADAPTER_GROUPS.filter( + (group) => findDocsMdxFiles(join(ADAPTERS_DIR, group)).length > 0 + ); + expect([...present].sort()).toEqual([...ADAPTER_GROUPS].sort()); + }); + + for (const group of ADAPTER_GROUPS) { + describe(group, () => { + const files = findDocsMdxFiles(join(ADAPTERS_DIR, group)); + + it("contains at least one adapter", () => { + expect(files.length).toBeGreaterThan(0); + }); + + for (const file of files) { + const slug = basename(file.path).replace(MDX_EXTENSION, ""); + + it(`${slug} maps to a well-formed markdown URL`, () => { + const markdownPath = `/adapters/${group}/${slug}.md`; + expect(markdownPath).toMatch(ADAPTER_MARKDOWN_URL); + }); + } + }); + } +}); + +describe("Adapter Accept-header markdown negotiation", () => { + const proxy = readFileSync(PROXY_PATH, "utf-8"); + + // `rewritePath().rewrite` returns `string | false`. With `??`, a `false` + // result from the docs rewriter does NOT fall through to the adapters + // rewriter, so `Accept: text/markdown` on an adapter page (e.g. + // /adapters/official/slack) would keep serving HTML. It must use `||`. + it("falls through to the adapters rewriter with `||`", () => { + expect(proxy).toContain( + "rewriteLLM(pathname) || rewriteAdaptersLLM(pathname)" + ); + }); + + it("does not regress to `??` for the rewriter fallback", () => { + expect(proxy).not.toContain("rewriteLLM(pathname) ?? rewriteAdaptersLLM"); + }); +});