diff --git a/lib/mcp_registry/registry.ex b/lib/mcp_registry/registry.ex index a81eb6a..d9ddfd6 100644 --- a/lib/mcp_registry/registry.ex +++ b/lib/mcp_registry/registry.ex @@ -220,12 +220,12 @@ defmodule McpRegistry.Registry do struct, because `article_content` alone runs to tens of kilobytes and there are thousands of these. """ - def servers_with(kind, page, per_page) when kind in [:prompts, :resources] and page >= 1 do + def servers_with(kind, offset, limit) when kind in [:prompts, :resources] do Server |> having_any(kind) |> order_by([s], asc: s.id) - |> offset(^((page - 1) * per_page)) - |> limit(^per_page) + |> offset(^offset) + |> limit(^limit) |> select([s], { struct(s, [:name, :transport, :remote_url, :package_registry, :package_identifier]), field(s, ^kind), @@ -239,6 +239,49 @@ defmodule McpRegistry.Registry do Server |> having_any(kind) |> Repo.aggregate(:count) end + @doc """ + How many listings belong in each sitemap file, so no file exceeds `budget`. + + A fixed number of listings per file cannot work here. One listing has 960 + resources and most have a handful, so a file holding 300 of them was 30,000 + URLs in one place and 57,000 in another -- and a sitemap over 50,000 URLs is + rejected without a message. Packing to a URL budget is the only chunking + that survives a distribution that skewed. + + Returns a list of listing counts, one per file, in `id` order. Cached with + the rest of the catalogue figures, so a sync or an approval refreshes it. + """ + def capability_chunks(kind, budget) when kind in [:prompts, :resources] do + Cache.fetch({:capability_chunks, kind, budget}, fn -> + Server + |> having_any(kind) + |> order_by([s], asc: s.id) + |> select([s], fragment("cardinality(?)", field(s, ^kind))) + |> Repo.all() + |> pack(budget) + end) + end + + # The index page, plus each item's own page and one per client. Twelve is + # the widest client list, so this is an upper bound rather than an estimate + # -- a file may come in under budget, never over it. + @urls_per_item 13 + + defp pack(sizes, budget) do + {full, last, _} = + Enum.reduce(sizes, {[], 0, 0}, fn size, {full, in_file, urls} -> + cost = 1 + size * @urls_per_item + + if in_file > 0 and urls + cost > budget do + {[in_file | full], 1, cost} + else + {full, in_file + 1, urls + cost} + end + end) + + Enum.reverse(if last > 0, do: [last | full], else: full) + end + defp having_any(query, kind) do query |> where([s], s.status == "active") diff --git a/lib/mcp_registry_web/controllers/sitemap_controller.ex b/lib/mcp_registry_web/controllers/sitemap_controller.ex index 10ecc96..625141d 100644 --- a/lib/mcp_registry_web/controllers/sitemap_controller.ex +++ b/lib/mcp_registry_web/controllers/sitemap_controller.ex @@ -23,11 +23,13 @@ defmodule McpRegistryWeb.SitemapController do @servers_per_tool_file 150 # Twelve or so agent pages a listing, so 3,000 listings is about 36,000 URLs. @servers_per_agent_file 3_000 - # Prompts and resources run far fewer per listing than tools -- a handful - # rather than eighteen -- so more listings fit in a file at the same budget. - # Held at 300 because the distribution is skewed: most listings have a few - # and one has 195, and a file over 50,000 URLs is rejected in silence. - @servers_per_capability_file 300 + # Prompts and resources are packed to a URL budget rather than a listing + # count, because the distribution is far too skewed for a count to work: one + # listing has 960 resources and most have a handful. At 300 listings a file + # that produced 30,187 URLs in one file and 56,798 in another -- and a file + # over 50,000 is rejected in silence. 40,000 leaves room for the estimate to + # be generous without approaching the limit. + @urls_per_capability_file 40_000 # `/live` is LiveView's transport, not content. Its long-poll fallback carries # a fresh CSRF token in the query string, so every fetch mints a URL that has @@ -166,8 +168,13 @@ defmodule McpRegistryWeb.SitemapController do true <- within(page, capability_files(kind)) do base = McpRegistryWeb.Endpoint.url() + chunks = Registry.capability_chunks(kind, @urls_per_capability_file) + kind - |> Registry.servers_with(page, @servers_per_capability_file) + |> Registry.servers_with( + chunks |> Enum.take(page - 1) |> Enum.sum(), + Enum.at(chunks, page - 1) + ) |> Enum.flat_map(fn {server, items, updated_at} -> clients = Clients.ids(server) @@ -195,7 +202,7 @@ defmodule McpRegistryWeb.SitemapController do end defp capability_files(kind), - do: ceil(Registry.count_servers_with(kind) / @servers_per_capability_file) + do: length(Registry.capability_chunks(kind, @urls_per_capability_file)) defp urlset(entries) do [ diff --git a/test/mcp_registry_web/live/capabilities_test.exs b/test/mcp_registry_web/live/capabilities_test.exs index 6a96c26..4f56f1f 100644 --- a/test/mcp_registry_web/live/capabilities_test.exs +++ b/test/mcp_registry_web/live/capabilities_test.exs @@ -173,6 +173,36 @@ defmodule McpRegistryWeb.ServerLive.CapabilitiesTest do refute index =~ "skills" end + test "files are packed so none can exceed the 50,000-URL limit", %{conn: conn} do + # One listing has 960 resources and most have a handful. Chunking by a + # fixed listing count produced a 56,798-URL file, which a search engine + # rejects without saying so. + for n <- 1..10 do + server_fixture(%{ + name: "io.github.acme/bulk-#{n}", + resources: Enum.map(1..400, &"file:///r#{n}-#{&1}") + }) + end + + index = conn |> get("/sitemap.xml") |> response(200) + files = Regex.scan(~r{/sitemaps/(resources-\d+)\.xml}, index) |> Enum.map(&List.last/1) + + # Ten listings of 400 resources is ~52,000 URLs, past both the 40,000 + # budget and the 50,000 limit, so the packing has to split it. + assert length(files) > 1 + + for file <- files do + urls = + conn + |> get("/sitemaps/#{file}.xml") + |> response(200) + |> then(&Regex.scan(~r{}, &1)) + |> length() + + assert urls <= 50_000, "#{file} has #{urls} URLs, over the sitemap limit" + end + end + test "nothing anywhere means no file, and a 404 rather than an empty 200", %{conn: conn} do server_fixture(%{prompts: [], resources: []})