Pack capability sitemaps to a URL budget, not a listing count

lib/mcp_registry/registry.ex

@@ -220,12 +220,12 @@ defmodule McpRegistry.Registry do
   struct, because `article_content` alone runs to tens of kilobytes and there
   are thousands of these.
   """
-  def servers_with(kind, page, per_page) when kind in [:prompts, :resources] and page >= 1 do
+  def servers_with(kind, offset, limit) when kind in [:prompts, :resources] do
     Server
     |> having_any(kind)
     |> order_by([s], asc: s.id)
-    |> offset(^((page - 1) * per_page))
-    |> limit(^per_page)
+    |> offset(^offset)
+    |> limit(^limit)
     |> select([s], {
       struct(s, [:name, :transport, :remote_url, :package_registry, :package_identifier]),
       field(s, ^kind),
@@ -239,6 +239,49 @@ defmodule McpRegistry.Registry do
     Server |> having_any(kind) |> Repo.aggregate(:count)
   end
 
+  @doc """
+  How many listings belong in each sitemap file, so no file exceeds `budget`.
+
+  A fixed number of listings per file cannot work here. One listing has 960
+  resources and most have a handful, so a file holding 300 of them was 30,000
+  URLs in one place and 57,000 in another -- and a sitemap over 50,000 URLs is
+  rejected without a message. Packing to a URL budget is the only chunking
+  that survives a distribution that skewed.
+
+  Returns a list of listing counts, one per file, in `id` order. Cached with
+  the rest of the catalogue figures, so a sync or an approval refreshes it.
+  """
+  def capability_chunks(kind, budget) when kind in [:prompts, :resources] do
+    Cache.fetch({:capability_chunks, kind, budget}, fn ->
+      Server
+      |> having_any(kind)
+      |> order_by([s], asc: s.id)
+      |> select([s], fragment("cardinality(?)", field(s, ^kind)))
+      |> Repo.all()
+      |> pack(budget)
+    end)
+  end
+
+  # The index page, plus each item's own page and one per client. Twelve is
+  # the widest client list, so this is an upper bound rather than an estimate
+  # -- a file may come in under budget, never over it.
+  @urls_per_item 13
+
+  defp pack(sizes, budget) do
+    {full, last, _} =
+      Enum.reduce(sizes, {[], 0, 0}, fn size, {full, in_file, urls} ->
+        cost = 1 + size * @urls_per_item
+
+        if in_file > 0 and urls + cost > budget do
+          {[in_file | full], 1, cost}
+        else
+          {full, in_file + 1, urls + cost}
+        end
+      end)
+
+    Enum.reverse(if last > 0, do: [last | full], else: full)
+  end
+
   defp having_any(query, kind) do
     query
     |> where([s], s.status == "active")

lib/mcp_registry_web/controllers/sitemap_controller.ex

@@ -23,11 +23,13 @@ defmodule McpRegistryWeb.SitemapController do
   @servers_per_tool_file 150
   # Twelve or so agent pages a listing, so 3,000 listings is about 36,000 URLs.
   @servers_per_agent_file 3_000
-  # Prompts and resources run far fewer per listing than tools -- a handful
-  # rather than eighteen -- so more listings fit in a file at the same budget.
-  # Held at 300 because the distribution is skewed: most listings have a few
-  # and one has 195, and a file over 50,000 URLs is rejected in silence.
-  @servers_per_capability_file 300
+  # Prompts and resources are packed to a URL budget rather than a listing
+  # count, because the distribution is far too skewed for a count to work: one
+  # listing has 960 resources and most have a handful. At 300 listings a file
+  # that produced 30,187 URLs in one file and 56,798 in another -- and a file
+  # over 50,000 is rejected in silence. 40,000 leaves room for the estimate to
+  # be generous without approaching the limit.
+  @urls_per_capability_file 40_000
 
   # `/live` is LiveView's transport, not content. Its long-poll fallback carries
   # a fresh CSRF token in the query string, so every fetch mints a URL that has
@@ -166,8 +168,13 @@ defmodule McpRegistryWeb.SitemapController do
          true <- within(page, capability_files(kind)) do
       base = McpRegistryWeb.Endpoint.url()
 
+      chunks = Registry.capability_chunks(kind, @urls_per_capability_file)
+
       kind
-      |> Registry.servers_with(page, @servers_per_capability_file)
+      |> Registry.servers_with(
+        chunks |> Enum.take(page - 1) |> Enum.sum(),
+        Enum.at(chunks, page - 1)
+      )
       |> Enum.flat_map(fn {server, items, updated_at} ->
         clients = Clients.ids(server)
 
@@ -195,7 +202,7 @@ defmodule McpRegistryWeb.SitemapController do
   end
 
   defp capability_files(kind),
-    do: ceil(Registry.count_servers_with(kind) / @servers_per_capability_file)
+    do: length(Registry.capability_chunks(kind, @urls_per_capability_file))
 
   defp urlset(entries) do
     [

test/mcp_registry_web/live/capabilities_test.exs

@@ -173,6 +173,36 @@ defmodule McpRegistryWeb.ServerLive.CapabilitiesTest do
       refute index =~ "skills"
     end
 
+    test "files are packed so none can exceed the 50,000-URL limit", %{conn: conn} do
+      # One listing has 960 resources and most have a handful. Chunking by a
+      # fixed listing count produced a 56,798-URL file, which a search engine
+      # rejects without saying so.
+      for n <- 1..10 do
+        server_fixture(%{
+          name: "io.github.acme/bulk-#{n}",
+          resources: Enum.map(1..400, &"file:///r#{n}-#{&1}")
+        })
+      end
+
+      index = conn |> get("/sitemap.xml") |> response(200)
+      files = Regex.scan(~r{/sitemaps/(resources-\d+)\.xml}, index) |> Enum.map(&List.last/1)
+
+      # Ten listings of 400 resources is ~52,000 URLs, past both the 40,000
+      # budget and the 50,000 limit, so the packing has to split it.
+      assert length(files) > 1
+
+      for file <- files do
+        urls =
+          conn
+          |> get("/sitemaps/#{file}.xml")
+          |> response(200)
+          |> then(&Regex.scan(~r{<loc>}, &1))
+          |> length()
+
+        assert urls <= 50_000, "#{file} has #{urls} URLs, over the sitemap limit"
+      end
+    end
+
     test "nothing anywhere means no file, and a 404 rather than an empty 200", %{conn: conn} do
       server_fixture(%{prompts: [], resources: []})