Record what changes about each MCP server, and publish changelogs

config/config.exs

@@ -114,7 +114,9 @@ config :mcp_registry, :probe,
   enabled: false,
   batch_size: 50,
   concurrency: 4,
-  recheck_days: 14,
+  # Weekly, so a listing's changelog is at most a week behind its server.
+  # About 3,000 endpoints a day, against a scheduler that can ask 28 times that.
+  recheck_days: 7,
   interval_ms: :timer.minutes(5),
   initial_delay_ms: :timer.minutes(3),
   req_options: []

lib/mcp_registry/changes.ex

@@ -0,0 +1,225 @@
+defmodule McpRegistry.Changes do
+  @moduledoc """
+  What changed about a listing, recorded as it is noticed.
+
+  Two processes already re-read every listing on a schedule: the official
+  registry sync (every six hours, for `server.json`) and the prober (for the
+  tools, prompts and resources a live server reports). Both used to overwrite
+  what they found. This keeps the difference.
+
+  ## What is not a change
+
+  A changelog that reports our own bookkeeping as the publisher's activity is
+  worse than none, so most of the rules here are about what *not* to record:
+
+    * **Nothing is recorded on first sight.** A listing's first import, or a
+      server's first answer, is a baseline, not an event.
+    * **Tools changing from declared to probed is not recorded.** That is our
+      knowledge improving, not the server changing. Only probe-to-probe
+      differences count.
+    * **Prompts and resources from before #{"2026-09-24"} are not compared.**
+      Probes before then never asked for them, so an empty list then meant
+      "not asked", and comparing it to a real answer would report every
+      prompt the server has always had as newly added.
+    * **A list at the probe's 500-item cap is not compared.** When a server
+      with 1,045 resources adds one, another falls off our end of the list,
+      and that would read as a removal that never happened.
+    * **The sync's re-apply runs are not recorded.** They change how we read
+      upstream data, not the data.
+  """
+  import Ecto.Query
+
+  alias McpRegistry.Changes.Change
+  alias McpRegistry.Probe
+  alias McpRegistry.Registry.Server
+  alias McpRegistry.Repo
+
+  # When the prober started asking for prompts and resources. See moduledoc.
+  @capabilities_since ~U[2026-09-24 19:47:00Z]
+
+  # The server.json fields a reader would call a change. synced_at,
+  # source_updated_at and the like are ours, not the publisher's.
+  @tracked_fields ~w(title description version transport remote_url package_registry
+                     package_identifier repository_url website_url icon_url env_vars
+                     license status)a
+
+  @labels %{
+    "tools" => "Tools",
+    "prompts" => "Prompts",
+    "resources" => "Resources",
+    "server_json" => "server.json",
+    "llms_txt" => "llms.txt",
+    "agents_md" => "AGENTS.md"
+  }
+
+  # --- Recording -------------------------------------------------------------
+
+  @doc """
+  Records what a successful probe found that differs from what the listing
+  held before it. `before` is the row as it was; `found` is the probe's map of
+  `tools`, `prompts` and `resources`.
+  """
+  def record_probe(%Server{} = before, %{} = found) do
+    [
+      tools_change(before, found[:tools] || []),
+      list_change(before, :prompts, found[:prompts] || []),
+      list_change(before, :resources, found[:resources] || [])
+    ]
+    |> Enum.reject(&is_nil/1)
+    |> Enum.map(fn {kind, diff} -> entry(before, Atom.to_string(kind), diff, "probe") end)
+    |> insert_all()
+  end
+
+  @doc """
+  Records the publisher-visible fields a sync is about to change on an
+  existing listing, from the changeset that will write them.
+  """
+  def record_sync(%Server{id: id} = before, %Ecto.Changeset{changes: changes})
+      when not is_nil(id) do
+    fields =
+      for field <- @tracked_fields,
+          Map.has_key?(changes, field),
+          Map.get(before, field) != changes[field],
+          into: %{},
+          do: {Atom.to_string(field), [Map.get(before, field), changes[field]]}
+
+    if map_size(fields) == 0 do
+      []
+    else
+      insert_all([entry(before, "server_json", %{fields: fields}, "sync")])
+    end
+  end
+
+  def record_sync(_before, _changeset), do: []
+
+  defp tools_change(%Server{tools_source: "probed", tools: old}, [_ | _] = new),
+    do: list_entry(%{tools: old}, :tools, new)
+
+  defp tools_change(_before, _new), do: nil
+
+  defp list_change(%Server{probe_status: "ok", probed_at: %DateTime{} = at} = before, kind, new) do
+    old = Map.fetch!(before, kind)
+    cap = Probe.max_items()
+
+    cond do
+      DateTime.compare(at, @capabilities_since) == :lt -> nil
+      length(old) >= cap or length(new) >= cap -> nil
+      true -> list_entry(before, kind, new)
+    end
+  end
+
+  defp list_change(_before, _kind, _new), do: nil
+
+  # A set difference, kept in the server's own order: the order a server lists
+  # things in is usually the order its author thinks of them in.
+  defp list_entry(before, kind, new) do
+    old = Map.fetch!(before, kind)
+    old_set = MapSet.new(old)
+    new_set = MapSet.new(new)
+
+    added = Enum.reject(new, &MapSet.member?(old_set, &1))
+    removed = Enum.reject(old, &MapSet.member?(new_set, &1))
+
+    if added == [] and removed == [] do
+      nil
+    else
+      {kind, %{added: added, removed: removed}}
+    end
+  end
+
+  defp entry(%Server{id: id}, kind, attrs, source) do
+    Map.merge(
+      %{
+        server_id: id,
+        document_url: nil,
+        kind: kind,
+        added: [],
+        removed: [],
+        fields: %{},
+        source: source,
+        inserted_at: DateTime.utc_now()
+      },
+      attrs
+    )
+  end
+
+  defp insert_all([]), do: []
+
+  defp insert_all(rows) do
+    {_, inserted} = Repo.insert_all(Change, rows, returning: true)
+    inserted
+  end
+
+  # --- Reading ---------------------------------------------------------------
+
+  @doc "A listing's changes, newest first. `kind` narrows to one kind."
+  def for_server(%Server{} = server, kind \\ nil, limit \\ 200) do
+    server
+    |> subject_query()
+    |> then(fn q -> if kind, do: where(q, [c], c.kind == ^kind), else: q end)
+    |> order_by([c], desc: c.inserted_at, desc: c.id)
+    |> limit(^limit)
+    |> Repo.all()
+  end
+
+  @doc "How many changes a listing has of each kind, as `%{kind => count}`."
+  def kinds_for_server(%Server{} = server) do
+    server
+    |> subject_query()
+    |> group_by([c], c.kind)
+    |> select([c], {c.kind, count(c.id)})
+    |> Repo.all()
+    |> Map.new()
+  end
+
+  @doc "The most recent changes across the whole registry, with their listing."
+  def recent(limit \\ 100) do
+    Change
+    |> where([c], not is_nil(c.server_id))
+    |> order_by([c], desc: c.inserted_at, desc: c.id)
+    |> limit(^limit)
+    |> preload(server: ^from(s in Server, select: struct(s, [:id, :name, :title])))
+    |> Repo.all()
+  end
+
+  @doc "How many active listings have at least one recorded change."
+  def count_servers_with_changes do
+    Change
+    |> join(:inner, [c], s in Server, on: s.id == c.server_id and s.status == "active")
+    |> select([c], count(c.server_id, :distinct))
+    |> Repo.one()
+  end
+
+  @doc """
+  A page of active listings with changes, as `{name, kinds, latest}` -- the
+  shape the sitemap needs, without loading any listing in full.
+  """
+  def servers_with_changes(offset, limit) do
+    Change
+    |> join(:inner, [c], s in Server, on: s.id == c.server_id and s.status == "active")
+    |> group_by([c, s], [s.id, s.name])
+    |> order_by([c, s], asc: s.id)
+    |> offset(^offset)
+    |> limit(^limit)
+    |> select([c, s], {s.name, fragment("array_agg(DISTINCT ?)", c.kind), max(c.inserted_at)})
+    |> Repo.all()
+  end
+
+  # Extended by the document fetcher: a listing's changelog also includes the
+  # documents it points at, which are recorded against their URL.
+  defp subject_query(%Server{id: id}), do: where(Change, [c], c.server_id == ^id)
+
+  # --- Naming ----------------------------------------------------------------
+
+  @doc "How a kind is shown to a reader: `llms.txt`, `server.json`, `Tools`."
+  def label(kind), do: Map.fetch!(@labels, kind)
+
+  @doc "A kind as a URL segment: `server_json` becomes `server-json`."
+  def slug(kind), do: String.replace(kind, "_", "-")
+
+  @doc "The kind a URL segment names, or `nil`."
+  def from_slug(slug) when is_binary(slug) do
+    kind = String.replace(slug, "-", "_")
+    if Map.has_key?(@labels, kind), do: kind
+  end
+end

lib/mcp_registry/changes/change.ex

@@ -0,0 +1,24 @@
+defmodule McpRegistry.Changes.Change do
+  @moduledoc """
+  One recorded change: items added and removed from a list, or fields that
+  moved from one value to another. See `McpRegistry.Changes` for what is
+  recorded and, as importantly, what is not.
+  """
+  use Ecto.Schema
+
+  @kinds ~w(tools prompts resources server_json llms_txt agents_md)
+
+  schema "changes" do
+    belongs_to :server, McpRegistry.Registry.Server
+    field :document_url, :string
+    field :kind, :string
+    field :added, {:array, :string}, default: []
+    field :removed, {:array, :string}, default: []
+    field :fields, :map, default: %{}
+    field :source, :string
+
+    timestamps(type: :utc_datetime_usec, updated_at: false)
+  end
+
+  def kinds, do: @kinds
+end

lib/mcp_registry/discovery.ex

@@ -32,7 +32,7 @@ defmodule McpRegistry.Discovery do
   @batch_size 10_000
 
   # Pages that are not listings, for the backfill.
-  @static_paths ["/", "/servers", "/submit", "/book"]
+  @static_paths ["/", "/servers", "/submit", "/book", "/changelog"]
 
   def config, do: Application.get_env(:mcp_registry, :discovery, [])
 

lib/mcp_registry/official_registry.ex

@@ -31,6 +31,7 @@ defmodule McpRegistry.OfficialRegistry do
   require Logger
 
   alias McpRegistry.Analytics
+  alias McpRegistry.Changes
   alias McpRegistry.Discovery
   alias McpRegistry.OfficialRegistry.SyncRun
   alias McpRegistry.Registry.{Manifest, Server}
@@ -364,7 +365,7 @@ defmodule McpRegistry.OfficialRegistry do
       if reapply? and not is_nil(existing) and only_bookkeeping?(changeset) do
         :unchanged
       else
-        write(existing, changeset)
+        write(existing, changeset, reapply?)
       end
     end
   end
@@ -376,12 +377,22 @@ defmodule McpRegistry.OfficialRegistry do
     changeset.changes |> Map.drop([:synced_at, :source_updated_at]) |> map_size() == 0
   end
 
-  defp write(existing, changeset) do
+  defp write(existing, changeset, reapply?) do
     case {existing, Repo.insert_or_update(changeset)} do
-      {_, {:error, _changeset}} -> :invalid
-      {nil, {:ok, _}} -> :inserted
-      {%Server{origin: "local"}, {:ok, _}} -> :replaced_pending
-      {_, {:ok, _}} -> :updated
+      {_, {:error, _changeset}} ->
+        :invalid
+
+      {nil, {:ok, _}} ->
+        :inserted
+
+      {%Server{origin: "local"}, {:ok, _}} ->
+        :replaced_pending
+
+      {_, {:ok, _}} ->
+        # A re-apply run changes how we read upstream, not what the publisher
+        # published, so it leaves the changelog alone.
+        unless reapply?, do: Changes.record_sync(existing, changeset)
+        :updated
     end
   end
 

lib/mcp_registry/probe.ex

@@ -153,6 +153,9 @@ defmodule McpRegistry.Probe do
   @max_items 500
   @max_length 2_000
 
+  @doc "The most items of one kind a probe keeps. A list this long was cut short."
+  def max_items, do: @max_items
+
   # A resource is named by uri, a tool and a prompt by name.
   defp names(items) do
     items

lib/mcp_registry/probe/runner.ex

@@ -21,12 +21,12 @@ defmodule McpRegistry.Probe.Runner do
 
   import Ecto.Query
 
-  alias McpRegistry.{Probe, Repo}
+  alias McpRegistry.{Changes, Probe, Repo}
   alias McpRegistry.Registry.Server
 
   @default_batch 100
   @default_concurrency 6
-  @default_recheck_days 14
+  @default_recheck_days 7
 
   @doc """
   Probes one batch and returns a tally.
@@ -108,7 +108,13 @@ defmodule McpRegistry.Probe.Runner do
         # exactly the count this is being collected to settle.
         changes = Map.merge(tools, %{prompts: found.prompts, resources: found.resources})
 
-        record(server, :ok, changes)
+        # The changelog is written only once the new values are, so it can
+        # never describe a state the listing did not reach. `server` is still
+        # the row as it was before this probe.
+        with {:ok, _updated} <- record(server, :ok, changes) do
+          Changes.record_probe(server, found)
+        end
+
         :ok
 
       {:error, reason} ->
@@ -129,11 +135,11 @@ defmodule McpRegistry.Probe.Runner do
     |> Repo.update()
     |> case do
       {:ok, updated} ->
-        updated
+        {:ok, updated}
 
       {:error, changeset} ->
         Logger.warning("Probe could not record #{server.name}: #{inspect(changeset.errors)}")
-        server
+        {:error, server}
     end
   rescue
     # `Repo.update/1` returns a changeset for a validation failure but raises
@@ -143,6 +149,6 @@ defmodule McpRegistry.Probe.Runner do
     # answers is not under our control, so this must not be fatal to the rest.
     error ->
       Logger.warning("Probe could not record #{server.name}: #{Exception.message(error)}")
-      server
+      {:error, server}
   end
 end