Validate remote mcp server tools (github#223)

* Validate remote mcp server tools.

* Improve CI validation script.

* Deduplicate server handling.

Author: cmrigney

.github/workflows/ci.yaml

@@ -29,16 +29,4 @@ jobs:
 
       - name: Build and catalog changed servers
         shell: bash
-        run: |
-          set -eo pipefail
-          while IFS= read -r file; do
-            dir=$(dirname "$file")
-            name=$(basename "$dir")
-            task validate -- --name $name
-            task build -- --tools --pull-community $name
-            echo "--------------------------------"
-            task catalog -- $name
-            echo "--------------------------------"
-            cat catalogs/$name/catalog.yaml
-            echo "--------------------------------"
-          done < changed-servers.txt
+        run: ./scripts/ci-validation.sh

cmd/validate/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"image"
@@ -11,11 +12,13 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"time"
 
 	_ "image/jpeg"
 	_ "image/png"
 
 	"github.com/docker/mcp-registry/internal/licenses"
+	"github.com/docker/mcp-registry/internal/mcp"
 	"github.com/docker/mcp-registry/pkg/github"
 	"github.com/docker/mcp-registry/pkg/servers"
 	"gopkg.in/yaml.v3"
@@ -56,7 +59,7 @@ func run(name string) error {
 	if err := isRemoteValid(name); err != nil {
 		return err
 	}
-	
+
 	if err := isOAuthDynamicValid(name); err != nil {
 		return err
 	}
@@ -147,13 +150,13 @@ func IsLicenseValid(name string) error {
 	if err != nil {
 		return err
 	}
-	
+
 	// Skip license validation for remote servers without source
 	if server.Source.Project == "" {
 		fmt.Println("✅ License validation skipped (remote server)")
 		return nil
 	}
-	
+
 	repository, err := client.GetProjectRepository(ctx, server.Source.Project)
 	if err != nil {
 		return err
@@ -193,14 +196,14 @@ func isIconValid(name string) error {
 		fmt.Println("🛑 Icon is too large. It must be less than 2MB")
 		return nil
 	}
-	
+
 	// Check content type for SVG support
 	contentType := resp.Header.Get("Content-Type")
 	if contentType == "image/svg+xml" {
 		fmt.Println("✅ Icon is valid (SVG)")
 		return nil
 	}
-	
+
 	img, format, err := image.DecodeConfig(resp.Body)
 	if err != nil {
 		return err
@@ -250,10 +253,52 @@ func isRemoteValid(name string) error {
 		return fmt.Errorf("remote server transport_type must be one of: stdio, sse, streamable-http (got: %s)", server.Remote.TransportType)
 	}
 
+	if err := hasValidTools(server); err != nil {
+		return err
+	}
+
 	fmt.Println("✅ Remote is valid")
 	return nil
 }
 
+// Check that there is either a tools.json, dynamic tools, or can fetch remote tools
+func hasValidTools(server servers.Server) error {
+	defaultErr := fmt.Errorf("server must have either a tools.json, dynamic tools, or can fetch remote tools")
+
+	// Dynamic tools are valid
+	if server.Dynamic != nil && server.Dynamic.Tools {
+		fmt.Println("✅ Dynamic tools are valid")
+		return nil
+	}
+
+	// Tools.json is valid
+	tools, err := readToolsJson(server.Name)
+	if err == nil {
+		toolCount := len(tools)
+		fmt.Printf("✅ tools.json is valid. Found %d tools.\n", toolCount)
+		return nil
+	}
+	if !os.IsNotExist(err) {
+		fmt.Printf("🛑 Tools.json could not be read: %v\n", err)
+		return defaultErr
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// Remote tools are valid
+	remoteTools, err := mcp.RemoteTools(ctx, server)
+	if err != nil {
+		fmt.Printf("🛑 Remote tools could not be fetched (if auth is required, specify \ndynamic:\n  tools: true\n): %v\n", err)
+		return defaultErr
+	}
+
+	toolCount := len(remoteTools)
+
+	fmt.Printf("✅ Remote tools are valid. Found %d tools.\n", toolCount)
+	return nil
+}
+
 // check if servers with OAuth have dynamic tools enabled
 func isOAuthDynamicValid(name string) error {
 	server, err := readServerYaml(name)
@@ -284,3 +329,18 @@ func readServerYaml(name string) (servers.Server, error) {
 	}
 	return server, nil
 }
+
+func readToolsJson(name string) ([]mcp.Tool, error) {
+	path := filepath.Join("servers", name, "tools.json")
+	buf, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var tools []mcp.Tool
+	if err := json.Unmarshal(buf, &tools); err != nil {
+		return nil, err
+	}
+
+	return tools, nil
+}

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect

go.sum

@@ -53,6 +53,8 @@ github.com/google/go-github/v70 v70.0.0 h1:/tqCp5KPrcvqCc7vIvYyFYTiCGrYvaWoYMGHS
 github.com/google/go-github/v70 v70.0.0/go.mod h1:xBUZgo8MI3lUL/hwxl3hlceJW1U8MVnXP3zUyI+rhQY=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=

internal/mcp/helper.go

@@ -28,10 +28,13 @@ import (
 	"slices"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/mark3labs/mcp-go/mcp"
 
 	"github.com/docker/mcp-registry/pkg/servers"
+	mcpclient "github.com/mark3labs/mcp-go/client"
+	"github.com/mark3labs/mcp-go/client/transport"
 )
 
 func Tools(ctx context.Context, server servers.Server, pull, cleanup, debug bool) ([]Tool, error) {
@@ -167,6 +170,144 @@ func Tools(ctx context.Context, server servers.Server, pull, cleanup, debug bool
 	return list, nil
 }
 
+func RemoteTools(ctx context.Context, server servers.Server) ([]Tool, error) {
+	var c *mcpclient.Client
+	var err error
+
+	ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
+	defer cancel()
+
+	switch server.Remote.TransportType {
+	case "sse":
+		sseTransport, err := transport.NewSSE(server.Remote.URL)
+		if err != nil {
+			return nil, err
+		}
+		c = mcpclient.NewClient(sseTransport)
+
+	case "http":
+	case "streamable-http":
+		httpTransport, err := transport.NewStreamableHTTP(server.Remote.URL)
+		if err != nil {
+			return nil, err
+		}
+		c = mcpclient.NewClient(httpTransport)
+
+	default:
+		return nil, fmt.Errorf("invalid transport type: %s", server.Remote.TransportType)
+	}
+
+	err = c.Start(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer c.Close()
+
+	initRequest := mcp.InitializeRequest{}
+	initRequest.Params.ProtocolVersion = mcp.LATEST_PROTOCOL_VERSION
+	initRequest.Params.ClientInfo = mcp.Implementation{
+		Name:    "docker",
+		Version: "1.0.0",
+	}
+
+	serverInfo, err := c.Initialize(ctx, initRequest)
+	if err != nil {
+		return nil, err
+	}
+
+	if serverInfo.Capabilities.Tools == nil {
+		return nil, fmt.Errorf("tools not supported")
+	}
+
+	toolsRequest := mcp.ListToolsRequest{}
+	toolsResult, err := c.ListTools(ctx, toolsRequest)
+	if err != nil {
+		return nil, err
+	}
+
+	var list []Tool
+	for _, tool := range toolsResult.Tools {
+		var arguments []ToolArgument
+		var requiredPropertyNames []string
+		var optionalPropertyNames []string
+		for name := range tool.InputSchema.Properties {
+			if slices.Contains(tool.InputSchema.Required, name) {
+				requiredPropertyNames = append(requiredPropertyNames, name)
+			} else {
+				optionalPropertyNames = append(optionalPropertyNames, name)
+			}
+		}
+		sort.Strings(requiredPropertyNames)
+		sort.Strings(optionalPropertyNames)
+
+		propertyNames := append(requiredPropertyNames, optionalPropertyNames...)
+
+		for _, name := range propertyNames {
+			v := tool.InputSchema.Properties[name]
+
+			// Type
+			argumentType := "string"
+			rawType := v.(map[string]any)["type"]
+			if rawType != "" && rawType != nil {
+				if str, ok := rawType.(string); ok {
+					argumentType = str
+				}
+			}
+
+			// Item types
+			var items *Items
+			if argumentType == "array" {
+				itemsType := "string"
+				if rawItems, found := v.(map[string]any)["items"]; found {
+					if kv, ok := rawItems.(map[string]any); ok {
+						if rawItemsType, found := kv["type"]; found {
+							if str, ok := rawItemsType.(string); ok {
+								itemsType = str
+							}
+						}
+					}
+				}
+				items = &Items{
+					Type: itemsType,
+				}
+			}
+
+			// Description
+			desc := v.(map[string]any)["description"]
+
+			// Properties
+			arguments = append(arguments, ToolArgument{
+				Name:        name,
+				Type:        argumentType,
+				Items:       items,
+				Optional:    !slices.Contains(tool.InputSchema.Required, name),
+				Description: argumentDescription(name, desc, tool.Description),
+			})
+		}
+
+		// Annotations
+		var annotations *ToolAnnotations
+		if tool.Annotations != (mcp.ToolAnnotation{}) {
+			annotations = &ToolAnnotations{
+				Title:           tool.Annotations.Title,
+				ReadOnlyHint:    tool.Annotations.ReadOnlyHint,
+				DestructiveHint: tool.Annotations.DestructiveHint,
+				IdempotentHint:  tool.Annotations.IdempotentHint,
+				OpenWorldHint:   tool.Annotations.OpenWorldHint,
+			}
+		}
+
+		list = append(list, Tool{
+			Name:        tool.Name,
+			Description: removeArgs(tool.Description),
+			Arguments:   arguments,
+			Annotations: annotations,
+		})
+	}
+
+	return list, nil
+}
+
 func removeArgs(input string) string {
 	var result []string