Merge pull request #293 from koic/conformance_client

Support running MCP client conformance tests against the Ruby SDK

Author: koic

Rakefile

@@ -20,22 +20,25 @@ desc "Run MCP conformance tests (PORT, SCENARIO, SPEC_VERSION, VERBOSE)"
 task :conformance do |t|
   next unless npx_available?(t.name)
 
-  require_relative "conformance/runner"
+  require_relative "conformance/server_runner"
+  require_relative "conformance/client_runner"
 
   options = {}
   options[:port] = Integer(ENV["PORT"]) if ENV["PORT"]
   options[:scenario] = ENV["SCENARIO"] if ENV["SCENARIO"]
   options[:spec_version] = ENV["SPEC_VERSION"] if ENV["SPEC_VERSION"]
   options[:verbose] = true if ENV["VERBOSE"]
 
-  Conformance::Runner.new(**options).run
+  Conformance::ServerRunner.new(**options).run
+  Conformance::ClientRunner.new(**options.except(:port)).run
 end
 
 desc "List available conformance scenarios"
 task :conformance_list do |t|
   next unless npx_available?(t.name)
 
   system("npx", "--yes", "@modelcontextprotocol/conformance", "list", "--server")
+  system("npx", "--yes", "@modelcontextprotocol/conformance", "list", "--client")
 end
 
 desc "Start the conformance server (PORT)"

conformance/README.md

@@ -15,22 +15,22 @@ Validates the Ruby SDK's conformance to the MCP specification using [`@modelcont
 bundle exec rake conformance
 ```
 
-Starts the conformance server, runs all active scenarios against it, prints a pass/fail
-summary for each scenario, and exits with a non-zero status code if any unexpected failures
-are detected. Scenarios listed in `expected_failures.yml` are allowed to fail without
-affecting the exit code.
+Runs both server and client conformance tests in sequence. Server conformance starts the
+conformance server and tests it. Client conformance spawns test servers for each scenario
+and invokes `conformance/client.rb` against them. Scenarios listed in `expected_failures.yml`
+are allowed to fail without affecting the exit code.
 
 ### Environment variables
 
-| Variable       | Description                          | Default |
-|----------------|--------------------------------------|---------|
-| `PORT`         | Server port                          | `9292`  |
-| `SCENARIO`     | Run a single scenario by name        | (all)   |
-| `SPEC_VERSION` | Filter scenarios by spec version     | (all)   |
-| `VERBOSE`      | Show raw JSON output when set        | (off)   |
+| Variable       | Description                           | Default |
+|----------------|---------------------------------------|---------|
+| `PORT`         | Server port (server conformance only) | `9292`  |
+| `SCENARIO`     | Run a single scenario by name         | (all)   |
+| `SPEC_VERSION` | Filter scenarios by spec version      | (all)   |
+| `VERBOSE`      | Show raw JSON output when set         | (off)   |
 
 ```bash
-# Run a single scenario
+# Run a single server scenario
 bundle exec rake conformance SCENARIO=ping
 
 # Use a different port with verbose output
@@ -91,8 +91,10 @@ submissions.
 ```
 conformance/
   server.rb              # Conformance server (Rack + Puma, default port 9292)
-  runner.rb              # Starts the server, runs npx conformance, exits with result code
-  expected_failures.yml  # Baseline of known-failing scenarios
+  server_runner.rb       # Starts the server, runs npx conformance server, exits with result code
+  client.rb              # Conformance client (invoked by npx conformance client)
+  client_runner.rb       # Runs npx conformance client, exits with result code
+  expected_failures.yml  # Baseline of known-failing scenarios (server and client)
   README.md              # This file
 ```
 

conformance/client.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# Conformance test client for the MCP Ruby SDK.
+# Invoked by the conformance runner:
+#   MCP_CONFORMANCE_SCENARIO=<scenario> bundle exec ruby conformance/client.rb <server-url>
+#
+# The server URL is passed as the last positional argument.
+# The scenario name is read from the MCP_CONFORMANCE_SCENARIO environment variable,
+# which is set automatically by the conformance test runner.
+
+require_relative "../lib/mcp"
+
+scenario = ENV["MCP_CONFORMANCE_SCENARIO"]
+server_url = ARGV.last
+
+unless scenario && server_url
+  abort("Usage: MCP_CONFORMANCE_SCENARIO=<scenario> ruby conformance/client.rb <server-url>")
+end
+
+transport = MCP::Client::HTTP.new(url: server_url)
+client = MCP::Client.new(transport: transport)
+client.connect(client_info: { name: "ruby-sdk-conformance-client", version: MCP::VERSION })
+
+case scenario
+when "initialize"
+  client.tools
+when "tools_call"
+  tools = client.tools
+  add_numbers = tools.find { |t| t.name == "add_numbers" }
+  abort("Tool add_numbers not found") unless add_numbers
+  client.call_tool(tool: add_numbers, arguments: { a: 1, b: 2 })
+else
+  abort("Unknown or unsupported scenario: #{scenario}")
+end

conformance/client_runner.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# Runs `npx @modelcontextprotocol/conformance client` against the conformance client script.
+require "English"
+
+module Conformance
+  class ClientRunner
+    def initialize(scenario: nil, spec_version: nil, verbose: false)
+      @scenario = scenario
+      @spec_version = spec_version
+      @verbose = verbose
+    end
+
+    def run
+      command = build_command
+      puts "Command: #{command.join(" ")}\n\n"
+
+      system(*command)
+      conformance_exit_code = $CHILD_STATUS.exitstatus
+      exit(conformance_exit_code || 1) unless conformance_exit_code == 0
+    end
+
+    private
+
+    def build_command
+      expected_failures_yml = File.expand_path("expected_failures.yml", __dir__)
+      client_script = File.expand_path("client.rb", __dir__)
+
+      npx_command = [
+        "npx",
+        "--yes",
+        "@modelcontextprotocol/conformance",
+        "client",
+        "--command",
+        "bundle exec ruby #{client_script}",
+      ]
+      npx_command += if @scenario
+        ["--scenario", @scenario]
+      else
+        ["--suite", "all"]
+      end
+      npx_command += ["--spec-version", @spec_version] if @spec_version
+      npx_command += ["--verbose"] if @verbose
+      npx_command += ["--expected-failures", expected_failures_yml]
+      npx_command
+    end
+  end
+end

conformance/expected_failures.yml

@@ -1 +1,28 @@
 server: []
+client:
+  # TODO: SSE reconnection not implemented in Ruby client.
+  - sse-retry
+  # TODO: Elicitation not implemented in Ruby client.
+  - elicitation-sep1034-client-defaults
+  # TODO: OAuth/auth not implemented in Ruby client.
+  - auth/metadata-default
+  - auth/metadata-var1
+  - auth/metadata-var2
+  - auth/metadata-var3
+  - auth/basic-cimd
+  - auth/scope-from-www-authenticate
+  - auth/scope-from-scopes-supported
+  - auth/scope-omitted-when-undefined
+  - auth/scope-step-up
+  - auth/scope-retry-limit
+  - auth/token-endpoint-auth-basic
+  - auth/token-endpoint-auth-post
+  - auth/token-endpoint-auth-none
+  - auth/pre-registration
+  - auth/2025-03-26-oauth-metadata-backcompat
+  - auth/2025-03-26-oauth-endpoint-fallback
+  - auth/client-credentials-jwt
+  - auth/client-credentials-basic
+  - auth/cross-app-access-complete-flow
+  - auth/offline-access-scope
+  - auth/offline-access-not-supported

conformance/runner.rb conformance/server_runner.rbconformance/runner.rb renamed to conformance/server_runner.rb

@@ -6,7 +6,7 @@
 require_relative "server"
 
 module Conformance
-  class Runner
+  class ServerRunner
     # Timeout for waiting for the Puma server to start.
     SERVER_START_TIMEOUT = 20
     SERVER_POLL_INTERVAL = 0.5
@@ -83,7 +83,7 @@ def run_conformance(command, server_pid:)
         terminate_server(server_pid)
       end
 
-      exit(conformance_exit_code || 1)
+      exit(conformance_exit_code || 1) unless conformance_exit_code == 0
     end
 
     def terminate_server(pid)