From 0cffc605ad2c27fb7e2cd2b7b0b326d6bac2b8da Mon Sep 17 00:00:00 2001 From: Koichi ITO Date: Fri, 13 Mar 2026 03:33:13 +0900 Subject: [PATCH] Support running MCP client conformance tests against the Ruby SDK ## Motivation and Context PR #248 introduced server conformance testing using `@modelcontextprotocol/conformance`. This extends coverage to the client side so that `MCP::Client` behavior can also be validated against the MCP specification. ### Scope Client conformance is driven by the conformance runner, which spawns test servers for each scenario and invokes `conformance/client.rb` against them. The client script performs the MCP initialize handshake and executes scenario-specific operations (for example listing tools or calling a tool). Because `MCP::Client` does not yet support the full initialize handshake natively (pending #210), the client uses a lightweight `ConformanceTransport` that handles both JSON and SSE responses and performs the handshake manually. ### Usage Run all conformance tests (server + client): ```console bundle exec rake conformance ``` Run a single client scenario: ```console bundle exec rake conformance SCENARIO=initialize ``` List available scenarios: ```console bundle exec rake conformance_list ``` --- Rakefile | 7 +- conformance/README.md | 28 +++--- conformance/client.rb | 104 ++++++++++++++++++++ conformance/client_runner.rb | 48 +++++++++ conformance/expected_failures.yml | 28 ++++++ conformance/{runner.rb => server_runner.rb} | 4 +- 6 files changed, 202 insertions(+), 17 deletions(-) create mode 100644 conformance/client.rb create mode 100644 conformance/client_runner.rb rename conformance/{runner.rb => server_runner.rb} (96%) diff --git a/Rakefile b/Rakefile index 6904e571..62e92d57 100644 --- a/Rakefile +++ b/Rakefile @@ -20,7 +20,8 @@ desc "Run MCP conformance tests (PORT, SCENARIO, SPEC_VERSION, VERBOSE)" task :conformance do |t| next unless npx_available?(t.name) - require_relative "conformance/runner" + require_relative "conformance/server_runner" + require_relative "conformance/client_runner" options = {} options[:port] = Integer(ENV["PORT"]) if ENV["PORT"] @@ -28,7 +29,8 @@ task :conformance do |t| options[:spec_version] = ENV["SPEC_VERSION"] if ENV["SPEC_VERSION"] options[:verbose] = true if ENV["VERBOSE"] - Conformance::Runner.new(**options).run + Conformance::ServerRunner.new(**options).run + Conformance::ClientRunner.new(**options.except(:port)).run end desc "List available conformance scenarios" @@ -36,6 +38,7 @@ task :conformance_list do |t| next unless npx_available?(t.name) system("npx", "--yes", "@modelcontextprotocol/conformance", "list", "--server") + system("npx", "--yes", "@modelcontextprotocol/conformance", "list", "--client") end desc "Start the conformance server (PORT)" diff --git a/conformance/README.md b/conformance/README.md index 159def02..5179012f 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -15,22 +15,22 @@ Validates the Ruby SDK's conformance to the MCP specification using [`@modelcont bundle exec rake conformance ``` -Starts the conformance server, runs all active scenarios against it, prints a pass/fail -summary for each scenario, and exits with a non-zero status code if any unexpected failures -are detected. Scenarios listed in `expected_failures.yml` are allowed to fail without -affecting the exit code. +Runs both server and client conformance tests in sequence. Server conformance starts the +conformance server and tests it. Client conformance spawns test servers for each scenario +and invokes `conformance/client.rb` against them. Scenarios listed in `expected_failures.yml` +are allowed to fail without affecting the exit code. ### Environment variables -| Variable | Description | Default | -|----------------|--------------------------------------|---------| -| `PORT` | Server port | `9292` | -| `SCENARIO` | Run a single scenario by name | (all) | -| `SPEC_VERSION` | Filter scenarios by spec version | (all) | -| `VERBOSE` | Show raw JSON output when set | (off) | +| Variable | Description | Default | +|----------------|---------------------------------------|---------| +| `PORT` | Server port (server conformance only) | `9292` | +| `SCENARIO` | Run a single scenario by name | (all) | +| `SPEC_VERSION` | Filter scenarios by spec version | (all) | +| `VERBOSE` | Show raw JSON output when set | (off) | ```bash -# Run a single scenario +# Run a single server scenario bundle exec rake conformance SCENARIO=ping # Use a different port with verbose output @@ -91,8 +91,10 @@ submissions. ``` conformance/ server.rb # Conformance server (Rack + Puma, default port 9292) - runner.rb # Starts the server, runs npx conformance, exits with result code - expected_failures.yml # Baseline of known-failing scenarios + server_runner.rb # Starts the server, runs npx conformance server, exits with result code + client.rb # Conformance client (invoked by npx conformance client) + client_runner.rb # Runs npx conformance client, exits with result code + expected_failures.yml # Baseline of known-failing scenarios (server and client) README.md # This file ``` diff --git a/conformance/client.rb b/conformance/client.rb new file mode 100644 index 00000000..94d852d8 --- /dev/null +++ b/conformance/client.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +# Conformance test client for the MCP Ruby SDK. +# Invoked by the conformance runner: +# MCP_CONFORMANCE_SCENARIO= bundle exec ruby conformance/client.rb +# +# The server URL is passed as the last positional argument. +# The scenario name is read from the MCP_CONFORMANCE_SCENARIO environment variable, +# which is set automatically by the conformance test runner. + +require "net/http" +require "json" +require "securerandom" +require "uri" +require_relative "../lib/mcp" + +# A transport that handles both JSON and SSE (text/event-stream) responses. +# The standard `MCP::Client::HTTP` transport only accepts application/json, +# but the MCP `StreamableHTTPServerTransport` may return text/event-stream. +class ConformanceTransport + def initialize(url:) + @uri = URI(url) + end + + def send_request(request:) + http = Net::HTTP.new(@uri.host, @uri.port) + req = Net::HTTP::Post.new(@uri.path.empty? ? "/" : @uri.path) + req["Content-Type"] = "application/json" + req["Accept"] = "application/json, text/event-stream" + req.body = JSON.generate(request) + + response = http.request(req) + + case response.content_type + when "application/json" + JSON.parse(response.body) + when "text/event-stream" + parse_sse_response(response.body) + else + raise "Unexpected content type: #{response.content_type}" + end + end + + private + + def parse_sse_response(body) + body.each_line do |line| + next unless line.start_with?("data: ") + + data = line.delete_prefix("data: ").strip + next if data.empty? + + return JSON.parse(data) + end + nil + end +end + +scenario = ENV["MCP_CONFORMANCE_SCENARIO"] +server_url = ARGV.last + +unless scenario && server_url + abort("Usage: MCP_CONFORMANCE_SCENARIO= ruby conformance/client.rb ") +end + +# +# TODO: Once https://github.com/modelcontextprotocol/ruby-sdk/pull/210 is merged, +# replace `ConformanceTransport` and the manual initialize handshake below with: +# +# ``` +# transport = MCP::Client::HTTP.new(url: server_url) +# client = MCP::Client.new(transport: transport) +# client.connect(client_info: { ... }, protocol_version: "2025-11-25") +# ``` +# +# After that `ConformanceTransport` will be removed. +# +transport = ConformanceTransport.new(url: server_url) + +# MCP initialize handshake (the MCP::Client API does not expose this yet). +transport.send_request(request: { + jsonrpc: "2.0", + id: SecureRandom.uuid, + method: "initialize", + params: { + clientInfo: { name: "ruby-sdk-conformance-client", version: MCP::VERSION }, + protocolVersion: "2025-11-25", + capabilities: {}, + }, +}) + +client = MCP::Client.new(transport: transport) + +case scenario +when "initialize" + client.tools +when "tools_call" + tools = client.tools + add_numbers = tools.find { |t| t.name == "add_numbers" } + abort("Tool add_numbers not found") unless add_numbers + client.call_tool(tool: add_numbers, arguments: { a: 1, b: 2 }) +else + abort("Unknown or unsupported scenario: #{scenario}") +end diff --git a/conformance/client_runner.rb b/conformance/client_runner.rb new file mode 100644 index 00000000..a99fbac1 --- /dev/null +++ b/conformance/client_runner.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +# Runs `npx @modelcontextprotocol/conformance client` against the conformance client script. +require "English" + +module Conformance + class ClientRunner + def initialize(scenario: nil, spec_version: nil, verbose: false) + @scenario = scenario + @spec_version = spec_version + @verbose = verbose + end + + def run + command = build_command + puts "Command: #{command.join(" ")}\n\n" + + system(*command) + conformance_exit_code = $CHILD_STATUS.exitstatus + exit(conformance_exit_code || 1) unless conformance_exit_code == 0 + end + + private + + def build_command + expected_failures_yml = File.expand_path("expected_failures.yml", __dir__) + client_script = File.expand_path("client.rb", __dir__) + + npx_command = [ + "npx", + "--yes", + "@modelcontextprotocol/conformance", + "client", + "--command", + "bundle exec ruby #{client_script}", + ] + npx_command += if @scenario + ["--scenario", @scenario] + else + ["--suite", "all"] + end + npx_command += ["--spec-version", @spec_version] if @spec_version + npx_command += ["--verbose"] if @verbose + npx_command += ["--expected-failures", expected_failures_yml] + npx_command + end + end +end diff --git a/conformance/expected_failures.yml b/conformance/expected_failures.yml index cb7a3fa8..948fc420 100644 --- a/conformance/expected_failures.yml +++ b/conformance/expected_failures.yml @@ -3,3 +3,31 @@ server: - tools-call-elicitation - elicitation-sep1034-defaults - elicitation-sep1330-enums + +client: + # TODO: SSE reconnection not implemented in Ruby client. + - sse-retry + # TODO: Elicitation not implemented in Ruby client. + - elicitation-sep1034-client-defaults + # TODO: OAuth/auth not implemented in Ruby client. + - auth/metadata-default + - auth/metadata-var1 + - auth/metadata-var2 + - auth/metadata-var3 + - auth/basic-cimd + - auth/scope-from-www-authenticate + - auth/scope-from-scopes-supported + - auth/scope-omitted-when-undefined + - auth/scope-step-up + - auth/scope-retry-limit + - auth/token-endpoint-auth-basic + - auth/token-endpoint-auth-post + - auth/token-endpoint-auth-none + - auth/pre-registration + - auth/2025-03-26-oauth-metadata-backcompat + - auth/2025-03-26-oauth-endpoint-fallback + - auth/client-credentials-jwt + - auth/client-credentials-basic + - auth/cross-app-access-complete-flow + - auth/offline-access-scope + - auth/offline-access-not-supported diff --git a/conformance/runner.rb b/conformance/server_runner.rb similarity index 96% rename from conformance/runner.rb rename to conformance/server_runner.rb index 033a3bd6..0a4e9e23 100644 --- a/conformance/runner.rb +++ b/conformance/server_runner.rb @@ -6,7 +6,7 @@ require_relative "server" module Conformance - class Runner + class ServerRunner # Timeout for waiting for the Puma server to start. SERVER_START_TIMEOUT = 20 SERVER_POLL_INTERVAL = 0.5 @@ -83,7 +83,7 @@ def run_conformance(command, server_pid:) terminate_server(server_pid) end - exit(conformance_exit_code || 1) + exit(conformance_exit_code || 1) unless conformance_exit_code == 0 end def terminate_server(pid)