Merge pull request modelcontextprotocol#463 from koic/work_conformance_client_merged

[Conformance] Report expected client terminal failures quietly

Author: koic

conformance/client.rb

@@ -163,55 +163,67 @@ def build_provider_for(scenario, context)
 transport = MCP::Client::HTTP.new(url: server_url, oauth: oauth)
 client = MCP::Client.new(transport: transport)
 capabilities = scenario == "elicitation-sep1034-client-defaults" ? { elicitation: {} } : {}
-client.connect(
-  client_info: { name: "ruby-sdk-conformance-client", version: MCP::VERSION },
-  capabilities: capabilities,
-)
-
-case scenario
-when "initialize"
-  client.tools
-when "tools_call"
-  tools = client.tools
-  add_numbers = tools.find { |t| t.name == "add_numbers" }
-  abort("Tool add_numbers not found") unless add_numbers
-  client.call_tool(tool: add_numbers, arguments: { a: 1, b: 2 })
-when "sse-retry"
-  # SEP-1699: the server closes the tools/call SSE stream right after a priming event.
-  # The transport waits the server's `retry:` interval, reconnects with a GET carrying `Last-Event-ID`,
-  # and receives the tool result on the resumed stream; the harness verifies the reconnect,
-  # its timing, and the header.
-  tools = client.tools
-  test_reconnection = tools.find { |t| t.name == "test_reconnection" }
-  abort("Tool test_reconnection not found") unless test_reconnection
-  client.call_tool(tool: test_reconnection, arguments: {})
-when "elicitation-sep1034-client-defaults"
-  # SEP-1034: the server sends `elicitation/create` on the tools/call SSE stream with every property declaring
-  # a `default` and none required. The handler accepts and fills the omitted fields from those defaults.
-  client.on_elicitation do |params|
-    { action: "accept", content: MCP::Client::Elicitation.apply_defaults(params["requestedSchema"] || {}) }
-  end
+# The conformance harness asserts on the observed protocol exchange, not on the client's exit status,
+# and echoes the client's output only when the exit status is non-zero. Several negative auth scenarios
+# (e.g. `auth/resource-mismatch`, `auth/scope-retry-limit`) are expected to drive the client into
+# a terminal failure: the SDK correctly refuses a mismatched resource or stops after the scope step-up
+# retry limit. Report such an expected failure as a one-line reason and exit cleanly so the run stays
+# free of noise, while letting any unexpected error (a real SDK bug) still raise with a full backtrace
+# and a failing exit status.
+begin
+  client.connect(
+    client_info: { name: "ruby-sdk-conformance-client", version: MCP::VERSION },
+    capabilities: capabilities,
+  )
+
+  case scenario
+  when "initialize"
+    client.tools
+  when "tools_call"
+    tools = client.tools
+    add_numbers = tools.find { |t| t.name == "add_numbers" }
+    abort("Tool add_numbers not found") unless add_numbers
+    client.call_tool(tool: add_numbers, arguments: { a: 1, b: 2 })
+  when "sse-retry"
+    # SEP-1699: the server closes the tools/call SSE stream right after a priming event.
+    # The transport waits the server's `retry:` interval, reconnects with a GET carrying `Last-Event-ID`,
+    # and receives the tool result on the resumed stream; the harness verifies the reconnect,
+    # its timing, and the header.
+    tools = client.tools
+    test_reconnection = tools.find { |t| t.name == "test_reconnection" }
+    abort("Tool test_reconnection not found") unless test_reconnection
+    client.call_tool(tool: test_reconnection, arguments: {})
+  when "elicitation-sep1034-client-defaults"
+    # SEP-1034: the server sends `elicitation/create` on the tools/call SSE stream with every property declaring
+    # a `default` and none required. The handler accepts and fills the omitted fields from those defaults.
+    client.on_elicitation do |params|
+      { action: "accept", content: MCP::Client::Elicitation.apply_defaults(params["requestedSchema"] || {}) }
+    end
 
-  tools = client.tools
-  defaults_tool = tools.find { |t| t.name == "test_client_elicitation_defaults" }
-
-  abort("Tool test_client_elicitation_defaults not found") unless defaults_tool
-
-  client.call_tool(tool: defaults_tool, arguments: {})
-when %r|\Aauth/|
-  # Auth-only scenarios: the protocol-level checks (PRM/AS metadata, DCR, PKCE, token usage)
-  # are observed by the conformance server during `connect` and the subsequent request below.
-  # Listing tools forces a second authenticated MCP request so the bearer token usage check fires.
-  tools = client.tools
-
-  # `auth/scope-step-up` only fires its escalation 403 on `tools/call`, not `tools/list`,
-  # so the client must actually invoke a tool to drive the second authorization request
-  # the scenario asserts on.
-  if scenario == "auth/scope-step-up"
-    tool = tools.find { |t| t.name == "test-tool" } || tools.first
-    abort("No tool exposed by conformance server for #{scenario}") unless tool
-    client.call_tool(tool: tool, arguments: {})
+    tools = client.tools
+    defaults_tool = tools.find { |t| t.name == "test_client_elicitation_defaults" }
+
+    abort("Tool test_client_elicitation_defaults not found") unless defaults_tool
+
+    client.call_tool(tool: defaults_tool, arguments: {})
+  when %r|\Aauth/|
+    # Auth-only scenarios: the protocol-level checks (PRM/AS metadata, DCR, PKCE, token usage)
+    # are observed by the conformance server during `connect` and the subsequent request below.
+    # Listing tools forces a second authenticated MCP request so the bearer token usage check fires.
+    tools = client.tools
+
+    # `auth/scope-step-up` only fires its escalation 403 on `tools/call`, not `tools/list`,
+    # so the client must actually invoke a tool to drive the second authorization request
+    # the scenario asserts on.
+    if scenario == "auth/scope-step-up"
+      tool = tools.find { |t| t.name == "test-tool" } || tools.first
+      abort("No tool exposed by conformance server for #{scenario}") unless tool
+      client.call_tool(tool: tool, arguments: {})
+    end
+  else
+    abort("Unknown or unsupported scenario: #{scenario}")
   end
-else
-  abort("Unknown or unsupported scenario: #{scenario}")
+rescue MCP::Client::OAuth::Flow::AuthorizationError, MCP::Client::RequestHandlerError, Faraday::Error => e
+  warn("#{scenario}: client ended in the expected terminal failure (#{e.class}: #{e.message})")
+  exit
 end