feat: add OTLP/JSON (http/json) export support

exporter/otlp-common/lib/opentelemetry/exporter/otlp/common.rb

@@ -26,10 +26,19 @@ module Common # rubocop:disable Metrics/ModuleLength
         # @param [Enumerable<OpenTelemetry::SDK::Trace::SpanData>] span_data the
         #   list of recorded {OpenTelemetry::SDK::Trace::SpanData} structs to be
         #   encoded.
+        # @param [Symbol] format the wire format to encode to, either +:protobuf+
+        #   (default, protobuf-encoded) or +:json+ (spec-compliant OTLP/JSON with
+        #   hex ids and integer enums).
         #
         # @return [String] returns an encoded ETSR of the provided span data
-        def as_encoded_etsr(span_data)
-          Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.encode(as_etsr(span_data))
+        def as_encoded_etsr(span_data, format: :protobuf)
+          etsr = as_etsr(span_data, format:)
+          if format == :json
+            # Spec-compliant OTLP/JSON: hex ids (see #format_id) and integer enums.
+            etsr.to_json(format_enums_as_integers: true)
+          else
+            Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.encode(etsr)
+          end
         rescue StandardError => e
           OpenTelemetry.handle_error(exception: e, message: 'unexpected error in OTLP::Common#as_encoded_etsr')
           nil
@@ -40,10 +49,13 @@ def as_encoded_etsr(span_data)
         # @param [Enumerable<OpenTelemetry::SDK::Trace::SpanData>] span_data the
         #   list of recorded {OpenTelemetry::SDK::Trace::SpanData} structs to be
         #   encoded.
+        # @param [Symbol] format the wire format the ETSR is destined for, either
+        #   +:protobuf+ (default) or +:json+. Controls how ids are formatted
+        #   (binary for protobuf, hex for JSON).
         #
         # @return [Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest]
         #   returns an ETSR of the provided span data
-        def as_etsr(span_data)
+        def as_etsr(span_data, format: :protobuf)
           Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.new(
             resource_spans: span_data
                             .group_by(&:resource)
@@ -60,7 +72,7 @@ def as_etsr(span_data)
                                                    name: il.name,
                                                    version: il.version
                                                  ),
-                                                 spans: sds.map { |sd| as_otlp_span(sd) }
+                                                 spans: sds.map { |sd| as_otlp_span(sd, format) }
                                                )
                                              end
                               )
@@ -70,36 +82,22 @@ def as_etsr(span_data)
 
         private
 
-        def as_otlp_span(span_data)
+        def as_otlp_span(span_data, format)
+          parent_span_id = span_data.parent_span_id == OpenTelemetry::Trace::INVALID_SPAN_ID ? nil : span_data.parent_span_id
           Opentelemetry::Proto::Trace::V1::Span.new(
-            trace_id: span_data.trace_id,
-            span_id: span_data.span_id,
+            trace_id: format_id(span_data.trace_id, format),
+            span_id: format_id(span_data.span_id, format),
             trace_state: span_data.tracestate.to_s,
-            parent_span_id: span_data.parent_span_id == OpenTelemetry::Trace::INVALID_SPAN_ID ? nil : span_data.parent_span_id,
+            parent_span_id: format_id(parent_span_id, format),
             name: span_data.name,
             kind: as_otlp_span_kind(span_data.kind),
             start_time_unix_nano: span_data.start_timestamp,
             end_time_unix_nano: span_data.end_timestamp,
             attributes: span_data.attributes&.map { |k, v| as_otlp_key_value(k, v) },
             dropped_attributes_count: span_data.total_recorded_attributes - span_data.attributes&.size.to_i,
-            events: span_data.events&.map do |event|
-              Opentelemetry::Proto::Trace::V1::Span::Event.new(
-                time_unix_nano: event.timestamp,
-                name: event.name,
-                attributes: event.attributes&.map { |k, v| as_otlp_key_value(k, v) }
-                # TODO: track dropped_attributes_count in Span#append_event
-              )
-            end,
+            events: span_data.events&.map { |event| as_otlp_span_event(event) },
             dropped_events_count: span_data.total_recorded_events - span_data.events&.size.to_i,
-            links: span_data.links&.map do |link|
-              Opentelemetry::Proto::Trace::V1::Span::Link.new(
-                trace_id: link.span_context.trace_id,
-                span_id: link.span_context.span_id,
-                trace_state: link.span_context.tracestate.to_s,
-                attributes: link.attributes&.map { |k, v| as_otlp_key_value(k, v) }
-                # TODO: track dropped_attributes_count in Span#trim_links
-              )
-            end,
+            links: span_data.links&.map { |link| as_otlp_span_link(link, format) },
             dropped_links_count: span_data.total_recorded_links - span_data.links&.size.to_i,
             status: span_data.status&.then do |status|
               Opentelemetry::Proto::Trace::V1::Status.new(
@@ -110,6 +108,35 @@ def as_otlp_span(span_data)
           )
         end
 
+        def as_otlp_span_event(event)
+          Opentelemetry::Proto::Trace::V1::Span::Event.new(
+            time_unix_nano: event.timestamp,
+            name: event.name,
+            attributes: event.attributes&.map { |k, v| as_otlp_key_value(k, v) }
+            # TODO: track dropped_attributes_count in Span#append_event
+          )
+        end
+
+        def as_otlp_span_link(link, format)
+          Opentelemetry::Proto::Trace::V1::Span::Link.new(
+            trace_id: format_id(link.span_context.trace_id, format),
+            span_id: format_id(link.span_context.span_id, format),
+            trace_state: link.span_context.tracestate.to_s,
+            attributes: link.attributes&.map { |k, v| as_otlp_key_value(k, v) }
+            # TODO: track dropped_attributes_count in Span#trim_links
+          )
+        end
+
+        # OTLP/JSON requires trace/span ids as hex, but protobuf JSON base64-encodes.
+        # For the JSON path, applying initial base64 decoding to the hex string
+        # yields bytes that protobuf re-encodes back into that hex string.
+        def format_id(id_bytes, format)
+          return id_bytes unless format == :json
+          return id_bytes if id_bytes.nil? || id_bytes.empty?
+
+          id_bytes.unpack1('H*').unpack1('m0')
+        end
+
         def as_otlp_status_code(code)
           case code
           when OpenTelemetry::Trace::Status::OK then Opentelemetry::Proto::Trace::V1::Status::StatusCode::STATUS_CODE_OK

exporter/otlp-common/opentelemetry-exporter-otlp-common.gemspec

@@ -26,7 +26,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 3.3'
 
   spec.add_dependency 'googleapis-common-protos-types', '~> 1.3'
-  spec.add_dependency 'google-protobuf', '~> 3.19'
+  spec.add_dependency 'google-protobuf', '>= 3.23'
   spec.add_dependency 'opentelemetry-api', '~> 1.1'
 
   if spec.respond_to?(:metadata)

exporter/otlp-common/test/opentelemetry/exporter/otlp/common/common_test.rb

@@ -45,6 +45,67 @@
         end
       end
     end
+
+    it 'encodes json with hex ids and integer enums' do
+      trace_id = OpenTelemetry::Trace.generate_trace_id
+      span_id = OpenTelemetry::Trace.generate_span_id
+      parent_span_id = OpenTelemetry::Trace.generate_span_id
+      span_data = OpenTelemetry::TestHelpers.create_span_data(
+        kind: :server,
+        status: OpenTelemetry::Trace::Status.error,
+        trace_id: trace_id,
+        span_id: span_id,
+        parent_span_id: parent_span_id
+      )
+
+      result = OpenTelemetry::Exporter::OTLP::Common.as_encoded_etsr([span_data], format: :json)
+      _(result).must_be_kind_of(String)
+
+      span = JSON.parse(result)['resourceSpans'][0]['scopeSpans'][0]['spans'][0]
+
+      # Ids must be hex, not base64.
+      _(span['traceId']).must_match(/\A[0-9a-f]{32}\z/)
+      _(span['spanId']).must_match(/\A[0-9a-f]{16}\z/)
+      _(span['parentSpanId']).must_match(/\A[0-9a-f]{16}\z/)
+      _(span['traceId']).must_equal(trace_id.unpack1('H*'))
+      _(span['spanId']).must_equal(span_id.unpack1('H*'))
+      _(span['parentSpanId']).must_equal(parent_span_id.unpack1('H*'))
+
+      # Enums must be integers, not name strings.
+      _(span['kind']).must_be_kind_of(Integer)
+      _(span['kind']).must_equal(Opentelemetry::Proto::Trace::V1::Span::SpanKind::SPAN_KIND_SERVER)
+      _(span['status']['code']).must_be_kind_of(Integer)
+      _(span['status']['code']).must_equal(Opentelemetry::Proto::Trace::V1::Status::StatusCode::STATUS_CODE_ERROR)
+    end
+
+    it 'encodes nested json link ids as hex strings' do
+      OpenTelemetry.tracer_provider = OpenTelemetry::SDK::Trace::TracerProvider.new
+      tracer = OpenTelemetry.tracer_provider.tracer('tracer', 'v0.0.1')
+
+      link_trace_id = OpenTelemetry::Trace.generate_trace_id
+      linked_span_id = OpenTelemetry::Trace.generate_span_id
+      linked = OpenTelemetry::TestHelpers.with_ids(link_trace_id, linked_span_id) { tracer.start_root_span('linked') }
+      linking_span = tracer.start_root_span('span', links: [OpenTelemetry::Trace::Link.new(linked.context)])
+      linking_span.finish
+
+      link_result = OpenTelemetry::Exporter::OTLP::Common.as_encoded_etsr([linking_span.to_span_data], format: :json)
+      link = JSON.parse(link_result)['resourceSpans'][0]['scopeSpans'][0]['spans'][0]['links'][0]
+
+      _(link['traceId']).must_match(/\A[0-9a-f]{32}\z/)
+      _(link['spanId']).must_match(/\A[0-9a-f]{16}\z/)
+      _(link['spanId']).must_equal(linked_span_id.unpack1('H*'))
+    end
+
+    it 'handles errors gracefully' do
+      OpenTelemetry::TestHelpers.with_test_logger do |log_stream|
+        span_data = OpenTelemetry::TestHelpers.create_span_data
+        Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.stub(:new, ->(*) { raise StandardError, 'boom' }) do
+          result = OpenTelemetry::Exporter::OTLP::Common.as_encoded_etsr([span_data], format: :json)
+          _(result).must_be_nil
+          _(log_stream.string).must_match(/ERROR -- : OpenTelemetry error: unexpected error in OTLP::Common#as_encoded_etsr/)
+        end
+      end
+    end
   end
 
   describe '#as_etsr' do

exporter/otlp-http/README.md

@@ -77,11 +77,14 @@ The collector exporter can be configured explicitly in code, or via environment
 | `headers:`                | `OTEL_EXPORTER_OTLP_HEADERS`                 |                                     |
 | `compression:`            | `OTEL_EXPORTER_OTLP_COMPRESSION`             | `"gzip"`                            |
 | `timeout:`                | `OTEL_EXPORTER_OTLP_TIMEOUT`                 | `10`                                |
+| `protocol:`               | `OTEL_EXPORTER_OTLP_PROTOCOL`                | `"http/protobuf"`                   |
 | `ssl_verify_mode:`        | `OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_PEER` or | `OpenSSL::SSL:VERIFY_PEER`          |
 |                           | `OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_NONE`    |                                     |
 
 `ssl_verify_mode:` parameter values should be flags for server certificate verification: `OpenSSL::SSL:VERIFY_PEER` and `OpenSSL::SSL:VERIFY_NONE` are acceptable. These values can also be set using the appropriately named environment variables as shown where `VERIFY_PEER` will take precedence over `VERIFY_NONE`.  Please see [the Net::HTTP docs](https://ruby-doc.org/stdlib-2.5.1/libdoc/net/http/rdoc/Net/HTTP.html#verify_mode) for more information about these flags.
 
+`protocol:` selects the OTLP encoding: `"http/protobuf"` (the default) sends binary Protobuf, and `"http/json"` sends spec-compliant OTLP/JSON.
+
 ## How can I get involved?
 
 The `opentelemetry-exporter-otlp-http` gem source is [on github][repo-github], along with related gems including `opentelemetry-sdk`.

exporter/otlp-http/lib/opentelemetry/exporter/otlp/http/trace_exporter.rb

@@ -37,8 +37,10 @@ def initialize(endpoint: nil,
                          ssl_verify_mode: fetch_ssl_verify_mode,
                          headers: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_TRACES_HEADERS', 'OTEL_EXPORTER_OTLP_HEADERS', default: {}),
                          compression: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_TRACES_COMPRESSION', 'OTEL_EXPORTER_OTLP_COMPRESSION', default: 'gzip'),
-                         timeout: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_TRACES_TIMEOUT', 'OTEL_EXPORTER_OTLP_TIMEOUT', default: 10))
+                         timeout: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_TRACES_TIMEOUT', 'OTEL_EXPORTER_OTLP_TIMEOUT', default: 10),
+                         protocol: OpenTelemetry::Common::Utilities.config_opt('OTEL_EXPORTER_OTLP_TRACES_PROTOCOL', 'OTEL_EXPORTER_OTLP_PROTOCOL', default: 'http/protobuf'))
             raise ArgumentError, "unsupported compression key #{compression}" unless compression.nil? || %w[gzip none].include?(compression)
+            raise ArgumentError, "unsupported protocol #{protocol}" unless %w[http/json http/protobuf].include?(protocol)
 
             @uri = prepare_endpoint(endpoint)
 
@@ -48,6 +50,8 @@ def initialize(endpoint: nil,
             @headers = prepare_headers(headers)
             @timeout = timeout.to_f
             @compression = compression
+            @format = protocol == 'http/json' ? :json : :protobuf
+            @content_type = protocol == 'http/json' ? 'application/json' : 'application/x-protobuf'
             @shutdown = false
           end
 
@@ -61,7 +65,8 @@ def initialize(endpoint: nil,
           def export(span_data, timeout: nil)
             return FAILURE if @shutdown
 
-            send_bytes(OpenTelemetry::Exporter::OTLP::Common.as_encoded_etsr(span_data), timeout: timeout)
+            bytes = OpenTelemetry::Exporter::OTLP::Common.as_encoded_etsr(span_data, format: @format)
+            send_bytes(bytes, timeout: timeout)
           end
 
           # Called when {OpenTelemetry::SDK::Trace::TracerProvider#force_flush} is called, if
@@ -135,7 +140,7 @@ def send_bytes(bytes, timeout:) # rubocop:disable Metrics/MethodLength
                        bytes
                      end
               request.body = body
-              request.add_field('Content-Type', 'application/x-protobuf')
+              request.add_field('Content-Type', @content_type)
               @headers.each { |key, value| request.add_field(key, value) }
 
               remaining_timeout = OpenTelemetry::Common::Utilities.maybe_timeout(timeout, start_time)

exporter/otlp-http/test/opentelemetry/exporter/otlp/http/trace_exporter_test.rb

@@ -71,6 +71,36 @@
       end
     end
 
+    it 'only allows http/protobuf or http/json protocol' do
+      assert_raises ArgumentError do
+        OpenTelemetry::Exporter::OTLP::HTTP::TraceExporter.new(protocol: 'grpc')
+      end
+
+      %w[http/protobuf http/json].each do |protocol|
+        exp = OpenTelemetry::Exporter::OTLP::HTTP::TraceExporter.new(protocol: protocol)
+        expected_content_type = protocol == 'http/protobuf' ? 'application/x-protobuf' : 'application/json'
+        _(exp.instance_variable_get(:@content_type)).must_equal(expected_content_type)
+      end
+
+      [
+        { envar: 'OTEL_EXPORTER_OTLP_PROTOCOL', value: 'http/protobuf' },
+        { envar: 'OTEL_EXPORTER_OTLP_PROTOCOL', value: 'http/json' },
+        { envar: 'OTEL_EXPORTER_OTLP_TRACES_PROTOCOL', value: 'http/protobuf' },
+        { envar: 'OTEL_EXPORTER_OTLP_TRACES_PROTOCOL', value: 'http/json' }
+      ].each do |example|
+        OpenTelemetry::TestHelpers.with_env(example[:envar] => example[:value]) do
+          exp = OpenTelemetry::Exporter::OTLP::HTTP::TraceExporter.new
+          expected_content_type = example[:value] == 'http/protobuf' ? 'application/x-protobuf' : 'application/json'
+          _(exp.instance_variable_get(:@content_type)).must_equal(expected_content_type)
+        end
+      end
+    end
+
+    it 'defaults to application/x-protobuf content type' do
+      exp = OpenTelemetry::Exporter::OTLP::HTTP::TraceExporter.new
+      _(exp.instance_variable_get(:@content_type)).must_equal('application/x-protobuf')
+    end
+
     it 'sets parameters from the environment' do
       exp = OpenTelemetry::TestHelpers.with_env('OTEL_EXPORTER_OTLP_ENDPOINT' => 'https://localhost:1234',
                                                 'OTEL_EXPORTER_OTLP_CERTIFICATE' => '/foo/bar',
@@ -569,6 +599,32 @@
       _(result).must_equal(success)
     end
 
+    it 'encodes as spec-compliant JSON when protocol is http/json' do
+      exp = OpenTelemetry::Exporter::OTLP::HTTP::TraceExporter.new(protocol: 'http/json', compression: 'none')
+      trace_id = OpenTelemetry::Trace.generate_trace_id
+      span_id = OpenTelemetry::Trace.generate_span_id
+      parsed = nil
+      stub_post = stub_request(:post, 'http://localhost:4318/v1/traces').to_return do |request|
+        _(request.headers['Content-Type']).must_equal('application/json')
+        parsed = JSON.parse(request.body)
+        { status: 200 }
+      end
+
+      span_data = OpenTelemetry::TestHelpers.create_span_data(kind: :server, status: OpenTelemetry::Trace::Status.error, trace_id: trace_id, span_id: span_id)
+      result = exp.export([span_data])
+
+      _(result).must_equal(success)
+      assert_requested(stub_post)
+
+      span = parsed['resourceSpans'][0]['scopeSpans'][0]['spans'][0]
+      _(span['traceId']).must_equal(trace_id.unpack1('H*'))
+      _(span['spanId']).must_equal(span_id.unpack1('H*'))
+      _(span['traceId']).must_match(/\A[0-9a-f]{32}\z/)
+      _(span['spanId']).must_match(/\A[0-9a-f]{16}\z/)
+      _(span['kind']).must_be_kind_of(Integer)
+      _(span['status']['code']).must_be_kind_of(Integer)
+    end
+
     it 'handles encoding errors with poise and grace' do
       log_stream = StringIO.new
       logger = OpenTelemetry.logger

exporter/otlp-logs/README.md

@@ -81,11 +81,14 @@ The collector exporter can be configured explicitly in code, or via environment
 | `headers:`                | `OTEL_EXPORTER_OTLP_HEADERS`                 |                                     |
 | `compression:`            | `OTEL_EXPORTER_OTLP_COMPRESSION`             | `"gzip"`                            |
 | `timeout:`                | `OTEL_EXPORTER_OTLP_TIMEOUT`                 | `10`                                |
+| `protocol:`               | `OTEL_EXPORTER_OTLP_PROTOCOL`                | `"http/protobuf"`                   |
 | `ssl_verify_mode:`        | `OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_PEER` or | `OpenSSL::SSL:VERIFY_PEER`          |
 |                           | `OTEL_RUBY_EXPORTER_OTLP_SSL_VERIFY_NONE`    |                                     |
 
 `ssl_verify_mode:` parameter values should be flags for server certificate verification: `OpenSSL::SSL:VERIFY_PEER` and `OpenSSL::SSL:VERIFY_NONE` are acceptable. These values can also be set using the appropriately named environment variables as shown where `VERIFY_PEER` will take precedence over `VERIFY_NONE`.  Please see [the Net::HTTP docs](https://ruby-doc.org/stdlib-2.7.6/libdoc/net/http/rdoc/Net/HTTP.html#verify_mode) for more information about these flags.
 
+`protocol:` selects the OTLP encoding: `"http/protobuf"` (the default) sends binary Protobuf, and `"http/json"` sends spec-compliant OTLP/JSON.
+
 ## How can I get involved?
 
 The `opentelemetry-exporter-otlp-logs` gem source is [on github][repo-github], along with related gems including `opentelemetry-logs-sdk`.