feat: OpenTelemetry integration (#14)

* feat: OTLP integration

* feat: OTLP integration

* feat: OTLP integration

* feat: OTLP integration

* feat: OTLP integration

* feat: OTLP integration

* feat: OTLP integration

* feat: OTLP integration

* feat: OTLP integration

* feat: OTLP integration

* feat: OTLP integration

* feat: OTLP integration

* feat: OTLP integration

Author: hibare

Makefile

@@ -300,8 +300,8 @@ dev-init:
 .PHONY: dev-deps
 dev-deps:
 	@echo "Starting service dependencies..."
-	$(COMPOSE_CMD) up -d postgres redis redis-insight localstack adminer
-	@echo "Dependencies started (postgres:5432, redis:6379, redis-insight:5540, localstack/dynamodb:4566, adminer:8081)"
+	$(COMPOSE_CMD) up -d postgres redis redis-insight localstack adminer aspire-dashboard
+	@echo "Dependencies started (postgres:5432, redis:6379, redis-insight:5540, localstack/dynamodb:4566, adminer:8081, aspire-dashboard:18888)"
 
 .PHONY: dev
 dev: dev-deps

README.md

@@ -10,13 +10,14 @@ Modern CI/CD pipelines and automation workflows frequently require access to Git
 - [2. Key capabilities](#key-capabilities)
 - [3. Getting started](#getting-started)
 - [4. Configuration](#configuration)
-- [5. API reference](#api-reference)
-- [6. Development](#development)
-- [7. Deployment](#deployment)
-- [8. FIPS 140-3 compliance](#fips-140-3-compliance)
-- [9. Security considerations](#security-considerations)
-- [10. Contributing](#contributing)
-- [11. License](#license)
+- [5. Observability](#observability)
+- [6. API reference](#api-reference)
+- [7. Development](#development)
+- [8. Deployment](#deployment)
+- [9. FIPS 140-3 compliance](#fips-140-3-compliance)
+- [10. Security considerations](#security-considerations)
+- [11. Contributing](#contributing)
+- [12. License](#license)
 
 ## How it works
 
@@ -241,6 +242,48 @@ origin:
 
 The shared secret is injected via the `GATE_ORIGIN_HEADER_VALUE` environment variable. See the [Helm chart documentation](deployments/helm/gate/README.md) for Kubernetes configuration and the [Terraform modules](deployments/terraform/modules/aws/README.md) for CloudFront setup.
 
+## Observability
+
+GATE has built-in OpenTelemetry instrumentation for traces, metrics, and logs, exported via OTLP/gRPC. It is **disabled by default**; enable it via the `otel:` config block or `GATE_OTEL_*` environment variables.
+
+### What is instrumented
+
+- **HTTP server**: every request to `/api/v1/*` produces a span (`otelhttp.NewMiddleware`) with the matched chi route as `http.route`.
+- **Outbound HTTP**: GitHub API and OIDC discovery/JWKS calls.
+- **Database**: PostgreSQL (via `gorm.io/plugin/opentelemetry/tracing`) and DynamoDB (via `otelaws`).
+- **Redis**: traces and metrics via `redisotel`.
+- **Manual spans**: `TokenExchange` (handler), and `ValidateOIDC`, `EvaluatePolicy`, `SelectApp`, `MintInstallationToken` (service).
+- **Metrics**: `token_exchange_total{outcome}`, `token_exchange_duration_seconds`, `token_issued_total{repository}`, `caller_exchange_total{sub,issuer,outcome}`, plus Go runtime metrics.
+- **Logs**: when enabled, slog records are exported via OTLP **in addition to** stdout. Records always carry `trace_id` and `span_id` when emitted inside an active span.
+
+### Configuration
+
+| Key | Env var | Default | Notes |
+|---|---|---|---|
+| `otel.enabled` | `GATE_OTEL_ENABLED` | `false` | Master switch. |
+| `otel.service_name` | `GATE_OTEL_SERVICE_NAME` | `gate` | Becomes `service.name` resource attr. |
+| `otel.endpoint` | `GATE_OTEL_ENDPOINT` | `localhost:4317` | OTLP/gRPC collector. |
+| `otel.protocol` | `GATE_OTEL_PROTOCOL` | `grpc` | Only `grpc` supported. |
+| `otel.insecure` | `GATE_OTEL_INSECURE` | `false` | Disable TLS to collector. Set to `true` for local development. |
+| `otel.sample_rate` | `GATE_OTEL_SAMPLE_RATE` | `1.0` | Trace sampler ratio (0.0-1.0). |
+| `otel.exporter_timeout` | `GATE_OTEL_EXPORTER_TIMEOUT` | `10s` | OTLP exporter timeout per batch flush. |
+
+### Local development
+
+The local compose stack ships with [Microsoft Aspire Dashboard](https://learn.microsoft.com/en-us/dotnet/aspire/fundamentals/dashboard/overview) — a single image that bundles an OTLP receiver and a web UI for browsing traces, metrics, and logs.
+
+```bash
+# Start the dashboard (OTLP/gRPC on :4317, UI on :18888)
+docker compose -f local/compose.dev.yml up aspire-dashboard -d
+
+# Run gate with OTel env vars sourced
+set -a; source local/otel.env; set +a
+go run . server -c config.yaml
+
+# Open the dashboard
+open http://localhost:18888
+```
+
 ## API reference
 
 ### Exchange endpoint

cmd/server/handlers/exchange.go

@@ -17,13 +17,22 @@ package handlers
 
 import (
 	"errors"
+	"fmt"
+	"log/slog"
 	"net/http"
 	"strconv"
+	"strings"
+	"time"
 
 	"github.com/ggicci/httpin"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/go-chi/render"
 	"github.com/thomsonreuters/gate/internal/sts"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/trace"
 )
 
 // ExchangeInput is the httpin-decoded input for the exchange endpoint.
@@ -33,7 +42,12 @@ type ExchangeInput struct {
 
 // ExchangeHandler handles POST /api/v1/exchange requests.
 type ExchangeHandler struct {
-	service sts.Exchanger
+	service             sts.Exchanger
+	tracer              trace.Tracer
+	exchangeCount       metric.Int64Counter
+	exchangeDuration    metric.Float64Histogram
+	tokenIssuedCount    metric.Int64Counter
+	callerExchangeCount metric.Int64Counter
 }
 
 // ErrorResponse is the JSON body returned for all exchange failures.
@@ -44,12 +58,43 @@ type ErrorResponse struct {
 	RetryAfterSeconds int    `json:"retry_after_seconds,omitempty"`
 }
 
-// maxBodySize is the maximum request body size in bytes.
-const maxBodySize = 1 << 20 // 1 MB
-
 // NewExchangeHandler creates an ExchangeHandler with the given STS service.
-func NewExchangeHandler(service sts.Exchanger) *ExchangeHandler {
-	return &ExchangeHandler{service: service}
+// It pulls a tracer and meters from the global OTel providers.
+func NewExchangeHandler(service sts.Exchanger) (*ExchangeHandler, error) {
+	meter := otel.GetMeterProvider().Meter("exchange")
+	counter, err := meter.Int64Counter("token_exchange_total",
+		metric.WithDescription("Token exchange requests by outcome"))
+	if err != nil {
+		return nil, fmt.Errorf("registering token exchange counter: %w", err)
+	}
+
+	histogram, err := meter.Float64Histogram("token_exchange_duration_seconds",
+		metric.WithUnit("s"),
+		metric.WithDescription("Token exchange handler latency in seconds"))
+	if err != nil {
+		return nil, fmt.Errorf("registering token exchange duration histogram: %w", err)
+	}
+
+	tokenIssued, err := meter.Int64Counter("token_issued_total",
+		metric.WithDescription("Tokens issued by repository"))
+	if err != nil {
+		return nil, fmt.Errorf("registering token issued counter: %w", err)
+	}
+
+	callerExchange, err := meter.Int64Counter("caller_exchange_total",
+		metric.WithDescription("Exchange attempts by caller identity and outcome"))
+	if err != nil {
+		return nil, fmt.Errorf("registering caller exchange counter: %w", err)
+	}
+
+	return &ExchangeHandler{
+		service:             service,
+		tracer:              otel.GetTracerProvider().Tracer("exchange"),
+		exchangeCount:       counter,
+		exchangeDuration:    histogram,
+		tokenIssuedCount:    tokenIssued,
+		callerExchangeCount: callerExchange,
+	}, nil
 }
 
 // httpStatusCode maps STS error codes to HTTP status codes for API responses.
@@ -77,43 +122,89 @@ func httpStatusCode(code string) int {
 
 // Exchange delegates to the STS service and writes the response.
 func (h *ExchangeHandler) Exchange(w http.ResponseWriter, r *http.Request) {
-	r.Body = http.MaxBytesReader(w, r.Body, maxBodySize)
 	requestID := middleware.GetReqID(r.Context())
+	ctx, span := h.tracer.Start(r.Context(), "TokenExchange")
+	defer span.End()
+
+	start := time.Now()
+	outcome := "ok"
+	var callerIssuer, callerSubject string
+
+	defer func() {
+		r := recover()
+		if r != nil {
+			outcome = "panic"
+			span.SetStatus(codes.Error, "handler panic")
+			span.SetAttributes(attribute.String("panic.value", fmt.Sprintf("%v", r)))
+			slog.ErrorContext(ctx, "Exchange handler panic", slog.Any("panic", r))
+		}
+		h.exchangeDuration.Record(ctx, time.Since(start).Seconds())
+		h.exchangeCount.Add(ctx, 1, metric.WithAttributes(attribute.String("outcome", outcome)))
+		if callerIssuer != "" || callerSubject != "" {
+			h.callerExchangeCount.Add(ctx, 1, metric.WithAttributes(
+				attribute.String("sub", callerSubject),
+				attribute.String("issuer", callerIssuer),
+				attribute.String("outcome", outcome),
+			))
+		}
+		if r != nil {
+			panic(r)
+		}
+	}()
 
 	input, ok := r.Context().Value(httpin.Input).(*ExchangeInput)
 	if !ok {
-		render.Status(r, http.StatusBadRequest)
-		render.JSON(w, r, &ErrorResponse{Code: sts.ErrInvalidRequest, Message: "Invalid request", RequestID: requestID})
+		outcome = h.writeError(w, r, span, sts.ErrInvalidRequest, "Invalid request", requestID, 0, nil)
 		return
 	}
+	span.SetAttributes(attribute.String("repository", input.Body.TargetRepository))
 
-	resp, err := h.service.Exchange(r.Context(), requestID, &input.Body)
+	response, err := h.service.Exchange(ctx, requestID, &input.Body)
 	if err != nil {
-		var exchangeErr *sts.ExchangeError
-		if errors.As(err, &exchangeErr) {
-			status := httpStatusCode(exchangeErr.Code)
-			if exchangeErr.Code == sts.ErrRateLimited && exchangeErr.RetryAfterSeconds > 0 {
-				w.Header().Set("Retry-After", strconv.Itoa(exchangeErr.RetryAfterSeconds))
-			}
-			render.Status(r, status)
-			render.JSON(w, r, &ErrorResponse{
-				Code:              exchangeErr.Code,
-				Message:           exchangeErr.Message,
-				RequestID:         exchangeErr.RequestID,
-				RetryAfterSeconds: exchangeErr.RetryAfterSeconds,
-			})
+		var exErr *sts.ExchangeError
+		if errors.As(err, &exErr) {
+			callerIssuer = exErr.Issuer
+			callerSubject = exErr.Subject
+			outcome = h.writeError(w, r, span, exErr.Code, exErr.Message, exErr.RequestID, exErr.RetryAfterSeconds, err)
 			return
 		}
-
-		render.Status(r, http.StatusInternalServerError)
-		render.JSON(w, r, &ErrorResponse{
-			Code:      sts.ErrInternalError,
-			Message:   "Internal server error",
-			RequestID: requestID,
-		})
+		outcome = h.writeError(w, r, span, sts.ErrInternalError, "Internal server error", requestID, 0, err)
 		return
 	}
 
+	callerIssuer = response.Issuer
+	callerSubject = response.Subject
+	h.tokenIssuedCount.Add(ctx, 1, metric.WithAttributes(
+		attribute.String("repository", input.Body.TargetRepository),
+	))
+
+	span.SetAttributes(
+		attribute.String("matched_policy", response.MatchedPolicy),
+	)
+
 	render.Status(r, http.StatusOK)
-	render.JSON(w, r, resp)
+	render.JSON(w, r, response)
+}
+
+// writeError renders a JSON error response, records the failure on the
+// span, and returns the metric outcome label (lowercased error code).
+func (h *ExchangeHandler) writeError(w http.ResponseWriter, r *http.Request, span trace.Span,
+	code, message, requestID string, retryAfter int, err error) string {
+	if err != nil {
+		span.RecordError(err)
+	}
+	span.SetStatus(codes.Error, message)
+
+	if code == sts.ErrRateLimited && retryAfter > 0 {
+		w.Header().Set("Retry-After", strconv.Itoa(retryAfter))
+	}
+
+	render.Status(r, httpStatusCode(code))
+	render.JSON(w, r, &ErrorResponse{
+		Code:              code,
+		Message:           message,
+		RequestID:         requestID,
+		RetryAfterSeconds: retryAfter,
+	})
+	return strings.ToLower(code)
 }

cmd/server/handlers/exchange_test.go

@@ -42,7 +42,8 @@ func (m *mockService) Exchange(_ context.Context, _ string, _ *sts.ExchangeReque
 
 func TestNewExchangeHandler(t *testing.T) {
 	t.Parallel()
-	h := NewExchangeHandler(&mockService{})
+	h, hErr := NewExchangeHandler(&mockService{})
+	require.NoError(t, hErr)
 	require.NotNil(t, h)
 }
 
@@ -63,7 +64,8 @@ func TestHTTPStatusCode(t *testing.T) {
 
 func TestExchange_MissingInput(t *testing.T) {
 	t.Parallel()
-	h := NewExchangeHandler(&mockService{})
+	h, hErr := NewExchangeHandler(&mockService{})
+	require.NoError(t, hErr)
 
 	req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/api/v1/exchange", nil)
 	w := httptest.NewRecorder()
@@ -89,7 +91,8 @@ func TestExchange_Success(t *testing.T) {
 			RequestID:     "req-123",
 		},
 	}
-	h := NewExchangeHandler(svc)
+	h, hErr := NewExchangeHandler(svc)
+	require.NoError(t, hErr)
 
 	input := &ExchangeInput{
 		Body: sts.ExchangeRequest{
@@ -167,7 +170,8 @@ func TestExchange_ExchangeError(t *testing.T) {
 			t.Parallel()
 
 			svc := &mockService{err: tt.err}
-			h := NewExchangeHandler(svc)
+			h, hErr := NewExchangeHandler(svc)
+			require.NoError(t, hErr)
 
 			input := &ExchangeInput{
 				Body: sts.ExchangeRequest{OIDCToken: "tok", TargetRepository: "org/repo"},
@@ -200,7 +204,8 @@ func TestExchange_RateLimitedWithRetryAfter(t *testing.T) {
 			RetryAfterSeconds: 60,
 		},
 	}
-	h := NewExchangeHandler(svc)
+	h, hErr := NewExchangeHandler(svc)
+	require.NoError(t, hErr)
 
 	input := &ExchangeInput{
 		Body: sts.ExchangeRequest{OIDCToken: "tok", TargetRepository: "org/repo"},
@@ -220,7 +225,8 @@ func TestExchange_NonExchangeError(t *testing.T) {
 	t.Parallel()
 
 	svc := &mockService{err: errors.New("unexpected")}
-	h := NewExchangeHandler(svc)
+	h, hErr := NewExchangeHandler(svc)
+	require.NoError(t, hErr)
 
 	input := &ExchangeInput{
 		Body: sts.ExchangeRequest{OIDCToken: "tok", TargetRepository: "org/repo"},

cmd/server/middlewares/otel.go

@@ -0,0 +1,43 @@
+// Copyright 2026 Thomson Reuters
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package middlewares
+
+import (
+	"net/http"
+
+	"github.com/go-chi/chi/v5"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	"go.opentelemetry.io/otel/attribute"
+)
+
+// ChiRouteLabeler attaches the matched chi route pattern as the
+// "http.route" attribute on the active otelhttp span. It must run after
+// otelhttp.NewMiddleware so the labeler is present in the request context.
+func ChiRouteLabeler(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		next.ServeHTTP(w, r)
+		rctx := chi.RouteContext(r.Context())
+		if rctx == nil {
+			return
+		}
+		route := rctx.RoutePattern()
+		if route == "" {
+			return
+		}
+		if labeler, ok := otelhttp.LabelerFromContext(r.Context()); ok {
+			labeler.Add(attribute.String("http.route", route))
+		}
+	})
+}