Initial version of supporting metrics in MoP

README.md

@@ -25,6 +25,7 @@ MCP OAuth Proxy solves the challenge of managing multiple OAuth providers and co
 
 ## Documentation
 
+- [Grafana / Chronosphere dashboard (PromQL for `mop_*` metrics)](docs/grafana-chronosphere-dashboard.md) — includes [importable `grafana-mop-dashboard.json`](docs/grafana-mop-dashboard.json)
 - [CLAUDE.md](CLAUDE.md) - Detailed technical documentation and architecture
 - [CONTRIBUTING.md](CONTRIBUTING.md) - Development guidelines and contribution process
 - [SECURITY.md](SECURITY.md) - Security policies and vulnerability reporting

deploy/mcp-oauth-proxy/templates/Instrumentation.yaml

@@ -0,0 +1,49 @@
+{{- if .Values.otel.instrumentation.enabled }}
+apiVersion: opentelemetry.io/v1alpha1
+kind: Instrumentation
+metadata:
+  name: {{ include "mcp-oauth-proxy.fullname" . }}
+{{ include "mcp-oauth-proxy.namespace" . | indent 2 }}
+  labels:
+    {{- include "mcp-oauth-proxy.labels" . | nindent 4 }}
+spec:
+  exporter:
+    endpoint: {{ .Values.otel.collector.endpoint }}
+  propagators:
+    - tracecontext
+    - baggage
+  sampler:
+    type: parentbased_traceidratio
+    argument: "1"
+  java:
+    image: {{ .Values.otel.instrumentation.java.image }}
+    env:
+{{- if .Values.otel.instrumentation.java.disabledResourceProviders }}
+      - name: OTEL_JAVA_DISABLED_RESOURCE_PROVIDERS
+        value: {{ .Values.otel.instrumentation.java.disabledResourceProviders | quote }}
+{{- end }}
+      - name: OTEL_RESOURCE_PROVIDERS_AWS_ENABLED
+        value: {{ .Values.otel.instrumentation.resource.providers.aws.enabled | quote }}
+      - name: OTEL_SERVICE_NAME
+        value: {{ include "mcp-oauth-proxy.fullname" . }}:{{ .Chart.AppVersion }}
+      - name: OTEL_EXPORTER_OTLP_PROTOCOL
+        value: http/protobuf
+      - name: OTEL_EXPORTER_OTLP_CERTIFICATE
+        value: {{ .Values.otel.instrumentation.trustStore }}
+      - name: OTEL_JAVAAGENT_LOGGING
+        value: "application"
+{{- if .Values.otel.instrumentation.runtime.telemetry.enabled }}
+      - name: OTEL_INSTRUMENTATION_RUNTIME_TELEMETRY_EMIT_EXPERIMENTAL_TELEMETRY
+        value: {{ .Values.otel.instrumentation.runtime.telemetry.experimental.enabled | quote }}
+      - name: OTEL_INSTRUMENTATION_RUNTIME_TELEMETRY_JAVA17_ENABLED
+        value: {{ .Values.otel.instrumentation.runtime.telemetry.java17.enabled | quote }}
+      - name: OTEL_INSTRUMENTATION_RUNTIME_TELEMETRY_JAVA17_ENABLE_ALL
+        value: {{ .Values.otel.instrumentation.runtime.telemetry.java17.allEnabled | quote }}
+{{- end }}
+{{- if .Values.otel.exporter.mtls.enabled }}
+      - name: OTEL_EXPORTER_OTLP_CLIENT_KEY
+        value: {{ include "mcp-oauth-proxy.tlsKey" . }}
+      - name: OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE
+        value: {{ include "mcp-oauth-proxy.tlsCert" . }}
+{{- end }}
+{{- end }}

deploy/mcp-oauth-proxy/templates/_helpers.tpl

@@ -34,6 +34,38 @@ Create chart name and version as used by the chart label.
 {{- printf "%s:%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
+{{/*
+Return the raw namespace (forceNamespace or release namespace), matching b2b-auth0-api chart.
+*/}}
+{{- define "mcp-oauth-proxy.rawnamespace" -}}
+{{- if .Values.forceNamespace -}}
+{{ print .Values.forceNamespace }}
+{{- else -}}
+{{ print .Release.Namespace }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Namespace line for Instrumentation metadata (indent under metadata:).
+*/}}
+{{- define "mcp-oauth-proxy.namespace" -}}
+{{ printf "namespace: %s" (include "mcp-oauth-proxy.rawnamespace" .) }}
+{{- end }}
+
+{{/*
+TLS cert path for OTEL exporter mTLS (Athenz SIA layout, same as b2b-auth0-api).
+*/}}
+{{- define "mcp-oauth-proxy.tlsCert" -}}
+{{- printf "/var/lib/sia/certs/%s.%s.cert.pem" .Values.athenz.domain .Values.athenz.service -}}
+{{- end -}}
+
+{{/*
+TLS key path for OTEL exporter mTLS (Athenz SIA layout, same as b2b-auth0-api).
+*/}}
+{{- define "mcp-oauth-proxy.tlsKey" -}}
+{{- printf "/var/lib/sia/keys/%s.%s.key.pem" .Values.athenz.domain .Values.athenz.service -}}
+{{- end -}}
+
 {{/*
 Common labels
 */}}

deploy/mcp-oauth-proxy/templates/configmap.yaml

@@ -130,6 +130,15 @@ data:
               access_type: "offline"
 
       otel:
+        enabled: {{ .Values.otel.enabled | default false }}
+        metrics:
+          enabled: true
+        logs:
+          enabled: false
+          handler:
+            enabled: false
+        instrument:
+          vertx-http: false
         resource:
           attributes: service.name={{ include "mcp-oauth-proxy.serviceVersion" . }},service.instance.id=${POD_NAME},cloud.account.id=${CLOUD_ACCOUNT_ID}
         exporter:

deploy/mcp-oauth-proxy/templates/deployment.yaml

@@ -16,6 +16,9 @@ spec:
     metadata:
       annotations:
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        {{- if .Values.otel.instrumentation.enabled }}
+        instrumentation.opentelemetry.io/inject-java: {{ printf "%s/%s" .Release.Namespace (include "mcp-oauth-proxy.fullname" .) | quote }}
+        {{- end }}
       {{- with .Values.podAnnotations }}
       {{- toYaml . | nindent 8 }}
       {{- end }}

deploy/mcp-oauth-proxy/values.yaml

@@ -9,6 +9,7 @@ image:
 imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""
+forceNamespace: ""
 
 serviceAccount:
   create: true
@@ -58,6 +59,32 @@ nodeSelector: {}
 
 tolerations: []
 
+otel:
+  enabled: false
+  collector:
+    endpoint: http://otel-collector:4317
+  instrumentation:
+    enabled: false
+    java:
+      disabledResourceProviders: ""
+      image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-java:latest
+    resource:
+      providers:
+        aws:
+          enabled: "true"
+    trustStore: ""
+    runtime:
+      telemetry:
+        enabled: false
+        experimental:
+          enabled: false
+        java17:
+          enabled: false
+          allEnabled: false
+  exporter:
+    mtls:
+      enabled: false
+
 affinity:
   podAntiAffinity:
     preferredDuringSchedulingIgnoredDuringExecution:

pom.xml

@@ -75,6 +75,15 @@
       <groupId>io.quarkus</groupId>
       <artifactId>quarkus-micrometer-opentelemetry</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.quarkus</groupId>
+      <artifactId>quarkus-opentelemetry</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.yahoo.athenz</groupId>
+      <artifactId>athenz-server-common</artifactId>
+      <version>${athenz.version}</version>
+    </dependency>
     <dependency>
       <groupId>io.quarkus</groupId>
       <artifactId>quarkus-config-yaml</artifactId>
@@ -168,6 +177,11 @@
       <artifactId>aws-cryptographic-material-providers</artifactId>
       <version>${aws.cryptojava.version}</version>
     </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk-testing</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>io.quarkus</groupId>
       <artifactId>quarkus-junit5</artifactId>

src/main/java/io/athenz/mop/quarkus/OtelStartupObserver.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright The Athenz Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ */
+package io.athenz.mop.quarkus;
+
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.api.OpenTelemetry;
+import io.quarkus.runtime.StartupEvent;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.event.Observes;
+import java.lang.invoke.MethodHandles;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Confirms OpenTelemetry is available at runtime (Quarkus uses JBoss LogManager; log correlation
+ * uses {@code quarkus.otel} logs exporter when enabled, not Logback {@code OpenTelemetryAppender}).
+ */
+@ApplicationScoped
+public class OtelStartupObserver {
+
+    private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    void onStart(@Observes StartupEvent ev) {
+        OpenTelemetry otel = GlobalOpenTelemetry.get();
+        log.info("OpenTelemetry GlobalOpenTelemetry initialized: class={}", otel.getClass().getName());
+    }
+}

src/main/java/io/athenz/mop/resource/AuthorizeResource.java

@@ -25,6 +25,9 @@
 import io.athenz.mop.service.ConfigService;
 import io.athenz.mop.service.RedirectUriValidator;
 import io.athenz.mop.service.UpstreamRefreshService;
+import io.athenz.mop.telemetry.OauthClientLabel;
+import io.athenz.mop.telemetry.OauthProviderLabel;
+import io.athenz.mop.telemetry.OauthProxyMetrics;
 import io.quarkus.oidc.IdToken;
 import io.quarkus.oidc.RefreshToken;
 import io.quarkus.oidc.UserInfo;
@@ -74,6 +77,9 @@ public class AuthorizeResource extends BaseResource {
     @Inject
     UpstreamRefreshService upstreamRefreshService;
 
+    @Inject
+    OauthProxyMetrics oauthProxyMetrics;
+
     @ConfigProperty(name = "server.host", defaultValue = "localhost")
     String host;
 
@@ -92,51 +98,52 @@ public class AuthorizeResource extends BaseResource {
     @Produces(MediaType.TEXT_HTML)
     public Response authorize(@Valid @BeanParam OAuth2AuthorizationRequest request) {
         log.info("OAuth 2.1 authorization request from client: {}", request.getClientId());
+        String oauthClient = OauthClientLabel.normalize(request.getClientId());
 
         // Validate response_type (must be "code" for OAuth 2.1)
         if (!"code".equals(request.getResponseType())) {
             log.warn("Unsupported response_type: {}. OAuth 2.1 only supports 'code'", request.getResponseType());
-            return buildErrorRedirect(request.getRedirectUri(), request.getState(),
+            return recordAuthorizeRedirect(buildErrorRedirect(request.getRedirectUri(), request.getState(),
                     OAuth2ErrorResponse.ErrorCode.UNSUPPORTED_GRANT_TYPE,
-                    "OAuth 2.1 only supports response_type=code");
+                    "OAuth 2.1 only supports response_type=code"), oauthClient, false, "unsupported_grant_type");
         }
 
         // Validate code_challenge_method (must be S256, plain is deprecated)
         if (!"S256".equals(request.getCodeChallengeMethod())) {
             log.warn("Unsupported code_challenge_method: {}. OAuth 2.1 requires S256",
                     request.getCodeChallengeMethod());
-            return buildErrorRedirect(request.getRedirectUri(), request.getState(),
+            return recordAuthorizeRedirect(buildErrorRedirect(request.getRedirectUri(), request.getState(),
                     OAuth2ErrorResponse.ErrorCode.INVALID_REQUEST,
-                    "code_challenge_method must be S256 (plain is deprecated in OAuth 2.1)");
+                    "code_challenge_method must be S256 (plain is deprecated in OAuth 2.1)"), oauthClient, false, "invalid_request");
         }
 
         // Validate redirect_uri
         if (!redirectUriValidator.isValidRedirectUri(request.getRedirectUri(), request.getClientId())) {
             log.error("Invalid redirect_uri for client {}: {}", request.getClientId(), request.getRedirectUri());
             // Per RFC 6749 Section 4.1.2.1, do NOT redirect on invalid redirect_uri
-            return Response.status(Response.Status.BAD_REQUEST)
+            return recordAuthorizeRedirect(Response.status(Response.Status.BAD_REQUEST)
                     .entity(OAuth2ErrorResponse.of(
                             OAuth2ErrorResponse.ErrorCode.INVALID_REQUEST,
                             "Invalid or missing redirect_uri"))
-                    .build();
+                    .build(), oauthClient, false, "invalid_request");
         }
 
         if (request.getResource() == null || request.getResource().isEmpty()) {
             log.error("Invalid provider {} for client {}", request.getResource(), request.getClientId());
-            return Response.status(Response.Status.BAD_REQUEST)
+            return recordAuthorizeRedirect(Response.status(Response.Status.BAD_REQUEST)
                     .entity(OAuth2ErrorResponse.of(
                             OAuth2ErrorResponse.ErrorCode.INVALID_REQUEST,
                             "Invalid or missing resource"))
-                    .build();
+                    .build(), oauthClient, false, "invalid_request");
         }
 
         // Extract subject from authenticated key (OIDC IdToken)
         String subject = idToken != null ? idToken.getSubject() : accessToken.getSubject();
         if (subject == null || subject.isEmpty()) {
             log.error("Unable to extract subject from token");
-            return buildErrorRedirect(request.getRedirectUri(), request.getState(),
+            return recordAuthorizeRedirect(buildErrorRedirect(request.getRedirectUri(), request.getState(),
                     OAuth2ErrorResponse.ErrorCode.SERVER_ERROR,
-                    "Unable to determine user identity");
+                    "Unable to determine user identity"), oauthClient, false, "server_error");
         }
         ResourceMeta resourceMeta = configService.getResourceMeta(request.getResource());
 
@@ -175,11 +182,16 @@ public Response authorize(@Valid @BeanParam OAuth2AuthorizationRequest request)
                 log.error("no token found for subject: {} provider: {}", subject, resourceMeta.idpServer());
                 String redirectUri = String.format("https://%s/%s/authorize", host, resourceMeta.idpServer());
                 log.info("redirecting to {} for authorization", redirectUri);
-                return buildRedirect(redirectUri, code);
+                return recordAuthorizeRedirect(buildRedirect(redirectUri, code), oauthClient, true, null);
             }
         }
         // Build success redirect with authorization code
-        return buildSuccessRedirect(request.getRedirectUri(), code, request.getState());
+        return recordAuthorizeRedirect(buildSuccessRedirect(request.getRedirectUri(), code, request.getState()), oauthClient, true, null);
+    }
+
+    private Response recordAuthorizeRedirect(Response response, String oauthClient, boolean success, String errorType) {
+        oauthProxyMetrics.recordAuthorizeRedirect(OauthProviderLabel.normalize(providerDefault), success, errorType, oauthClient);
+        return response;
     }
 
     /**

src/main/java/io/athenz/mop/resource/RegisterResource.java

@@ -18,6 +18,8 @@
 import io.athenz.mop.model.RegisterRequest;
 import io.athenz.mop.model.RegisterResponse;
 import io.athenz.mop.service.RedirectUriValidator;
+import io.athenz.mop.telemetry.OauthClientLabel;
+import io.athenz.mop.telemetry.OauthProxyMetrics;
 import jakarta.inject.Inject;
 import jakarta.validation.Valid;
 import jakarta.ws.rs.*;
@@ -40,6 +42,9 @@ public class RegisterResource {
     @Inject
     RedirectUriValidator redirectUriValidator;
 
+    @Inject
+    OauthProxyMetrics oauthProxyMetrics;
+
     @ConfigProperty(name = "server.athenz.register.domain")
     String registerDomain;
 
@@ -55,27 +60,33 @@ public class RegisterResource {
     @Produces(MediaType.APPLICATION_JSON)
     public Response register(@Valid RegisterRequest request) {
         log.info("register call for client: {} and token subject: {}", request.clientName(), jwt.getSubject());
+        String oauthClient = OauthClientLabel.normalize(request.clientName());
         if (request.redirectUris() == null || request.redirectUris().isEmpty()) {
+            oauthProxyMetrics.recordDynamicClientRegistration(false, "invalid_request", oauthClient);
             return Response.status(Response.Status.BAD_REQUEST).entity("redirect_uris is required").build();
         }
 
         // Validate all redirect URIs using the centralized validator
         if (!redirectUriValidator.validateRedirectUris(request.redirectUris(), request.clientName())) {
             log.error("Invalid redirect_uris for client: {}, uris: {}", request.clientName(), request.redirectUris());
+            oauthProxyMetrics.recordDynamicClientRegistration(false, "invalid_redirect_uri", oauthClient);
             return Response.status(Response.Status.BAD_REQUEST)
                     .entity("redirect_uris validation failed - must use allowed prefixes and valid format")
                     .build();
         }
 
         if (request.clientName() == null || request.clientName().isEmpty()) {
+            oauthProxyMetrics.recordDynamicClientRegistration(false, "invalid_client_name", oauthClient);
             return Response.status(Response.Status.BAD_REQUEST).entity("client_name validation failed").build();
         }
 
         if (validateAttestationJwt && !isValidAttestationJwt(request)) {
+            oauthProxyMetrics.recordDynamicClientRegistration(false, "forbidden", oauthClient);
             return Response.status(Response.Status.FORBIDDEN).entity("token does not come from expected domain and/or role").build();
         }
         RegisterResponse registerResponse = new RegisterResponse(request.clientName(), request.clientName(), request.redirectUris());
         log.info("registered client: {} with redirect_uris: {}", registerResponse.clientName(), registerResponse.redirectUris());
+        oauthProxyMetrics.recordDynamicClientRegistration(true, null, oauthClient);
         return Response.ok(registerResponse).build();
     }