Feature/usm client metrics enabled

docs/VARIABLES.md

@@ -7132,6 +7132,14 @@ Default:  {}
 
 ***
 
+### usm_client_metrics_enabled
+
+Boolean to enable USM client metrics for Kafka Broker and Kafka Controller (KRaft). Requires USM Agent to be configured. Not supported for Kafka Connect.
+
+Default:  false
+
+***
+
 ### unified_stream_manager
 
 Unified Stream Manager configuration for remote Schema Registry connection

docs/sample_inventories/usm_agent/README_USM_AGENT.md

@@ -108,6 +108,14 @@ This document explains the variables used in the USM Agent sample inventories fo
 - No client credentials needed (certificate-based)
 - SSL certificates required for both client and server
 
+## USM Client Metrics
+
+### `usm_client_metrics_enabled`
+- **Default**: `false`
+- **Purpose**: Enables external client metrics collection for USM
+- **Affected Components**: Kafka Broker, Kafka Controller (KRaft)
+- **Not Supported**: Kafka Connect
+
 ## Usage
 
 ```bash

molecule/scram-rhel/molecule.yml

@@ -229,10 +229,14 @@ provisioner:
       kafka_controller:
         kafka_controller_kerberos_principal: "kafka-a/{{inventory_hostname}}.confluent@{{kerberos.realm | upper}}"
         kafka_controller_kerberos_keytab_path: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/keytabs/kafka_controller-{{inventory_hostname}}.keytab"
+        # Enable USM Client Metrics for Controller
+        usm_client_metrics_enabled: true
 
       kafka_broker:
         kafka_broker_kerberos_principal: "kafka-a/{{inventory_hostname}}.confluent@{{kerberos.realm | upper}}"
         kafka_broker_kerberos_keytab_path: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}/keytabs/kafka_broker-{{inventory_hostname}}.keytab"
+        # Enable USM Client Metrics for Broker
+        usm_client_metrics_enabled: true
 
       kerberos_server:
         realm_name: "{{ kerberos.realm | upper }}"

molecule/verify_usm_client_metrics.yml

@@ -0,0 +1,118 @@
+---
+### Verify playbook for USM Client Metrics properties
+
+- name: Verify USM Client Metrics - Kafka Controller
+  hosts: kafka_controller
+  gather_facts: false
+  tasks:
+    - name: Import variables role
+      import_role:
+        name: variables
+
+    - name: Verify USM Client Metrics - usm.push.enabled (Controller)
+      import_role:
+        name: confluent.test
+        tasks_from: check_property.yml
+      vars:
+        file_path: /etc/controller/server.properties
+        property: confluent.telemetry.external.client.metrics.usm.push.enabled
+        expected_value: "true"
+      when: usm_client_metrics_enabled | default(false)
+
+    - name: Verify USM Client Metrics - push.enabled (Controller)
+      import_role:
+        name: confluent.test
+        tasks_from: check_property.yml
+      vars:
+        file_path: /etc/controller/server.properties
+        property: confluent.telemetry.external.client.metrics.push.enabled
+        expected_value: "true"
+      when: usm_client_metrics_enabled | default(false)
+
+    - name: Verify USM Client Metrics - delta.temporality (Controller)
+      import_role:
+        name: confluent.test
+        tasks_from: check_property.yml
+      vars:
+        file_path: /etc/controller/server.properties
+        property: confluent.telemetry.external.client.metrics.delta.temporality
+        expected_value: "true"
+      when: usm_client_metrics_enabled | default(false)
+
+    - name: Verify USM Client Metrics - subscription.interval.ms.list (Controller)
+      import_role:
+        name: confluent.test
+        tasks_from: check_property.yml
+      vars:
+        file_path: /etc/controller/server.properties
+        property: confluent.telemetry.external.client.metrics.subscription.interval.ms.list
+        expected_value: "60000"
+      when: usm_client_metrics_enabled | default(false)
+
+    - name: Verify USM Client Metrics - subscription.metrics.list (Controller)
+      import_role:
+        name: confluent.test
+        tasks_from: check_property.yml
+      vars:
+        file_path: /etc/controller/server.properties
+        property: confluent.telemetry.external.client.metrics.subscription.metrics.list
+        expected_value: "org.apache.kafka.consumer.fetch.manager.fetch.latency.avg,org.apache.kafka.consumer.connection.creation.total,org.apache.kafka.consumer.fetch.manager.fetch.total,org.apache.kafka.consumer.fetch.manager.bytes.consumed.rate,org.apache.kafka.consumer.request.total,org.apache.kafka.producer.bufferpool.wait.ratio,org.apache.kafka.producer.record.queue.time.avg,org.apache.kafka.producer.request.latency.avg,org.apache.kafka.producer.produce.throttle.time.avg,org.apache.kafka.producer.connection.creation.total,org.apache.kafka.producer.request.total,org.apache.kafka.producer.topic.byte.rate"
+      when: usm_client_metrics_enabled | default(false)
+
+- name: Verify USM Client Metrics - Kafka Broker
+  hosts: kafka_broker
+  gather_facts: false
+  tasks:
+    - name: Import variables role
+      import_role:
+        name: variables
+
+    - name: Verify USM Client Metrics - usm.push.enabled (Broker)
+      import_role:
+        name: confluent.test
+        tasks_from: check_property.yml
+      vars:
+        file_path: /etc/kafka/server.properties
+        property: confluent.telemetry.external.client.metrics.usm.push.enabled
+        expected_value: "true"
+      when: usm_client_metrics_enabled | default(false)
+
+    - name: Verify USM Client Metrics - push.enabled (Broker)
+      import_role:
+        name: confluent.test
+        tasks_from: check_property.yml
+      vars:
+        file_path: /etc/kafka/server.properties
+        property: confluent.telemetry.external.client.metrics.push.enabled
+        expected_value: "true"
+      when: usm_client_metrics_enabled | default(false)
+
+    - name: Verify USM Client Metrics - delta.temporality (Broker)
+      import_role:
+        name: confluent.test
+        tasks_from: check_property.yml
+      vars:
+        file_path: /etc/kafka/server.properties
+        property: confluent.telemetry.external.client.metrics.delta.temporality
+        expected_value: "true"
+      when: usm_client_metrics_enabled | default(false)
+
+    - name: Verify USM Client Metrics - subscription.interval.ms.list (Broker)
+      import_role:
+        name: confluent.test
+        tasks_from: check_property.yml
+      vars:
+        file_path: /etc/kafka/server.properties
+        property: confluent.telemetry.external.client.metrics.subscription.interval.ms.list
+        expected_value: "60000"
+      when: usm_client_metrics_enabled | default(false)
+
+    - name: Verify USM Client Metrics - subscription.metrics.list (Broker)
+      import_role:
+        name: confluent.test
+        tasks_from: check_property.yml
+      vars:
+        file_path: /etc/kafka/server.properties
+        property: confluent.telemetry.external.client.metrics.subscription.metrics.list
+        expected_value: "org.apache.kafka.consumer.fetch.manager.fetch.latency.avg,org.apache.kafka.consumer.connection.creation.total,org.apache.kafka.consumer.fetch.manager.fetch.total,org.apache.kafka.consumer.fetch.manager.bytes.consumed.rate,org.apache.kafka.consumer.request.total,org.apache.kafka.producer.bufferpool.wait.ratio,org.apache.kafka.producer.record.queue.time.avg,org.apache.kafka.producer.request.latency.avg,org.apache.kafka.producer.produce.throttle.time.avg,org.apache.kafka.producer.connection.creation.total,org.apache.kafka.producer.request.total,org.apache.kafka.producer.topic.byte.rate"
+      when: usm_client_metrics_enabled | default(false)

roles/variables/defaults/main.yml

@@ -3429,6 +3429,10 @@ usm_agent_ccloud_ssl_trusted_ca_location: "{{ '/etc/pki/ca-trust/extracted/pem/t
 ### USM Agent custom properties
 usm_agent_custom_properties: {}
 
+### Enable USM client metrics for Kafka Broker and KRaft Controller.
+### Requires usm_agent to be configured. Not supported for Connect.
+usm_client_metrics_enabled: false
+
 ### Unified Stream Manager configuration for remote Schema Registry connection
 unified_stream_manager:
   schema_registry_endpoint: none

roles/variables/tasks/main.yml

@@ -23,3 +23,21 @@
 
 - name: Ensure old and new mTLS variables are consistent
   include_tasks: mtls.yml
+
+- name: Validate that usm_client_metrics_enabled is not enabled for Connect
+  assert:
+    that:
+      - not usm_client_metrics_enabled
+    fail_msg: |
+      ERROR: usm_client_metrics_enabled is not supported for Connect.
+      This field is only supported for Kafka Broker and Kafka Controller (KRaft).
+      Please set usm_client_metrics_enabled: false or remove it from your Connect configuration.
+    success_msg: "USM client metrics validation passed for Connect"
+  when:
+    - usm_client_metrics_enabled is defined
+    - usm_client_metrics_enabled
+    - "'kafka_connect' in groups"
+    - inventory_hostname in groups.kafka_connect
+  tags:
+    - validation
+    - usm_client_metrics_validation

roles/variables/vars/main.yml

@@ -533,6 +533,14 @@ kafka_controller_properties:
       confluent.telemetry.exporter._usm.events.https.ssl.keystore.location: "{{ kafka_controller_keystore_path }}"
       confluent.telemetry.exporter._usm.events.https.ssl.keystore.password: "{{ kafka_controller_keystore_storepass }}"
       confluent.telemetry.exporter._usm.events.https.ssl.key.password: "{{ kafka_controller_keystore_keypass }}"
+  usm_client_metrics:
+    enabled: "{{ 'usm_agent' in groups and usm_client_metrics_enabled }}"
+    properties:
+      confluent.telemetry.external.client.metrics.usm.push.enabled: "true"
+      confluent.telemetry.external.client.metrics.push.enabled: "true"
+      confluent.telemetry.external.client.metrics.delta.temporality: "true"
+      confluent.telemetry.external.client.metrics.subscription.interval.ms.list: "60000"
+      confluent.telemetry.external.client.metrics.subscription.metrics.list: "org.apache.kafka.consumer.fetch.manager.fetch.latency.avg,org.apache.kafka.consumer.connection.creation.total,org.apache.kafka.consumer.fetch.manager.fetch.total,org.apache.kafka.consumer.fetch.manager.bytes.consumed.rate,org.apache.kafka.consumer.request.total,org.apache.kafka.producer.bufferpool.wait.ratio,org.apache.kafka.producer.record.queue.time.avg,org.apache.kafka.producer.request.latency.avg,org.apache.kafka.producer.produce.throttle.time.avg,org.apache.kafka.producer.connection.creation.total,org.apache.kafka.producer.request.total,org.apache.kafka.producer.topic.byte.rate"
 
 kafka_controller_combined_properties: "{{kafka_controller_properties | confluent.platform.combine_properties}}"
 
@@ -1152,6 +1160,14 @@ kafka_broker_properties:
       confluent.telemetry.exporter._usm.https.ssl.keystore.location: "{{ kafka_broker_keystore_path }}"
       confluent.telemetry.exporter._usm.https.ssl.keystore.password: "{{ kafka_broker_keystore_storepass }}"
       confluent.telemetry.exporter._usm.https.ssl.key.password: "{{ kafka_broker_keystore_keypass }}"
+  usm_client_metrics:
+    enabled: "{{ 'usm_agent' in groups and usm_client_metrics_enabled }}"
+    properties:
+      confluent.telemetry.external.client.metrics.usm.push.enabled: "true"
+      confluent.telemetry.external.client.metrics.push.enabled: "true"
+      confluent.telemetry.external.client.metrics.delta.temporality: "true"
+      confluent.telemetry.external.client.metrics.subscription.interval.ms.list: "60000"
+      confluent.telemetry.external.client.metrics.subscription.metrics.list: "org.apache.kafka.consumer.fetch.manager.fetch.latency.avg,org.apache.kafka.consumer.connection.creation.total,org.apache.kafka.consumer.fetch.manager.fetch.total,org.apache.kafka.consumer.fetch.manager.bytes.consumed.rate,org.apache.kafka.consumer.request.total,org.apache.kafka.producer.bufferpool.wait.ratio,org.apache.kafka.producer.record.queue.time.avg,org.apache.kafka.producer.request.latency.avg,org.apache.kafka.producer.produce.throttle.time.avg,org.apache.kafka.producer.connection.creation.total,org.apache.kafka.producer.request.total,org.apache.kafka.producer.topic.byte.rate"
 
 kafka_broker_combined_properties: "{{kafka_broker_properties | confluent.platform.combine_properties}}"