add certificate rotation e2e test.

Signed-off-by: Morven Cao <lcao@redhat.com>

Author: Morven Cao

Makefile

@@ -108,6 +108,9 @@ readiness_probe_init_delay_seconds ?= 20
 subscription_type ?= shared
 agent_topic ?= "\$$share/statussubscribers/sources/maestro/consumers/+/agentevents"
 
+# default client certificate refresh/reload duration for message broker
+broker_client_cert_refresh_duration ?= 5m
+
 # Prints a list of useful targets.
 help:
 	@echo ""
@@ -329,6 +332,7 @@ cmds:
 		--param="READINESS_PROBE_INIT_DELAY_SECONDS"=$(readiness_probe_init_delay_seconds) \
 		--param="SUBSCRIPTION_TYPE"=$(subscription_type) \
 		--param="AGENT_TOPIC"=$(agent_topic) \
+		--param="BROKER_CLIENT_CERT_REFRESH_DURATION"=$(broker_client_cert_refresh_duration) \
 	> "templates/$*-template.json"
 
 

README.md

@@ -354,8 +354,6 @@ maestro-grpc          NodePort    10.96.114.161   <none>        8090:30090/TCP
 maestro-healthcheck   ClusterIP   10.96.67.145    <none>        8083/TCP         7m42s
 maestro-metrics       ClusterIP   10.96.201.253   <none>        8080/TCP         7m42s
 maestro-mqtt          ClusterIP   10.96.241.85    <none>        1883/TCP         7m59s
-maestro-mqtt-agent    ClusterIP   10.96.215.21    <none>        1883/TCP         7m59s
-maestro-mqtt-server   ClusterIP   10.96.72.129    <none>        1883/TCP         7m59s
 $ kubectl get pods -n maestro
 NAME                             READY   STATUS    RESTARTS   AGE
 maestro-85c847764-4xdt6          1/1     Running   0          5m

templates/agent-template.yml

@@ -63,12 +63,16 @@ parameters:
 - name: MQTT_ROOT_CERT
   description: Root Certificate for the mqtt broker.
 
-- name: MQTT_CLENT_CERT
+- name: MQTT_CLIENT_CERT
   description: Client certificate to access the mqtt broker.
 
-- name: MQTT_CLENT_KEY
+- name: MQTT_CLIENT_KEY
   description: Client private key to access the mqtt broker.
 
+- name: BROKER_CLIENT_CERT_REFRESH_DURATION
+  description: client certificate refresh duration for message broker
+  value: "5m"
+
 objects:
 - apiVersion: apiextensions.k8s.io/v1
   kind: CustomResourceDefinition
@@ -303,6 +307,8 @@ objects:
           env:
             - name: "MAESTRO_ENV"
               value: "${ENVIRONMENT}"
+            - name: CERT_CALLBACK_REFRESH_DURATION
+              value: "${BROKER_CLIENT_CERT_REFRESH_DURATION}"
           command:
             - /usr/local/bin/maestro
             - agent
@@ -337,8 +343,8 @@ objects:
       username: ${MQTT_USER}
       password: ${MQTT_PASSWORD}
       caFile: ${MQTT_ROOT_CERT}
-      clientCertFile: ${MQTT_CLENT_CERT}
-      clientKeyFile: ${MQTT_CLENT_KEY}
+      clientCertFile: ${MQTT_CLIENT_CERT}
+      clientKeyFile: ${MQTT_CLIENT_KEY}
       topics:
         sourceEvents: sources/maestro/consumers/${CONSUMER_NAME}/sourceevents
         agentEvents: sources/maestro/consumers/${CONSUMER_NAME}/agentevents

templates/agent-tls-template.yml

@@ -63,12 +63,16 @@ parameters:
 - name: MQTT_ROOT_CERT
   description: Root Certificate for the mqtt broker.
 
-- name: MQTT_CLENT_CERT
+- name: MQTT_CLIENT_CERT
   description: Client certificate to access the mqtt broker.
 
-- name: MQTT_CLENT_KEY
+- name: MQTT_CLIENT_KEY
   description: Client private key to access the mqtt broker.
 
+- name: BROKER_CLIENT_CERT_REFRESH_DURATION
+  description: client certificate refresh duration for message broker
+  value: "5m"
+
 objects:
 - apiVersion: apiextensions.k8s.io/v1
   kind: CustomResourceDefinition
@@ -303,6 +307,8 @@ objects:
           env:
             - name: "MAESTRO_ENV"
               value: "${ENVIRONMENT}"
+            - name: CERT_CALLBACK_REFRESH_DURATION
+              value: "${BROKER_CLIENT_CERT_REFRESH_DURATION}"
           command:
             - /usr/local/bin/maestro
             - agent
@@ -353,8 +359,8 @@ objects:
       username: ${MQTT_USER}
       password: ${MQTT_PASSWORD}
       caFile: ${MQTT_ROOT_CERT}
-      clientCertFile: ${MQTT_CLENT_CERT}
-      clientKeyFile: ${MQTT_CLENT_KEY}
+      clientCertFile: ${MQTT_CLIENT_CERT}
+      clientKeyFile: ${MQTT_CLIENT_KEY}
       topics:
         sourceEvents: sources/maestro/consumers/${CONSUMER_NAME}/sourceevents
         agentEvents: sources/maestro/consumers/${CONSUMER_NAME}/agentevents

templates/mqtt-tls-template.yml

@@ -72,34 +72,6 @@ objects:
     status:
       loadBalancer: {}
 
-  - apiVersion: v1
-    kind: Service
-    metadata:
-      name: ${MQTT_BROKER_NAME}-server
-    spec:
-      ports:
-      - name: mosquitto
-        port: 1883
-        protocol: TCP
-        targetPort: 1883
-      selector:
-        name: ${MQTT_BROKER_NAME}
-      type: ClusterIP
-
-  - apiVersion: v1
-    kind: Service
-    metadata:
-      name: ${MQTT_BROKER_NAME}-agent
-    spec:
-      ports:
-      - name: mosquitto
-        port: 1883
-        protocol: TCP
-        targetPort: 1883
-      selector:
-        name: ${MQTT_BROKER_NAME}
-      type: ClusterIP
-
   - kind: Deployment
     apiVersion: apps/v1
     metadata: