feat: setup kafka cluster for mimir

Author: geewynn

infra/gcp/terraform/k8s-infra-prow/kafka.tf

@@ -0,0 +1,68 @@
+// Create kafka cluster with 3vcpu, 12MB memory and autorebalance
+resource "google_managed_kafka_cluster" "mimir_cluster" {
+  project    = module.project.project_id
+  cluster_id = "mimir-cluster"
+  location   = "us-central1"
+  capacity_config {
+    vcpu_count   = 3
+    memory_bytes = 12884901888
+  }
+  gcp_config {
+    access_config {
+      network_configs {
+        subnet = "projects/${module.project.project_id}/regions/us-central1/subnetworks/subnet-01"
+      }
+    }
+  }
+  rebalance_config {
+    mode = "AUTO_REBALANCE_ON_SCALE_UP"
+  }
+}
+
+// Create mimir-ingest topic with 12 partition count
+resource "google_managed_kafka_topic" "mimir_ingest" {
+  project            = module.project.project_id
+  topic_id           = "mimir-ingest"
+  cluster            = google_managed_kafka_cluster.mimir_cluster.cluster_id
+  location           = "us-central1"
+  partition_count    = 12 # must be >= ingesters per zone
+  replication_factor = 3  # Managed Kafka requires >= 3
+
+  // Set max byte to 16MB because mimir default is 15.2MB
+  configs = {
+    "max.message.bytes" = "16000000"
+  }
+}
+
+
+resource "google_service_account" "mimir_kafka" {
+  project      = module.project.project_id
+  account_id   = "mimir-kafka"
+  display_name = "Mimir Kafka Client"
+}
+
+resource "google_service_account_key" "mimir_kafka" {
+  service_account_id = google_service_account.mimir_kafka.id
+}
+
+resource "google_project_iam_member" "mimir_kafka" {
+  project = module.project.project_id
+  role    = "roles/managedkafka.client"
+  member  = "serviceAccount:${google_service_account.mimir_kafka.email}"
+}
+
+resource "google_secret_manager_secret" "mimir_kafka_credentials" {
+  project   = module.project.project_id
+  secret_id = "mimir-kafka-credentials"
+  replication {
+    auto {}
+  }
+}
+
+resource "google_secret_manager_secret_version" "mimir_kafka_credentials" {
+  secret = google_secret_manager_secret.mimir_kafka_credentials.id
+  secret_data = jsonencode({
+    username = google_service_account.mimir_kafka.email
+    password = google_service_account_key.mimir_kafka.private_key
+  })
+}

infra/gcp/terraform/k8s-infra-prow/main.tf

@@ -41,6 +41,7 @@ module "project" {
     "certificatemanager.googleapis.com",
     "artifactregistry.googleapis.com",
     "secretmanager.googleapis.com",
-    "cloudbuild.googleapis.com"
+    "cloudbuild.googleapis.com",
+    "managedkafka.googleapis.com"
   ]
 }

kubernetes/gke-utility/mimir/external-secrets.yaml

@@ -12,3 +12,26 @@ spec:
   secretStoreRef:
     kind: ClusterSecretStore
     name: k8s-infra-prow
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: mimir-kafka-credentials
+  namespace: mimir
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: k8s-infra-prow
+    kind: ClusterSecretStore
+  target:
+    name: mimir-kafka-credentials
+    creationPolicy: Owner
+  data:
+    - secretKey: KAFKA_SASL_USERNAME
+      remoteRef:
+        key: mimir-kafka-credentials
+        property: username
+    - secretKey: KAFKA_SASL_PASSWORD
+      remoteRef:
+        key: mimir-kafka-credentials
+        property: password