feat: add Prometheus metrics for operator reconciliation

CONTRIBUTING.md

@@ -4,3 +4,15 @@ Please do! Thank you for your help in improving Meshery! :balloon:
 
  Find the complete set of contributor guides at https://docs.meshery.io/project/contributing
 
+
+## Running Tests Locally
+
+```bash
+# Install envtest binaries and run tests
+make test
+
+# Run tests with coverage
+make coverage
+```
+
+If you see `etcd not found in PATH`, ensure you run `make test` (which handles envtest setup automatically) rather than `go test ./...` directly.

Makefile

@@ -261,9 +261,14 @@ catalog-push: ## Push a catalog image.
 	$(MAKE) docker-push IMG=$(CATALOG_IMG)
 
 # Test coverage
+# Run tests with envtest binaries
+.PHONY: test
+test: envtest
+	KUBEBUILDER_ASSETS="$(shell $(BIN_DIR)/setup-envtest use $(ENVTEST_K8S_VERSION) --bin-dir $(BIN_DIR) -p path)" go test ./... -coverprofile cover.out
+
 .PHONY: coverage
-coverage: test-env
-	go test -v ./... -coverprofile cover.out
+coverage: envtest
+	KUBEBUILDER_ASSETS="$(shell $(BIN_DIR)/setup-envtest use $(ENVTEST_K8S_VERSION) --bin-dir $(BIN_DIR) -p path)" go test -v ./... -coverprofile cover.out
 	go tool cover -html=cover.out -o cover.html
 
 .PHONY: kind
@@ -284,9 +289,8 @@ $(BIN_DIR)/setup-envtest-$(SETUP_ENVTEST_VERSION):
 
 # Setting test envrioment
 .PHONY: test-env
-test-env:
-	make bin/setup-envtest
-	bin/setup-envtest use $(ENVTEST_K8S_VERSION) --bin-dir $(BIN_DIR)
+test-env: envtest
+	@echo "Test environment ready. Use 'make test' to run tests."
 
 ##@ Integration Tests
 

controllers/broker_controller.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"time"
 
+	"github.com/meshery/meshery-operator/pkg/metrics"
+
 	"github.com/go-logr/logr"
 	mesheryv1alpha1 "github.com/meshery/meshery-operator/api/v1alpha1"
 	brokerpackage "github.com/meshery/meshery-operator/pkg/broker"
@@ -57,27 +59,42 @@ const (
 
 // Reconcile is the main reconciliation loop
 func (r *BrokerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	start := time.Now()
+	var reconcileErr error
+	var result ctrl.Result
+
+	defer func() {
+		metrics.ReconcileTotal.WithLabelValues("broker").Inc()
+		metrics.ReconcileDuration.WithLabelValues("broker").Observe(time.Since(start).Seconds())
+		if reconcileErr != nil {
+			metrics.ReconcileErrors.WithLabelValues("broker").Inc()
+		}
+	}()
+
 	log := r.Log.WithValues("controller", "Broker", "namespace", req.NamespacedName)
 	log.Info("Reconciling broker")
 
 	baseResource := &mesheryv1alpha1.Broker{}
 	err := r.Get(ctx, req.NamespacedName, baseResource)
 	if err != nil {
-		return r.handleGetError(log, err)
+		result, reconcileErr = r.handleGetError(log, err)
+		return result, reconcileErr
 	}
 
 	// resource deletion
 	if baseResource.GetDeletionTimestamp() != nil {
-		return r.handleDeletion(ctx, log, baseResource)
+		result, reconcileErr = r.handleDeletion(ctx, log, baseResource)
+		return result, reconcileErr
 	}
 
 	// finalizer exists
-	if result, err := r.ensureFinalizer(ctx, log, baseResource); err != nil || result.RequeueAfter > 0 {
-		return result, err
+	if result, reconcileErr = r.ensureFinalizer(ctx, log, baseResource); reconcileErr != nil || result.RequeueAfter > 0 {
+		return result, reconcileErr
 	}
 
 	// main reconciliation
-	return r.performReconciliation(ctx, log, baseResource, req)
+	result, reconcileErr = r.performReconciliation(ctx, log, baseResource, req)
+	return result, reconcileErr
 }
 
 // handleGetError handles errors from fetching the broker resource

controllers/meshsync_controller.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"time"
 
+	"github.com/meshery/meshery-operator/pkg/metrics"
+
 	"github.com/go-logr/logr"
 	mesheryv1alpha1 "github.com/meshery/meshery-operator/api/v1alpha1"
 	brokerpackage "github.com/meshery/meshery-operator/pkg/broker"
@@ -57,6 +59,18 @@ const (
 
 // Reconcile reconciles the MeshSync resource
 func (r *MeshSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	start := time.Now()
+	var reconcileErr error
+	var result ctrl.Result
+
+	defer func() {
+		metrics.ReconcileTotal.WithLabelValues("meshsync").Inc()
+		metrics.ReconcileDuration.WithLabelValues("meshsync").Observe(time.Since(start).Seconds())
+		if reconcileErr != nil {
+			metrics.ReconcileErrors.WithLabelValues("meshsync").Inc()
+		}
+	}()
+
 	log := r.Log
 	log = log.WithValues("controller", "MeshSync")
 	log = log.WithValues("namespace", req.NamespacedName)
@@ -66,17 +80,20 @@ func (r *MeshSyncReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 	// Check if resource exists
 	err := r.Get(ctx, req.NamespacedName, baseResource)
 	if err != nil {
-		return r.handleGetError(log, err)
+		result, reconcileErr = r.handleGetError(log, err)
+		return result, reconcileErr
 	}
 	// resource deletion
 	if baseResource.GetDeletionTimestamp() != nil {
-		return r.handleDeletion(ctx, log, baseResource)
+		result, reconcileErr = r.handleDeletion(ctx, log, baseResource)
+		return result, reconcileErr
 	}
 
-	if result, err := r.ensureFinalizer(ctx, log, baseResource); err != nil || result.RequeueAfter > 0 {
-		return result, err
+	if result, reconcileErr = r.ensureFinalizer(ctx, log, baseResource); reconcileErr != nil || result.RequeueAfter > 0 {
+		return result, reconcileErr
 	}
-	return r.performReconciliation(ctx, log, baseResource, req)
+	result, reconcileErr = r.performReconciliation(ctx, log, baseResource, req)
+	return result, reconcileErr
 
 }
 

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/meshery/meshkit v0.8.64
 	github.com/onsi/ginkgo/v2 v2.28.1
 	github.com/onsi/gomega v1.39.0
+	github.com/prometheus/client_golang v1.23.2
 	k8s.io/api v0.35.0
 	k8s.io/apiextensions-apiserver v0.35.0
 	k8s.io/apimachinery v0.35.0
@@ -95,6 +96,7 @@ require (
 	github.com/jmoiron/sqlx v1.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.18.2 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/lib/pq v1.10.9 // indirect
@@ -119,7 +121,6 @@ require (
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.23.2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.4 // indirect
 	github.com/prometheus/procfs v0.19.2 // indirect

pkg/metrics/metrics.go

@@ -0,0 +1,35 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+var (
+	ReconcileTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "meshery_operator_reconcile_total",
+			Help: "Total number of reconciliations",
+		},
+		[]string{"controller"},
+	)
+	ReconcileErrors = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "meshery_operator_reconcile_errors_total",
+			Help: "Total number of reconciliation errors",
+		},
+		[]string{"controller"},
+	)
+	ReconcileDuration = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "meshery_operator_reconcile_duration_seconds",
+			Help:    "Reconciliation duration in seconds",
+			Buckets: prometheus.DefBuckets,
+		},
+		[]string{"controller"},
+	)
+)
+
+func init() {
+	metrics.Registry.MustRegister(ReconcileTotal, ReconcileErrors, ReconcileDuration)
+}

pkg/metrics/metrics_test.go

@@ -0,0 +1,22 @@
+package metrics
+
+import (
+	"testing"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestReconcileTotal(t *testing.T) {
+	ReconcileTotal.WithLabelValues("broker").Inc()
+	count := testutil.CollectAndCount(ReconcileTotal)
+	if count != 1 {
+		t.Errorf("Expected 1 metric, got %d", count)
+	}
+}
+
+func TestReconcileErrors(t *testing.T) {
+	ReconcileErrors.WithLabelValues("broker").Inc()
+	count := testutil.CollectAndCount(ReconcileErrors)
+	if count != 1 {
+		t.Errorf("Expected 1 metric, got %d", count)
+	}
+}