Refactor Performance Tests and expand Monitoring Stack (kptdev#1122)

* Refactor perfomrnace tests, add new grafana dashboards, metrics and pyroscope

Signed-off-by: Rendre Greyling <rendre.greyling@nokia.com>

* Add new Function Pod resource usage metrics + expand on the package available for perf test

Signed-off-by: Rendre Greyling <rendre.greyling@nokia.com>

* Allow for perf tests to work with both v1alpha1 and v1alpha2

Signed-off-by: Rendre Greyling <rendre.greyling@nokia.com>

* Add metrics for PR controller and Git Push

Signed-off-by: Rendre Greyling <rendre.greyling@nokia.com>

* Add Jaeger for tracing

Signed-off-by: Rendre Greyling <rendre.greyling@nokia.com>

* Some housekeeping tasks

Signed-off-by: Rendre Greyling <rendre.greyling@nokia.com>

* Address Copilot comments, add unit tests and some cleanup

Signed-off-by: Rendre Greyling <rendre.greyling@nokia.com>

* Cleanup on kptfiles and dashboards

Signed-off-by: Rendre Greyling <rendre.greyling@nokia.com>

* Fix fnrunner and controllers not setting image prefix from .env for local deployment

Signed-off-by: Rendre Greyling <rendre.greyling@nokia.com>

* Address reviews

Signed-off-by: Rendre Greyling <rendre.greyling@nokia.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

* Address further review comments

Signed-off-by: Rendre Greyling <rendre.greyling@nokia.com>

* Fix from rebase

Signed-off-by: Rendre Greyling <rendre.greyling@nokia.com>

* Address context review

Signed-off-by: Rendre Greyling <rendre.greyling@nokia.com>

---------

Signed-off-by: Rendre Greyling <rendre.greyling@nokia.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

Author: rendre-greyling

.env.template

@@ -1,3 +1,4 @@
 # PORCH_GHCR_PREFIX_URL=<custom registry>/kptdev/krm-functions-catalog
 # DOCKERHUB_MIRROR=<mirror registry url>
 # DB_MAVEN_MIRROR=<mirror repository url>
+# GRAFANA_ADMIN_PW=<password for grafana UI>

.gitignore

@@ -2,6 +2,8 @@ vendor/
 apiserver.local.config/
 /apiserver/porch
 *.log
+*.csv
+load_test_results.txt
 .env
 
 # Development artifact path
@@ -40,6 +42,9 @@ __debug*
 ### Jetbrains IDEs ###
 .idea/*
 
+### Cursor IDEs ###
+.cursor
+
 ### Hugo ###
 # Generated files by hugo
 docs/.hugo_build.lock

controllers/main.go

@@ -213,6 +213,8 @@ func newManager(scheme *runtime.Scheme) (ctrl.Manager, error) {
 
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
 		Scheme: scheme,
+		// Disable controller-runtime's default :8080 /metrics listener. Port 8080 is reserved for
+		// optional pprof (PORCH_PPROF_PORT); controller-runtime metrics are exposed on :9464 via OpenTelemetry.
 		Metrics: metricsserver.Options{
 			BindAddress: "0",
 		},

controllers/packagerevisions/pkg/controllers/packagerevision/ownership.go

@@ -17,9 +17,11 @@ package packagerevision
 import (
 	"context"
 	"fmt"
+	"time"
 
 	porchv1alpha2 "github.com/kptdev/porch/api/porch/v1alpha2"
 	configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1"
+	"github.com/kptdev/porch/internal/telemetry"
 	"github.com/kptdev/porch/pkg/repository"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -66,6 +68,9 @@ func (r *PackageRevisionReconciler) handleDeletion(ctx context.Context, pr *porc
 // shared content cache. "Not found" errors are treated as success — there
 // is nothing to clean up if the package or repo doesn't exist in the cache.
 func (r *PackageRevisionReconciler) deleteFromGit(ctx context.Context, pr *porchv1alpha2.PackageRevision) error {
+	start := time.Now()
+	defer telemetry.RecordControllerOperation(telemetry.ResourcePackageRevision, "DELETE", start)
+
 	repoKey := repository.RepositoryKey{
 		Namespace: pr.Namespace,
 		Name:      pr.Spec.RepositoryName,

controllers/packagerevisions/pkg/controllers/packagerevision/packagerevision_controller.go

@@ -21,6 +21,7 @@ import (
 
 	porchv1alpha2 "github.com/kptdev/porch/api/porch/v1alpha2"
 	"github.com/kptdev/porch/controllers/functionconfigs"
+	"github.com/kptdev/porch/internal/telemetry"
 	"github.com/kptdev/porch/pkg/repository"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -160,7 +161,9 @@ func (r *PackageRevisionReconciler) reconcileLifecycle(ctx context.Context, pr *
 
 	log.Info("lifecycle transition", "name", pr.Name, "current", current, "desired", desired)
 
+	start := time.Now()
 	updated, err := r.ContentCache.UpdateLifecycle(ctx, repoKey, pr.Spec.PackageName, pr.Spec.WorkspaceName, desired)
+	telemetry.RecordControllerOperation(telemetry.ResourcePackageRevision, "UPDATE", start)
 	if err != nil {
 		log.Error(err, "lifecycle transition failed")
 		r.updateStatus(ctx, pr, nil, "", readyCondition(pr.Generation, metav1.ConditionFalse, porchv1alpha2.ReasonFailed, err.Error()))
@@ -201,6 +204,8 @@ func (r *PackageRevisionReconciler) reconcileSource(ctx context.Context, pr *por
 	log := log.FromContext(ctx)
 	log.Info("applying source", "type", creationSource, "name", pr.Name)
 
+	start := time.Now()
+
 	// TODO: CreateNewDraft always receives lifecycle=Draft — consider removing the lifecycle parameter from the interface.
 	draft, err := r.ContentCache.CreateNewDraft(ctx, repoKey, pr.Spec.PackageName, pr.Spec.WorkspaceName, string(porchv1alpha2.PackageRevisionLifecycleDraft))
 	if err != nil {
@@ -230,6 +235,8 @@ func (r *PackageRevisionReconciler) reconcileSource(ctx context.Context, pr *por
 	)
 	r.ensureLatestRevisionLabel(ctx, pr)
 
+	telemetry.RecordControllerOperation(telemetry.ResourcePackageRevision, "CREATE", start)
+
 	result := ctrl.Result{Requeue: true}
 	return &result, nil
 }

controllers/packagerevisions/pkg/controllers/packagerevision/render.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	iofs "io/fs"
 	"strings"
+	"time"
 
 	fnresult "github.com/kptdev/kpt/api/fnresult/v1"
 	kptfilev1 "github.com/kptdev/kpt/api/kptfile/v1"
@@ -27,6 +28,7 @@ import (
 	"github.com/kptdev/kpt/pkg/lib/kptops"
 	"github.com/kptdev/kpt/pkg/lib/runneroptions"
 	porchv1alpha2 "github.com/kptdev/porch/api/porch/v1alpha2"
+	"github.com/kptdev/porch/internal/telemetry"
 	"github.com/kptdev/porch/pkg/repository"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -172,6 +174,8 @@ func isPushOnRenderFailure(pr *porchv1alpha2.PackageRevision) bool {
 // executeRender performs the render, handles failure/stale, and writes results.
 func (r *PackageRevisionReconciler) executeRender(ctx context.Context, pr *porchv1alpha2.PackageRevision, repoKey repository.RepositoryKey) (*ctrl.Result, error) {
 	log := log.FromContext(ctx)
+	start := time.Now()
+	defer telemetry.RecordControllerOperation(telemetry.ResourcePackageRevisionResources, "UPDATE", start)
 
 	resources, err := r.readPackageResources(ctx, repoKey, pr.Spec.PackageName, pr.Spec.WorkspaceName)
 	if err != nil {