Fix aggregated API server readiness race causing sporadic 503s in envtest suites (#1505)

* Fix aggregated API server readiness race causing sporadic 503s in envtest suites

Signed-off-by: Andreas Fritzler <andreas.fritzler@sap.com>

* Avoid reusing list objects across serve-readiness probes

Signed-off-by: Andreas Fritzler <andreas.fritzler@sap.com>

---------

Signed-off-by: Andreas Fritzler <andreas.fritzler@sap.com>

Author: afritzler

utils/envtest/envtest.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -18,6 +19,7 @@ import (
 	"github.com/ironcore-dev/ironcore/utils/envtest/internal/testing/addr"
 	"github.com/ironcore-dev/ironcore/utils/envtest/internal/testing/certs"
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -591,6 +593,10 @@ func WaitUntilGroupVersionsDiscoverable(ctx context.Context, cfg *rest.Config, c
 		return fmt.Errorf("error waiting for GVKs to be discoverable: %w", err)
 	}
 
+	if err := waitUntilGroupVersionsServe(ctx, c, scheme, gvs...); err != nil {
+		return fmt.Errorf("error waiting for aggregated api server to serve requests: %w", err)
+	}
+
 	log.Info("All group versions and REST mappings are discoverable", "gvs", gvs)
 	return nil
 }
@@ -661,6 +667,91 @@ func WaitUntilAPIServicesReady(ctx context.Context, ext *EnvironmentExtensions,
 	return nil
 }
 
+// poll interval for the aggregated API server serve probe.
+const serveReadinessPollInterval = 100 * time.Millisecond
+
+// sustained success window to absorb the 503 startup race after APIService goes Available.
+const serveReadinessStableDuration = 500 * time.Millisecond
+
+// waitUntilGroupVersionsServe polls a real List per GV until it succeeds for
+// serveReadinessStableDuration, absorbing the aggregated server's 503 window
+// after APIService reports Available.
+func waitUntilGroupVersionsServe(ctx context.Context, c client.Client, scheme *runtime.Scheme, gvs ...schema.GroupVersion) error {
+	var listGVKs []schema.GroupVersionKind
+	for _, gv := range gvs {
+		if listGVK, ok := firstListGVKForGroupVersion(scheme, gv); ok {
+			listGVKs = append(listGVKs, listGVK)
+		}
+	}
+	if len(listGVKs) == 0 {
+		return nil
+	}
+
+	requiredSuccesses := int(serveReadinessStableDuration / serveReadinessPollInterval)
+	successes := 0
+	var lastErr error
+	if err := wait.PollUntilContextCancel(ctx, serveReadinessPollInterval, true, func(ctx context.Context) (bool, error) {
+		for _, listGVK := range listGVKs {
+			list, err := scheme.New(listGVK)
+			if err != nil {
+				return false, fmt.Errorf("unexpected error constructing list for %s: %w", listGVK, err)
+			}
+			if err := c.List(ctx, list.(client.ObjectList)); err != nil {
+				if !isTransientServeError(err) {
+					return false, fmt.Errorf("unexpected error probing aggregated api server: %w", err)
+				}
+				lastErr = err
+				successes = 0
+				return false, nil
+			}
+		}
+		successes++
+		return successes >= requiredSuccesses, nil
+	}); err != nil {
+		if lastErr != nil {
+			return fmt.Errorf("%w (last error: %v)", err, lastErr)
+		}
+		return err
+	}
+	return nil
+}
+
+// firstListGVKForGroupVersion returns the first list-kind GVK registered for gv, sorted for determinism.
+func firstListGVKForGroupVersion(scheme *runtime.Scheme, gv schema.GroupVersion) (schema.GroupVersionKind, bool) {
+	kinds := make([]string, 0, len(scheme.KnownTypes(gv)))
+	for kind := range scheme.KnownTypes(gv) {
+		kinds = append(kinds, kind)
+	}
+	sort.Strings(kinds)
+	for _, kind := range kinds {
+		obj, err := scheme.New(gv.WithKind(kind))
+		if err != nil {
+			continue
+		}
+		if _, ok := obj.(client.ObjectList); ok {
+			return gv.WithKind(kind), true
+		}
+	}
+	return schema.GroupVersionKind{}, false
+}
+
+// isTransientServeError reports whether err is a transient startup condition worth retrying.
+func isTransientServeError(err error) bool {
+	if err == nil {
+		return false
+	}
+	switch {
+	case apierrors.IsServiceUnavailable(err),
+		apierrors.IsServerTimeout(err),
+		apierrors.IsTimeout(err),
+		apierrors.IsTooManyRequests(err),
+		apierrors.IsInternalError(err):
+		return true
+	}
+	var urlErr *url.Error
+	return errors.As(err, &urlErr)
+}
+
 func WaitUntilAPIServicesReadyWithTimeout(timeout time.Duration, ext *EnvironmentExtensions, cfg *rest.Config, c client.Client, scheme *runtime.Scheme) error {
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()