benchmarks: avoid unreliable kubectl attach when fetching metrics

benchmarks/record/record.go

@@ -234,14 +234,41 @@ func Metrics(experiment *gm.Experiment, suffix string) {
 }
 
 func getMetrics(res map[string]float64, url string, controllers ...string) {
-	pod := addRandomSuffix("curl")
 	var (
 		mfs    map[string]*dto.MetricFamily
 		parser = expfmt.NewTextParser(model.LegacyValidation)
 	)
 	Eventually(func() error {
+		pod := addRandomSuffix("curl")
 		GinkgoWriter.Print("Fetching metrics from " + url + "\n")
-		out, err := k.Run("run", "--rm", "--attach", "--quiet", "--restart=Never", pod, "--image=curlimages/curl", "--namespace", "cattle-fleet-system", "--command", "--", "curl", "-s", url)
+
+		// Create the pod without --attach to avoid kubectl's unreliable
+		// attach mechanism, which can duplicate output on fallback to logs.
+		_, _, err := k.RunStdout("run", "--restart=Never", pod, "--image=curlimages/curl", "--namespace", "cattle-fleet-system", "--command", "--", "curl", "-sf", url)
+		if err != nil {
+			// Pod may have been created despite the error; clean up.
+			_, _ = k.Run("delete", "pod", "--namespace", "cattle-fleet-system", pod, "--ignore-not-found")
+			return fmt.Errorf("kubectl run: %w", err)
+		}
+
+		// Wait for the pod to terminate. Use condition=Ready=false to
+		// detect both Succeeded and Failed without blocking for the full
+		// timeout on a failed curl.
+		_, _, err = k.RunStdout("wait", "--for=condition=Ready=false", "--namespace", "cattle-fleet-system", "pod/"+pod, "--timeout=30s")
+		if err != nil {
+			_, _ = k.Run("delete", "pod", "--namespace", "cattle-fleet-system", pod, "--ignore-not-found")
+			return fmt.Errorf("waiting for pod: %w", err)
+		}
+
+		phase, _, _ := k.RunStdout("get", "pod", pod, "--namespace", "cattle-fleet-system", "-o", "jsonpath={.status.phase}")
+		if strings.TrimSpace(phase) != "Succeeded" {
+			_, _ = k.Run("delete", "pod", "--namespace", "cattle-fleet-system", pod, "--ignore-not-found")
+			return fmt.Errorf("curl pod %s finished with phase %s", pod, phase)
+		}
+
+		out, _, err := k.RunStdout("logs", "--namespace", "cattle-fleet-system", pod)
+		// Always clean up the pod.
+		_, _ = k.Run("delete", "pod", "--namespace", "cattle-fleet-system", pod, "--ignore-not-found")
 		if err != nil {
 			return err
 		}

e2e/testenv/kubectl/kubectl.go

@@ -85,30 +85,50 @@ func (c Command) Run(args ...string) (string, error) {
 	}
 
 	GinkgoWriter.Printf("kubectl %s\n", strings.Join(args, " "))
-	result, err := c.exec("kubectl", args...)
+	stdout, stderr, err := c.exec("kubectl", args...)
+	result := stdout + stderr
 	if err != nil {
 		GinkgoWriter.Printf("result:%s err:%s\n", result, err)
 	}
 
 	return result, err
 }
 
-func (c Command) exec(command string, args ...string) (string, error) {
+// RunStdout behaves like Run but returns stdout and stderr separately.
+func (c Command) RunStdout(args ...string) (stdout, stderr string, err error) {
+	if c.cnt != "" {
+		args = append([]string{"--context", c.cnt}, args...)
+	}
+
+	if c.ns != "" {
+		args = append([]string{"-n", c.ns}, args...)
+	}
+
+	GinkgoWriter.Printf("kubectl %s\n", strings.Join(args, " "))
+	stdout, stderr, err = c.exec("kubectl", args...)
+	if err != nil {
+		GinkgoWriter.Printf("stdout:%s stderr:%s err:%s\n", stdout, stderr, err)
+	}
+
+	return stdout, stderr, err
+}
+
+func (c Command) exec(command string, args ...string) (string, string, error) {
 	cmd := exec.CommandContext(context.Background(), command, args...)
 
-	var b bytes.Buffer
+	var outBuf, errBuf bytes.Buffer
 	if c.stdout {
-		cmd.Stdout = io.MultiWriter(os.Stdout, &b)
-		cmd.Stderr = io.MultiWriter(os.Stderr, &b)
+		cmd.Stdout = io.MultiWriter(os.Stdout, &outBuf)
+		cmd.Stderr = io.MultiWriter(os.Stderr, &errBuf)
 	} else {
-		cmd.Stdout = &b
-		cmd.Stderr = &b
+		cmd.Stdout = &outBuf
+		cmd.Stderr = &errBuf
 	}
 
 	if c.dir != "" {
 		cmd.Dir = c.dir
 	}
 
 	err := cmd.Run()
-	return b.String(), err
+	return outBuf.String(), errBuf.String(), err
 }