fix(k8s): surface container status in PodNotAvailable error

When WaitUntilPodAvailable times out, the error only reported the pod-level
reason and message, which are usually empty when a container fails to start.
The actionable detail (CrashLoopBackOff, ImagePullBackOff, etc.) lives in the
container statuses, so include the waiting/terminated state of each unready
container, covering init and regular containers.

Closes #1251

Author: james00012

modules/k8s/errors.go

@@ -3,6 +3,7 @@ package k8s
 import (
 	"errors"
 	"fmt"
+	"strings"
 
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
@@ -145,7 +146,43 @@ type PodNotAvailable struct {
 
 // Error is a simple function to return a formatted error message as a string
 func (err PodNotAvailable) Error() string {
-	return fmt.Sprintf("Pod %s is not available, reason: %s, message: %s", err.pod.Name, err.pod.Status.Reason, err.pod.Status.Message)
+	msg := fmt.Sprintf("Pod %s is not available, reason: %s, message: %s", err.pod.Name, err.pod.Status.Reason, err.pod.Status.Message)
+
+	// The pod-level reason and message above are usually empty when a container fails to start. The actionable detail
+	// (for example CrashLoopBackOff or ImagePullBackOff) lives in the container statuses, so surface it here.
+	details := append(
+		unreadyContainerDetails("init container", err.pod.Status.InitContainerStatuses),
+		unreadyContainerDetails("container", err.pod.Status.ContainerStatuses)...,
+	)
+	if len(details) > 0 {
+		msg += ". " + strings.Join(details, "; ")
+	}
+
+	return msg
+}
+
+// unreadyContainerDetails returns a short description of the waiting or terminated state of each container that is not
+// ready, prefixed with the given kind (for example "container" or "init container").
+func unreadyContainerDetails(kind string, statuses []corev1.ContainerStatus) []string {
+	var details []string
+
+	for i := range statuses {
+		status := statuses[i]
+		if status.Ready {
+			continue
+		}
+
+		switch {
+		case status.State.Waiting != nil:
+			detail := fmt.Sprintf("%s %s waiting: %s", kind, status.Name, status.State.Waiting.Reason)
+			details = append(details, strings.TrimSpace(detail+" "+status.State.Waiting.Message))
+		case status.State.Terminated != nil:
+			detail := fmt.Sprintf("%s %s terminated: %s", kind, status.Name, status.State.Terminated.Reason)
+			details = append(details, strings.TrimSpace(detail+" "+status.State.Terminated.Message))
+		}
+	}
+
+	return details
 }
 
 // NewPodNotAvailableError returns a PodNotAvailable struct when Kubernetes deems a pod is not available

modules/k8s/errors_test.go

@@ -62,3 +62,65 @@ func TestErrorDeploymentNotAvailable(t *testing.T) {
 		})
 	}
 }
+
+func TestErrorPodNotAvailable(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		pod         *v1.Pod
+		title       string
+		expectedErr string
+	}{
+		{
+			title: "PodLevelReasonOnly",
+			pod: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+				Status:     v1.PodStatus{Reason: "Evicted", Message: "low memory"},
+			},
+			expectedErr: "Pod foo is not available, reason: Evicted, message: low memory",
+		},
+		{
+			title: "ContainerWaiting",
+			pod: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+				Status: v1.PodStatus{
+					ContainerStatuses: []v1.ContainerStatus{
+						{
+							Name:  "web",
+							Ready: false,
+							State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{
+								Reason:  "CrashLoopBackOff",
+								Message: "back-off 5m restarting failed container",
+							}},
+						},
+					},
+				},
+			},
+			expectedErr: "Pod foo is not available, reason: , message: . container web waiting: CrashLoopBackOff back-off 5m restarting failed container",
+		},
+		{
+			title: "InitContainerWaitingAndReadyContainerIgnored",
+			pod: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{Name: "foo"},
+				Status: v1.PodStatus{
+					InitContainerStatuses: []v1.ContainerStatus{
+						{Name: "setup", Ready: false, State: v1.ContainerState{Waiting: &v1.ContainerStateWaiting{Reason: "ImagePullBackOff"}}},
+					},
+					ContainerStatuses: []v1.ContainerStatus{
+						{Name: "web", Ready: true},
+					},
+				},
+			},
+			expectedErr: "Pod foo is not available, reason: , message: . init container setup waiting: ImagePullBackOff",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.title, func(t *testing.T) {
+			t.Parallel()
+
+			err := k8s.NewPodNotAvailableError(tc.pod)
+			assert.EqualError(t, err, tc.expectedErr)
+		})
+	}
+}