From 9a7f0a59107ae5fdcb835e1793c52eae52d43bc4 Mon Sep 17 00:00:00 2001 From: Alex Vulaj Date: Thu, 19 Nov 2026 14:48:40 -0500 Subject: [PATCH 1/3] dump logs for restarted pods --- tests/e2e/helpers.go | 80 +++++++++++++++++++++++++++++++-------- tests/e2e/helpers_test.go | 61 +++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 15 deletions(-) create mode 100644 tests/e2e/helpers_test.go diff --git a/tests/e2e/helpers.go b/tests/e2e/helpers.go index 193943ef..eb46fdf7 100644 --- a/tests/e2e/helpers.go +++ b/tests/e2e/helpers.go @@ -351,30 +351,80 @@ func runKubectlDump(args ...string) { fmt.Fprintln(os.Stderr) } +type containerRestartStatus struct { + RestartCount int32 `json:"restartCount"` +} + +type podLogDumpPod struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Status struct { + Phase string `json:"phase"` + ContainerStatuses []containerRestartStatus `json:"containerStatuses"` + InitContainerStatuses []containerRestartStatus `json:"initContainerStatuses"` + EphemeralContainerStatuses []containerRestartStatus `json:"ephemeralContainerStatuses"` + } `json:"status"` +} + +type podLogDumpList struct { + Items []podLogDumpPod `json:"items"` +} + +func (pod podLogDumpPod) restartCount() int32 { + var total int32 + for _, status := range pod.Status.ContainerStatuses { + total += status.RestartCount + } + for _, status := range pod.Status.InitContainerStatuses { + total += status.RestartCount + } + for _, status := range pod.Status.EphemeralContainerStatuses { + total += status.RestartCount + } + return total +} + +func shouldDumpPodLogs(phase string, restartCount int32) bool { + return (phase != "Running" && phase != "Succeeded") || restartCount > 0 +} + func dumpLogsForFailingPods(namespace string) { - cmd := exec.Command("kubectl", "get", "pods", "-n", namespace, - "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\t\"}{.status.phase}{\"\\n\"}{end}") + cmd := exec.Command("kubectl", "get", "pods", "-n", namespace, "-o", "json") output, err := cmd.Output() if err != nil { fmt.Fprintf(os.Stderr, "[dump] failed to list pods in %s: %v\n", namespace, err) return } - for line := range strings.SplitSeq(strings.TrimSpace(string(output)), "\n") { - if line == "" { - continue - } - parts := strings.SplitN(line, "\t", 2) - if len(parts) != 2 { - continue - } - podName, phase := parts[0], parts[1] - if phase == "Running" || phase == "Succeeded" { + var pods podLogDumpList + if err := json.Unmarshal(output, &pods); err != nil { + fmt.Fprintf(os.Stderr, "[dump] failed to parse pods in %s: %v\n", namespace, err) + return + } + + for _, pod := range pods.Items { + restartCount := pod.restartCount() + fmt.Fprintf( + os.Stderr, + "[dump] pod %s/%s (phase=%s, restarts=%d)\n", + namespace, + pod.Metadata.Name, + pod.Status.Phase, + restartCount, + ) + if !shouldDumpPodLogs(pod.Status.Phase, restartCount) { continue } - fmt.Fprintf(os.Stderr, "[dump] logs for pod %s/%s (phase=%s):\n", namespace, podName, phase) - runKubectlDump("logs", "-n", namespace, podName, "--all-containers", "--tail=100") - runKubectlDump("logs", "-n", namespace, podName, "--all-containers", "--previous", "--tail=50") + fmt.Fprintf( + os.Stderr, + "[dump] logs for pod %s/%s (phase=%s):\n", + namespace, + pod.Metadata.Name, + pod.Status.Phase, + ) + runKubectlDump("logs", "-n", namespace, pod.Metadata.Name, "--all-containers", "--tail=100") + runKubectlDump("logs", "-n", namespace, pod.Metadata.Name, "--all-containers", "--previous", "--tail=50") } } diff --git a/tests/e2e/helpers_test.go b/tests/e2e/helpers_test.go new file mode 100644 index 00000000..57ef664e --- /dev/null +++ b/tests/e2e/helpers_test.go @@ -0,0 +1,61 @@ +//go:build e2e + +package e2e + +import ( + "encoding/json" + "testing" +) + +func TestPodLogDumpSelection(t *testing.T) { + var pods podLogDumpList + err := json.Unmarshal([]byte(`{ + "items": [ + { + "metadata": {"name": "running-stable"}, + "status": {"phase": "Running", "containerStatuses": [{"restartCount": 0}]} + }, + { + "metadata": {"name": "running-restarted"}, + "status": {"phase": "Running", "containerStatuses": [{"restartCount": 1}, {"restartCount": 2}]} + }, + { + "metadata": {"name": "succeeded-init-restarted"}, + "status": {"phase": "Succeeded", "initContainerStatuses": [{"restartCount": 1}]} + }, + { + "metadata": {"name": "failed"}, + "status": {"phase": "Failed", "containerStatuses": [{"restartCount": 2}]} + } + ] + }`), &pods) + if err != nil { + t.Fatalf("unmarshal pod list: %v", err) + } + if len(pods.Items) != 4 { + t.Fatalf("unmarshaled %d pods, want 4", len(pods.Items)) + } + + tests := []struct { + name string + pod podLogDumpPod + wantRestartCount int32 + wantDumpLogs bool + }{ + {name: "running pod without restarts", pod: pods.Items[0], wantRestartCount: 0, wantDumpLogs: false}, + {name: "running pod with restarts", pod: pods.Items[1], wantRestartCount: 3, wantDumpLogs: true}, + {name: "succeeded pod with init container restarts", pod: pods.Items[2], wantRestartCount: 1, wantDumpLogs: true}, + {name: "failed pod with restarts", pod: pods.Items[3], wantRestartCount: 2, wantDumpLogs: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.pod.restartCount(); got != tt.wantRestartCount { + t.Errorf("restartCount() = %d, want %d", got, tt.wantRestartCount) + } + if got := shouldDumpPodLogs(tt.pod.Status.Phase, tt.pod.restartCount()); got != tt.wantDumpLogs { + t.Errorf("shouldDumpPodLogs() = %t, want %t", got, tt.wantDumpLogs) + } + }) + } +} From d0c3ac8621f694c09365b5ac17a48a281c9f2a40 Mon Sep 17 00:00:00 2001 From: Alex Vulaj Date: Fri, 20 Nov 2026 09:19:15 -0500 Subject: [PATCH 2/3] rename new structs, remove tests of our tests --- tests/e2e/helpers.go | 10 +++---- tests/e2e/helpers_test.go | 61 --------------------------------------- 2 files changed, 5 insertions(+), 66 deletions(-) delete mode 100644 tests/e2e/helpers_test.go diff --git a/tests/e2e/helpers.go b/tests/e2e/helpers.go index eb46fdf7..c47eb513 100644 --- a/tests/e2e/helpers.go +++ b/tests/e2e/helpers.go @@ -355,7 +355,7 @@ type containerRestartStatus struct { RestartCount int32 `json:"restartCount"` } -type podLogDumpPod struct { +type podSummary struct { Metadata struct { Name string `json:"name"` } `json:"metadata"` @@ -367,11 +367,11 @@ type podLogDumpPod struct { } `json:"status"` } -type podLogDumpList struct { - Items []podLogDumpPod `json:"items"` +type podSummaryList struct { + Items []podSummary `json:"items"` } -func (pod podLogDumpPod) restartCount() int32 { +func (pod podSummary) restartCount() int32 { var total int32 for _, status := range pod.Status.ContainerStatuses { total += status.RestartCount @@ -397,7 +397,7 @@ func dumpLogsForFailingPods(namespace string) { return } - var pods podLogDumpList + var pods podSummaryList if err := json.Unmarshal(output, &pods); err != nil { fmt.Fprintf(os.Stderr, "[dump] failed to parse pods in %s: %v\n", namespace, err) return diff --git a/tests/e2e/helpers_test.go b/tests/e2e/helpers_test.go deleted file mode 100644 index 57ef664e..00000000 --- a/tests/e2e/helpers_test.go +++ /dev/null @@ -1,61 +0,0 @@ -//go:build e2e - -package e2e - -import ( - "encoding/json" - "testing" -) - -func TestPodLogDumpSelection(t *testing.T) { - var pods podLogDumpList - err := json.Unmarshal([]byte(`{ - "items": [ - { - "metadata": {"name": "running-stable"}, - "status": {"phase": "Running", "containerStatuses": [{"restartCount": 0}]} - }, - { - "metadata": {"name": "running-restarted"}, - "status": {"phase": "Running", "containerStatuses": [{"restartCount": 1}, {"restartCount": 2}]} - }, - { - "metadata": {"name": "succeeded-init-restarted"}, - "status": {"phase": "Succeeded", "initContainerStatuses": [{"restartCount": 1}]} - }, - { - "metadata": {"name": "failed"}, - "status": {"phase": "Failed", "containerStatuses": [{"restartCount": 2}]} - } - ] - }`), &pods) - if err != nil { - t.Fatalf("unmarshal pod list: %v", err) - } - if len(pods.Items) != 4 { - t.Fatalf("unmarshaled %d pods, want 4", len(pods.Items)) - } - - tests := []struct { - name string - pod podLogDumpPod - wantRestartCount int32 - wantDumpLogs bool - }{ - {name: "running pod without restarts", pod: pods.Items[0], wantRestartCount: 0, wantDumpLogs: false}, - {name: "running pod with restarts", pod: pods.Items[1], wantRestartCount: 3, wantDumpLogs: true}, - {name: "succeeded pod with init container restarts", pod: pods.Items[2], wantRestartCount: 1, wantDumpLogs: true}, - {name: "failed pod with restarts", pod: pods.Items[3], wantRestartCount: 2, wantDumpLogs: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.pod.restartCount(); got != tt.wantRestartCount { - t.Errorf("restartCount() = %d, want %d", got, tt.wantRestartCount) - } - if got := shouldDumpPodLogs(tt.pod.Status.Phase, tt.pod.restartCount()); got != tt.wantDumpLogs { - t.Errorf("shouldDumpPodLogs() = %t, want %t", got, tt.wantDumpLogs) - } - }) - } -} From 4824148114631e641e64b41164956ba7cb5961d4 Mon Sep 17 00:00:00 2001 From: Alex Vulaj Date: Fri, 20 Nov 2026 09:30:17 -0500 Subject: [PATCH 3/3] add --ignore-errors=true to kubectl command --- tests/e2e/helpers.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/e2e/helpers.go b/tests/e2e/helpers.go index c47eb513..e03f52ee 100644 --- a/tests/e2e/helpers.go +++ b/tests/e2e/helpers.go @@ -424,7 +424,16 @@ func dumpLogsForFailingPods(namespace string) { pod.Status.Phase, ) runKubectlDump("logs", "-n", namespace, pod.Metadata.Name, "--all-containers", "--tail=100") - runKubectlDump("logs", "-n", namespace, pod.Metadata.Name, "--all-containers", "--previous", "--tail=50") + runKubectlDump( + "logs", + "-n", + namespace, + pod.Metadata.Name, + "--all-containers", + "--previous", + "--ignore-errors=true", + "--tail=50", + ) } }