From fa412f42168d41f4127c03bcd1c14833744a634b Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Thu, 19 Nov 2026 14:32:04 +0100 Subject: [PATCH 1/3] fix(workflows): cap create and destroy steps at 2h across all workflows Set a uniform activeDeadlineSeconds of 7200 on the create and destroy steps of every workflow template and all janitor workflows. The openshift-4, openshift-4-demo, and aks templates had no create deadline at all: a hung installer pod ran indefinitely, the workflow never reached the suspend node, and since it stayed in CREATING status the infra server's cleanupExpiredClusters loop (which only resumes READY/suspended workflows) skipped it forever. This wedged ~80 ga-ocp-cron workflows in production with their PVCs attached for days. Create steps previously capped at 1h (demo, qa-demo, eks, gke-default) are raised to 2h. Destroy steps that had a 1h deadline are raised to 2h; those without any deadline (including the janitors) gain one, so a hung cleanup step cannot pin a workflow and its PVC forever either. Co-Authored-By: Claude Code --- chart/infra-server/static/janitor-delete-gke-default.yaml | 2 +- chart/infra-server/static/janitor-delete-openshift-4.yaml | 1 + chart/infra-server/static/janitor-delete-openshift-rosa.yaml | 2 +- chart/infra-server/static/workflow-aks.yaml | 2 ++ chart/infra-server/static/workflow-demo.yaml | 4 ++-- chart/infra-server/static/workflow-eks.yaml | 4 ++-- chart/infra-server/static/workflow-gke-default.yaml | 4 ++-- chart/infra-server/static/workflow-openshift-4-demo.yaml | 2 ++ chart/infra-server/static/workflow-openshift-4.yaml | 2 ++ chart/infra-server/static/workflow-openshift-aro.yaml | 2 +- chart/infra-server/static/workflow-openshift-ibmroks.yaml | 2 +- chart/infra-server/static/workflow-openshift-rosa-hcp.yaml | 2 +- chart/infra-server/static/workflow-openshift-rosa.yaml | 2 +- chart/infra-server/static/workflow-osd-aws.yaml | 2 +- chart/infra-server/static/workflow-osd-gcp.yaml | 2 +- chart/infra-server/static/workflow-qa-demo.yaml | 4 ++-- 16 files changed, 23 insertions(+), 16 deletions(-) diff --git a/chart/infra-server/static/janitor-delete-gke-default.yaml b/chart/infra-server/static/janitor-delete-gke-default.yaml index 394a60d3a..79cde2e6e 100644 --- a/chart/infra-server/static/janitor-delete-gke-default.yaml +++ b/chart/infra-server/static/janitor-delete-gke-default.yaml @@ -22,7 +22,7 @@ spec: template: destroy - name: destroy - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 container: image: quay.io/stackrox-io/ci:automation-flavors-gke-default-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/janitor-delete-openshift-4.yaml b/chart/infra-server/static/janitor-delete-openshift-4.yaml index c0cc3370d..8433bf23e 100644 --- a/chart/infra-server/static/janitor-delete-openshift-4.yaml +++ b/chart/infra-server/static/janitor-delete-openshift-4.yaml @@ -28,6 +28,7 @@ spec: - name: destroy + activeDeadlineSeconds: 7200 container: image: quay.io/stackrox-io/ci:automation-flavors-openshift-4-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/janitor-delete-openshift-rosa.yaml b/chart/infra-server/static/janitor-delete-openshift-rosa.yaml index 84de4c6d2..5f081146e 100644 --- a/chart/infra-server/static/janitor-delete-openshift-rosa.yaml +++ b/chart/infra-server/static/janitor-delete-openshift-rosa.yaml @@ -25,7 +25,7 @@ spec: template: destroy - name: destroy - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 container: image: quay.io/stackrox-io/ci:automation-flavors-rosa-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-aks.yaml b/chart/infra-server/static/workflow-aks.yaml index e84315977..a1be4143e 100644 --- a/chart/infra-server/static/workflow-aks.yaml +++ b/chart/infra-server/static/workflow-aks.yaml @@ -34,6 +34,7 @@ spec: template: destroy - name: create + activeDeadlineSeconds: 7200 outputs: artifacts: - name: kubeconfig @@ -103,6 +104,7 @@ spec: suspend: {} - name: destroy + activeDeadlineSeconds: 7200 container: image: quay.io/stackrox-io/ci:automation-flavors-aks-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-demo.yaml b/chart/infra-server/static/workflow-demo.yaml index 9b098e5e3..4d2654248 100644 --- a/chart/infra-server/static/workflow-demo.yaml +++ b/chart/infra-server/static/workflow-demo.yaml @@ -70,7 +70,7 @@ spec: - /tmp/roxctl - name: create - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 inputs: artifacts: - name: roxctl @@ -176,7 +176,7 @@ spec: suspend: {} - name: destroy - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 inputs: artifacts: - name: tfstate diff --git a/chart/infra-server/static/workflow-eks.yaml b/chart/infra-server/static/workflow-eks.yaml index 805b00d39..727684c42 100644 --- a/chart/infra-server/static/workflow-eks.yaml +++ b/chart/infra-server/static/workflow-eks.yaml @@ -34,7 +34,7 @@ spec: template: destroy - name: create - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 outputs: artifacts: - name: kubeconfig @@ -88,7 +88,7 @@ spec: suspend: {} - name: destroy - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 container: image: quay.io/stackrox-io/ci:automation-flavors-eks-{{ .Chart.Annotations.automationFlavorsVersion }} command: diff --git a/chart/infra-server/static/workflow-gke-default.yaml b/chart/infra-server/static/workflow-gke-default.yaml index 4be3a1203..4ac197a5a 100644 --- a/chart/infra-server/static/workflow-gke-default.yaml +++ b/chart/infra-server/static/workflow-gke-default.yaml @@ -44,7 +44,7 @@ spec: value: '{{ "{{" }}workflow.parameters.name{{ "}}" }}' - name: create - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 outputs: artifacts: - name: kubeconfig @@ -90,7 +90,7 @@ spec: suspend: {} - name: destroy - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 container: image: quay.io/stackrox-io/ci:automation-flavors-gke-default-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-openshift-4-demo.yaml b/chart/infra-server/static/workflow-openshift-4-demo.yaml index 1a658809a..0f7a905d5 100644 --- a/chart/infra-server/static/workflow-openshift-4-demo.yaml +++ b/chart/infra-server/static/workflow-openshift-4-demo.yaml @@ -79,6 +79,7 @@ spec: template: destroy - name: create + activeDeadlineSeconds: 7200 container: image: quay.io/stackrox-io/ci:automation-flavors-openshift-4-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always @@ -272,6 +273,7 @@ spec: suspend: {} - name: destroy + activeDeadlineSeconds: 7200 container: image: quay.io/stackrox-io/ci:automation-flavors-openshift-4-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-openshift-4.yaml b/chart/infra-server/static/workflow-openshift-4.yaml index 74866f881..f6b7f836a 100644 --- a/chart/infra-server/static/workflow-openshift-4.yaml +++ b/chart/infra-server/static/workflow-openshift-4.yaml @@ -70,6 +70,7 @@ spec: template: destroy - name: create + activeDeadlineSeconds: 7200 outputs: artifacts: - name: kubeconfig @@ -208,6 +209,7 @@ spec: suspend: {} - name: destroy + activeDeadlineSeconds: 7200 container: image: quay.io/stackrox-io/ci:automation-flavors-openshift-4-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-openshift-aro.yaml b/chart/infra-server/static/workflow-openshift-aro.yaml index cab8217f9..e25509d0b 100644 --- a/chart/infra-server/static/workflow-openshift-aro.yaml +++ b/chart/infra-server/static/workflow-openshift-aro.yaml @@ -130,7 +130,7 @@ spec: suspend: {} - name: destroy - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 container: image: quay.io/stackrox-io/ci:automation-flavors-aro-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-openshift-ibmroks.yaml b/chart/infra-server/static/workflow-openshift-ibmroks.yaml index 4b62b1a51..450a31a50 100644 --- a/chart/infra-server/static/workflow-openshift-ibmroks.yaml +++ b/chart/infra-server/static/workflow-openshift-ibmroks.yaml @@ -113,7 +113,7 @@ spec: suspend: {} - name: destroy - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 container: image: quay.io/stackrox-io/ci:automation-flavors-ibmroks-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-openshift-rosa-hcp.yaml b/chart/infra-server/static/workflow-openshift-rosa-hcp.yaml index fdfbfde0b..3bb8c679c 100644 --- a/chart/infra-server/static/workflow-openshift-rosa-hcp.yaml +++ b/chart/infra-server/static/workflow-openshift-rosa-hcp.yaml @@ -145,7 +145,7 @@ spec: suspend: {} - name: destroy - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 container: image: quay.io/stackrox-io/ci:automation-flavors-rosa-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-openshift-rosa.yaml b/chart/infra-server/static/workflow-openshift-rosa.yaml index 8ca639b4c..962001c54 100644 --- a/chart/infra-server/static/workflow-openshift-rosa.yaml +++ b/chart/infra-server/static/workflow-openshift-rosa.yaml @@ -127,7 +127,7 @@ spec: suspend: {} - name: destroy - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 container: image: quay.io/stackrox-io/ci:automation-flavors-rosa-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-osd-aws.yaml b/chart/infra-server/static/workflow-osd-aws.yaml index b12bd94ff..ff4bb4f3d 100644 --- a/chart/infra-server/static/workflow-osd-aws.yaml +++ b/chart/infra-server/static/workflow-osd-aws.yaml @@ -128,7 +128,7 @@ spec: suspend: {} - name: destroy - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 container: image: quay.io/stackrox-io/ci:automation-flavors-osd-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-osd-gcp.yaml b/chart/infra-server/static/workflow-osd-gcp.yaml index 66ffa951c..19d1dc71d 100644 --- a/chart/infra-server/static/workflow-osd-gcp.yaml +++ b/chart/infra-server/static/workflow-osd-gcp.yaml @@ -128,7 +128,7 @@ spec: suspend: {} - name: destroy - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 container: image: quay.io/stackrox-io/ci:automation-flavors-osd-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-qa-demo.yaml b/chart/infra-server/static/workflow-qa-demo.yaml index 4a5681bb8..e0b857373 100644 --- a/chart/infra-server/static/workflow-qa-demo.yaml +++ b/chart/infra-server/static/workflow-qa-demo.yaml @@ -78,7 +78,7 @@ spec: - /tmp/roxctl - name: create - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 inputs: artifacts: - name: roxctl @@ -183,7 +183,7 @@ spec: suspend: {} - name: destroy - activeDeadlineSeconds: 3600 + activeDeadlineSeconds: 7200 inputs: artifacts: - name: tfstate From d2a37cb838998c1cc5f2f05ceeae03575917bd30 Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Thu, 19 Nov 2026 14:39:50 +0100 Subject: [PATCH 2/3] fix(workflows): bound pending time on create steps with template timeout activeDeadlineSeconds is only enforced by the kubelet once a pod runs on a node, so a create pod stuck Pending (e.g. unschedulable due to PVC attach limits) is never deadline-killed and holds the workflow indefinitely. Argo's template-level timeout field closes that gap: the controller fails nodes still Pending past the deadline and lowers the pod's activeDeadlineSeconds to the remaining time at creation (deployed workflow-controller v4.0.2, operator.go checkTemplateTimeout and workflowpod.go). Applied to create steps only. Destroy steps run as the exit handler, which the controller deliberately exempts from deadline enforcement; killing a pending destroy would fail cleanup and leak the cluster with no retry, whereas a failed pending create cascades into the onExit destroy. Co-Authored-By: Claude Code --- chart/infra-server/static/workflow-aks.yaml | 1 + chart/infra-server/static/workflow-demo.yaml | 1 + chart/infra-server/static/workflow-eks.yaml | 1 + chart/infra-server/static/workflow-gke-default.yaml | 1 + chart/infra-server/static/workflow-openshift-4-demo.yaml | 1 + chart/infra-server/static/workflow-openshift-4.yaml | 1 + chart/infra-server/static/workflow-openshift-aro.yaml | 1 + chart/infra-server/static/workflow-openshift-ibmroks.yaml | 1 + chart/infra-server/static/workflow-openshift-rosa-hcp.yaml | 1 + chart/infra-server/static/workflow-openshift-rosa.yaml | 1 + chart/infra-server/static/workflow-osd-aws.yaml | 1 + chart/infra-server/static/workflow-osd-gcp.yaml | 1 + chart/infra-server/static/workflow-qa-demo.yaml | 1 + 13 files changed, 13 insertions(+) diff --git a/chart/infra-server/static/workflow-aks.yaml b/chart/infra-server/static/workflow-aks.yaml index a1be4143e..0b2d58f3c 100644 --- a/chart/infra-server/static/workflow-aks.yaml +++ b/chart/infra-server/static/workflow-aks.yaml @@ -35,6 +35,7 @@ spec: - name: create activeDeadlineSeconds: 7200 + timeout: 2h outputs: artifacts: - name: kubeconfig diff --git a/chart/infra-server/static/workflow-demo.yaml b/chart/infra-server/static/workflow-demo.yaml index 4d2654248..933cfabc1 100644 --- a/chart/infra-server/static/workflow-demo.yaml +++ b/chart/infra-server/static/workflow-demo.yaml @@ -71,6 +71,7 @@ spec: - name: create activeDeadlineSeconds: 7200 + timeout: 2h inputs: artifacts: - name: roxctl diff --git a/chart/infra-server/static/workflow-eks.yaml b/chart/infra-server/static/workflow-eks.yaml index 727684c42..8a05de260 100644 --- a/chart/infra-server/static/workflow-eks.yaml +++ b/chart/infra-server/static/workflow-eks.yaml @@ -35,6 +35,7 @@ spec: - name: create activeDeadlineSeconds: 7200 + timeout: 2h outputs: artifacts: - name: kubeconfig diff --git a/chart/infra-server/static/workflow-gke-default.yaml b/chart/infra-server/static/workflow-gke-default.yaml index 4ac197a5a..51dbeb9a2 100644 --- a/chart/infra-server/static/workflow-gke-default.yaml +++ b/chart/infra-server/static/workflow-gke-default.yaml @@ -45,6 +45,7 @@ spec: - name: create activeDeadlineSeconds: 7200 + timeout: 2h outputs: artifacts: - name: kubeconfig diff --git a/chart/infra-server/static/workflow-openshift-4-demo.yaml b/chart/infra-server/static/workflow-openshift-4-demo.yaml index 0f7a905d5..893be2e31 100644 --- a/chart/infra-server/static/workflow-openshift-4-demo.yaml +++ b/chart/infra-server/static/workflow-openshift-4-demo.yaml @@ -80,6 +80,7 @@ spec: - name: create activeDeadlineSeconds: 7200 + timeout: 2h container: image: quay.io/stackrox-io/ci:automation-flavors-openshift-4-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-openshift-4.yaml b/chart/infra-server/static/workflow-openshift-4.yaml index f6b7f836a..eea8d2860 100644 --- a/chart/infra-server/static/workflow-openshift-4.yaml +++ b/chart/infra-server/static/workflow-openshift-4.yaml @@ -71,6 +71,7 @@ spec: - name: create activeDeadlineSeconds: 7200 + timeout: 2h outputs: artifacts: - name: kubeconfig diff --git a/chart/infra-server/static/workflow-openshift-aro.yaml b/chart/infra-server/static/workflow-openshift-aro.yaml index e25509d0b..fe661b14c 100644 --- a/chart/infra-server/static/workflow-openshift-aro.yaml +++ b/chart/infra-server/static/workflow-openshift-aro.yaml @@ -40,6 +40,7 @@ spec: - name: create activeDeadlineSeconds: 7200 + timeout: 2h container: image: quay.io/stackrox-io/ci:automation-flavors-aro-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-openshift-ibmroks.yaml b/chart/infra-server/static/workflow-openshift-ibmroks.yaml index 450a31a50..81ac5247a 100644 --- a/chart/infra-server/static/workflow-openshift-ibmroks.yaml +++ b/chart/infra-server/static/workflow-openshift-ibmroks.yaml @@ -47,6 +47,7 @@ spec: - name: create activeDeadlineSeconds: 7200 + timeout: 2h container: image: quay.io/stackrox-io/ci:automation-flavors-ibmroks-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-openshift-rosa-hcp.yaml b/chart/infra-server/static/workflow-openshift-rosa-hcp.yaml index 3bb8c679c..1c123854e 100644 --- a/chart/infra-server/static/workflow-openshift-rosa-hcp.yaml +++ b/chart/infra-server/static/workflow-openshift-rosa-hcp.yaml @@ -48,6 +48,7 @@ spec: - name: create activeDeadlineSeconds: 7200 + timeout: 2h container: image: quay.io/stackrox-io/ci:automation-flavors-rosa-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-openshift-rosa.yaml b/chart/infra-server/static/workflow-openshift-rosa.yaml index 962001c54..56780f59c 100644 --- a/chart/infra-server/static/workflow-openshift-rosa.yaml +++ b/chart/infra-server/static/workflow-openshift-rosa.yaml @@ -42,6 +42,7 @@ spec: - name: create activeDeadlineSeconds: 7200 + timeout: 2h container: image: quay.io/stackrox-io/ci:automation-flavors-rosa-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-osd-aws.yaml b/chart/infra-server/static/workflow-osd-aws.yaml index ff4bb4f3d..bdaa3fb8c 100644 --- a/chart/infra-server/static/workflow-osd-aws.yaml +++ b/chart/infra-server/static/workflow-osd-aws.yaml @@ -42,6 +42,7 @@ spec: - name: create activeDeadlineSeconds: 7200 + timeout: 2h container: image: quay.io/stackrox-io/ci:automation-flavors-osd-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-osd-gcp.yaml b/chart/infra-server/static/workflow-osd-gcp.yaml index 19d1dc71d..1a1972c9f 100644 --- a/chart/infra-server/static/workflow-osd-gcp.yaml +++ b/chart/infra-server/static/workflow-osd-gcp.yaml @@ -43,6 +43,7 @@ spec: - name: create activeDeadlineSeconds: 7200 + timeout: 2h container: image: quay.io/stackrox-io/ci:automation-flavors-osd-{{ .Chart.Annotations.automationFlavorsVersion }} imagePullPolicy: Always diff --git a/chart/infra-server/static/workflow-qa-demo.yaml b/chart/infra-server/static/workflow-qa-demo.yaml index e0b857373..a73eb7e29 100644 --- a/chart/infra-server/static/workflow-qa-demo.yaml +++ b/chart/infra-server/static/workflow-qa-demo.yaml @@ -79,6 +79,7 @@ spec: - name: create activeDeadlineSeconds: 7200 + timeout: 2h inputs: artifacts: - name: roxctl From be95325eb07db76ffc02754b9567e3a42b254343 Mon Sep 17 00:00:00 2001 From: Tom Martensen Date: Thu, 19 Nov 2026 14:54:34 +0100 Subject: [PATCH 3/3] fix(demo): make stop artifact references tolerate a partial create The create templates of the demo and qa-demo workflows export tfstate and tfvars as optional outputs, and the destroy template already declares its inputs optional, but the stop step referenced them non-optionally. If create is deadline-killed before exporting the artifacts, the exit handler fails to resolve the references and the destroy step never runs, leaking the partially created GCP resources. Mark the artifact arguments in stop as optional so destroy runs even when the state is missing. Co-Authored-By: Claude Code --- chart/infra-server/static/workflow-demo.yaml | 2 ++ chart/infra-server/static/workflow-qa-demo.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/chart/infra-server/static/workflow-demo.yaml b/chart/infra-server/static/workflow-demo.yaml index 933cfabc1..00a230c6e 100644 --- a/chart/infra-server/static/workflow-demo.yaml +++ b/chart/infra-server/static/workflow-demo.yaml @@ -49,8 +49,10 @@ spec: artifacts: - name: tfstate from: '{{ "{{" }}workflow.outputs.artifacts.global-tfstate{{ "}}" }}' + optional: true - name: tfvars from: '{{ "{{" }}workflow.outputs.artifacts.global-tfvars{{ "}}" }}' + optional: true - name: roxctl activeDeadlineSeconds: 600 diff --git a/chart/infra-server/static/workflow-qa-demo.yaml b/chart/infra-server/static/workflow-qa-demo.yaml index a73eb7e29..9961a0e2f 100644 --- a/chart/infra-server/static/workflow-qa-demo.yaml +++ b/chart/infra-server/static/workflow-qa-demo.yaml @@ -57,8 +57,10 @@ spec: artifacts: - name: tfstate from: '{{ "{{" }}workflow.outputs.artifacts.global-tfstate{{ "}}" }}' + optional: true - name: tfvars from: '{{ "{{" }}workflow.outputs.artifacts.global-tfvars{{ "}}" }}' + optional: true - name: roxctl activeDeadlineSeconds: 600