From 583edfd889617028180db9afb16ef78b577003ff Mon Sep 17 00:00:00 2001 From: Zihan Dai Date: Tue, 10 Nov 2026 01:38:08 +1100 Subject: [PATCH] fix(argocd): report a Degraded rollout as a failed deployment A sync can succeed while the rollout it starts never becomes healthy, and the plugin only looks at the sync phase, so such a deployment was stored with result SUCCESS. Record the application's health on the newest history entry when it is collected and turn a Succeeded sync whose rollout is Degraded into result FAILURE. Signed-off-by: Zihan Dai --- .../_raw_argocd_api_sync_operations.csv | 8 ++ .../_tool_argocd_sync_operations.csv | 4 + .../cicd_deployment_commits.csv | 4 + .../e2e/snapshot_tables/cicd_deployments.csv | 4 + .../plugins/argocd/e2e/sync_operation_test.go | 76 +++++++++++++++++++ .../argocd/tasks/sync_operation_collector.go | 32 ++++++++ .../tasks/sync_operation_collector_test.go | 40 ++++++++++ .../argocd/tasks/sync_operation_convertor.go | 16 +++- .../tasks/sync_operation_convertor_test.go | 21 +++++ .../argocd/tasks/sync_operation_extractor.go | 10 +++ 10 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 backend/plugins/argocd/e2e/raw_tables/_raw_argocd_api_sync_operations.csv create mode 100644 backend/plugins/argocd/e2e/snapshot_tables/_tool_argocd_sync_operations.csv create mode 100644 backend/plugins/argocd/e2e/snapshot_tables/cicd_deployment_commits.csv create mode 100644 backend/plugins/argocd/e2e/snapshot_tables/cicd_deployments.csv create mode 100644 backend/plugins/argocd/e2e/sync_operation_test.go diff --git a/backend/plugins/argocd/e2e/raw_tables/_raw_argocd_api_sync_operations.csv b/backend/plugins/argocd/e2e/raw_tables/_raw_argocd_api_sync_operations.csv new file mode 100644 index 00000000000..8cf9f5d5457 --- /dev/null +++ b/backend/plugins/argocd/e2e/raw_tables/_raw_argocd_api_sync_operations.csv @@ -0,0 +1,8 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""Name"":""myapp""}","{""phase"":""Succeeded"",""message"":""successfully synced (all tasks run)"",""startedAt"":""2026-11-03T09:59:30Z"",""finishedAt"":""2026-11-03T10:00:00Z"",""syncResult"":{""revision"":""bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb""}}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-03 12:00:00.000 +2,"{""ConnectionId"":1,""Name"":""myapp""}","{""id"":1,""revision"":""aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"",""deployedAt"":""2026-11-01T10:00:00Z"",""deployStartedAt"":""2026-11-01T09:59:30Z"",""source"":{""repoURL"":""https://github.com/example/app.git""},""initiatedBy"":{""automated"":true}}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-03 12:00:00.000 +3,"{""ConnectionId"":1,""Name"":""myapp""}","{""id"":2,""revision"":""bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"",""deployedAt"":""2026-11-03T10:00:00Z"",""deployStartedAt"":""2026-11-03T09:59:30Z"",""source"":{""repoURL"":""https://github.com/example/app.git""},""initiatedBy"":{""automated"":true},""applicationHealth"":""Degraded""}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-03 12:00:00.000 +4,"{""ConnectionId"":1,""Name"":""myapp""}","{""phase"":""Succeeded"",""message"":""successfully synced (all tasks run)"",""startedAt"":""2026-11-04T09:59:30Z"",""finishedAt"":""2026-11-04T10:00:00Z"",""syncResult"":{""revision"":""cccccccccccccccccccccccccccccccccccccccc""}}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-04 12:00:00.000 +5,"{""ConnectionId"":1,""Name"":""myapp""}","{""id"":1,""revision"":""aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"",""deployedAt"":""2026-11-01T10:00:00Z"",""deployStartedAt"":""2026-11-01T09:59:30Z"",""source"":{""repoURL"":""https://github.com/example/app.git""},""initiatedBy"":{""automated"":true}}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-04 12:00:00.000 +6,"{""ConnectionId"":1,""Name"":""myapp""}","{""id"":2,""revision"":""bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"",""deployedAt"":""2026-11-03T10:00:00Z"",""deployStartedAt"":""2026-11-03T09:59:30Z"",""source"":{""repoURL"":""https://github.com/example/app.git""},""initiatedBy"":{""automated"":true}}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-04 12:00:00.000 +7,"{""ConnectionId"":1,""Name"":""myapp""}","{""id"":3,""revision"":""cccccccccccccccccccccccccccccccccccccccc"",""deployedAt"":""2026-11-04T10:00:00Z"",""deployStartedAt"":""2026-11-04T09:59:30Z"",""source"":{""repoURL"":""https://github.com/example/app.git""},""initiatedBy"":{""automated"":true},""applicationHealth"":""Healthy""}",https://argocd.example.com/api/v1/applications/myapp,null,2026-11-04 12:00:00.000 diff --git a/backend/plugins/argocd/e2e/snapshot_tables/_tool_argocd_sync_operations.csv b/backend/plugins/argocd/e2e/snapshot_tables/_tool_argocd_sync_operations.csv new file mode 100644 index 00000000000..a347677c534 --- /dev/null +++ b/backend/plugins/argocd/e2e/snapshot_tables/_tool_argocd_sync_operations.csv @@ -0,0 +1,4 @@ +connection_id,application_name,deployment_id,revision,phase,health_status +1,myapp,1793527200,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,Succeeded, +1,myapp,1793700000,bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb,Succeeded,Degraded +1,myapp,1793786400,cccccccccccccccccccccccccccccccccccccccc,Succeeded,Healthy diff --git a/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployment_commits.csv b/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployment_commits.csv new file mode 100644 index 00000000000..30632647eec --- /dev/null +++ b/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployment_commits.csv @@ -0,0 +1,4 @@ +id,cicd_deployment_id,result,commit_sha +argocd:ArgocdSyncOperation:1:myapp:1793527200,argocd:ArgocdSyncOperation:1:myapp:1793527200,SUCCESS,aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +argocd:ArgocdSyncOperation:1:myapp:1793700000,argocd:ArgocdSyncOperation:1:myapp:1793700000,FAILURE,bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +argocd:ArgocdSyncOperation:1:myapp:1793786400,argocd:ArgocdSyncOperation:1:myapp:1793786400,SUCCESS,cccccccccccccccccccccccccccccccccccccccc diff --git a/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployments.csv b/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployments.csv new file mode 100644 index 00000000000..ec4e2d7308a --- /dev/null +++ b/backend/plugins/argocd/e2e/snapshot_tables/cicd_deployments.csv @@ -0,0 +1,4 @@ +id,result,status,original_status,original_result +argocd:ArgocdSyncOperation:1:myapp:1793527200,SUCCESS,DONE,Succeeded,Succeeded +argocd:ArgocdSyncOperation:1:myapp:1793700000,FAILURE,DONE,Succeeded,Degraded +argocd:ArgocdSyncOperation:1:myapp:1793786400,SUCCESS,DONE,Succeeded,Succeeded diff --git a/backend/plugins/argocd/e2e/sync_operation_test.go b/backend/plugins/argocd/e2e/sync_operation_test.go new file mode 100644 index 00000000000..27f24d3eb6d --- /dev/null +++ b/backend/plugins/argocd/e2e/sync_operation_test.go @@ -0,0 +1,76 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/devlake/core/models/domainlayer/devops" + "github.com/apache/devlake/helpers/e2ehelper" + "github.com/apache/devlake/plugins/argocd/impl" + "github.com/apache/devlake/plugins/argocd/models" + "github.com/apache/devlake/plugins/argocd/models/migrationscripts/archived" + "github.com/apache/devlake/plugins/argocd/tasks" +) + +func TestSyncOperationDataFlow(t *testing.T) { + var argocd impl.ArgoCD + dataflowTester := e2ehelper.NewDataFlowTester(t, "argocd", argocd) + + taskData := &tasks.ArgocdTaskData{ + Options: &tasks.ArgocdOptions{ + ConnectionId: 1, + ApplicationName: "myapp", + ScopeConfig: &models.ArgocdScopeConfig{}, + }, + } + + // Two collections: the first saw revision bbb Degraded, the second a newer Healthy + // sync, by which time bbb is an ordinary history entry again. + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_argocd_api_sync_operations.csv", "_raw_argocd_api_sync_operations") + dataflowTester.FlushTabler(&archived.ArgocdApplication{}) + dataflowTester.FlushTabler(&models.ArgocdRevisionImage{}) + + dataflowTester.FlushTabler(&models.ArgocdSyncOperation{}) + dataflowTester.Subtask(tasks.ExtractSyncOperationsMeta, taskData) + dataflowTester.VerifyTableWithOptions( + models.ArgocdSyncOperation{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_argocd_sync_operations.csv", + TargetFields: []string{"connection_id", "application_name", "deployment_id", "revision", "phase", "health_status"}, + }, + ) + + dataflowTester.FlushTabler(&devops.CICDDeployment{}) + dataflowTester.FlushTabler(&devops.CicdDeploymentCommit{}) + dataflowTester.Subtask(tasks.ConvertSyncOperationsMeta, taskData) + dataflowTester.VerifyTableWithOptions( + devops.CICDDeployment{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/cicd_deployments.csv", + TargetFields: []string{"id", "result", "status", "original_status", "original_result"}, + }, + ) + dataflowTester.VerifyTableWithOptions( + devops.CicdDeploymentCommit{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/cicd_deployment_commits.csv", + TargetFields: []string{"id", "cicd_deployment_id", "result", "commit_sha"}, + }, + ) +} diff --git a/backend/plugins/argocd/tasks/sync_operation_collector.go b/backend/plugins/argocd/tasks/sync_operation_collector.go index bac7e5e64cf..55303760723 100644 --- a/backend/plugins/argocd/tasks/sync_operation_collector.go +++ b/backend/plugins/argocd/tasks/sync_operation_collector.go @@ -69,6 +69,9 @@ func CollectSyncOperations(taskCtx plugin.SubTaskContext) errors.Error { ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { var appResponse struct { Status struct { + Health struct { + Status string `json:"status"` + } `json:"health"` History []json.RawMessage `json:"history"` OperationState json.RawMessage `json:"operationState"` } `json:"status"` @@ -79,6 +82,7 @@ func CollectSyncOperations(taskCtx plugin.SubTaskContext) errors.Error { return nil, err } + recordHealthOnLatestSync(appResponse.Status.OperationState, appResponse.Status.History, appResponse.Status.Health.Status) return sanitizeOperationEntries(appResponse.Status.OperationState, appResponse.Status.History), nil }, }) @@ -90,6 +94,34 @@ func CollectSyncOperations(taskCtx plugin.SubTaskContext) errors.Error { return collector.Execute() } +// recordHealthOnLatestSync stores the application's current health on the newest +// history entry. Argo CD only reports health for the application as a whole, so +// this is the one moment the value can be tied to a sync. When the operation in +// progress failed or is still running, that newer sync owns the current health +// and the last history entry is left untouched. +func recordHealthOnLatestSync(operationState json.RawMessage, history []json.RawMessage, health string) { + if health == "" { + return + } + var operation struct { + Phase string `json:"phase"` + } + if err := json.Unmarshal(operationState, &operation); err == nil && operation.Phase != "" && !isRedundantOperationState(operation.Phase) { + return + } + for i := len(history) - 1; i >= 0; i-- { + var entry map[string]json.RawMessage + if err := json.Unmarshal(history[i], &entry); err != nil || entry == nil { + continue + } + entry["applicationHealth"], _ = json.Marshal(health) + if tagged, err := json.Marshal(entry); err == nil { + history[i] = tagged + } + return + } +} + func sanitizeOperationEntries(operationState json.RawMessage, history []json.RawMessage) []json.RawMessage { results := make([]json.RawMessage, 0, len(history)+1) diff --git a/backend/plugins/argocd/tasks/sync_operation_collector_test.go b/backend/plugins/argocd/tasks/sync_operation_collector_test.go index 9737e880e93..62576e9b1ad 100644 --- a/backend/plugins/argocd/tasks/sync_operation_collector_test.go +++ b/backend/plugins/argocd/tasks/sync_operation_collector_test.go @@ -66,3 +66,43 @@ func TestSanitizeOperationEntries_TrimsWhitespace(t *testing.T) { assert.JSONEq(t, `{ "id":3 }`, string(result[0])) } } + +func TestRecordHealthOnLatestSync_TagsOnlyNewestHistoryEntry(t *testing.T) { + history := []json.RawMessage{ + json.RawMessage(`{"id": 1, "revision": "aaa"}`), + json.RawMessage(`{"id": 2, "revision": "bbb"}`), + json.RawMessage("null"), + } + + recordHealthOnLatestSync(json.RawMessage(`{"phase": "Succeeded"}`), history, "Degraded") + + assert.JSONEq(t, `{"id": 1, "revision": "aaa"}`, string(history[0])) + assert.JSONEq(t, `{"id": 2, "revision": "bbb", "applicationHealth": "Degraded"}`, string(history[1])) + assert.Equal(t, "null", string(history[2])) +} + +func TestRecordHealthOnLatestSync_NoOperationState(t *testing.T) { + history := []json.RawMessage{json.RawMessage(`{"id": 7}`)} + + recordHealthOnLatestSync(nil, history, "Healthy") + + assert.JSONEq(t, `{"id": 7, "applicationHealth": "Healthy"}`, string(history[0])) +} + +func TestRecordHealthOnLatestSync_NewerOperationOwnsTheHealth(t *testing.T) { + for _, phase := range []string{"Failed", "Error", "Running", "Terminating"} { + history := []json.RawMessage{json.RawMessage(`{"id": 1}`)} + + recordHealthOnLatestSync(json.RawMessage(`{"phase": "`+phase+`"}`), history, "Degraded") + + assert.JSONEq(t, `{"id": 1}`, string(history[0]), phase) + } +} + +func TestRecordHealthOnLatestSync_NoHealthReported(t *testing.T) { + history := []json.RawMessage{json.RawMessage(`{"id": 1}`)} + + recordHealthOnLatestSync(nil, history, "") + + assert.JSONEq(t, `{"id": 1}`, string(history[0])) +} diff --git a/backend/plugins/argocd/tasks/sync_operation_convertor.go b/backend/plugins/argocd/tasks/sync_operation_convertor.go index debf6c5a78b..5df8e650175 100644 --- a/backend/plugins/argocd/tasks/sync_operation_convertor.go +++ b/backend/plugins/argocd/tasks/sync_operation_convertor.go @@ -109,6 +109,8 @@ func ConvertSyncOperations(taskCtx plugin.SubTaskContext) errors.Error { } created = created.UTC() + result, originalResult := convertSyncOperationResult(syncOp) + deployment := &devops.CICDDeployment{ DomainEntity: domainlayer.DomainEntity{ Id: deploymentId, @@ -116,10 +118,10 @@ func ConvertSyncOperations(taskCtx plugin.SubTaskContext) errors.Error { CicdScopeId: scopeId, Name: fmt.Sprintf("%s:%d", syncOp.ApplicationName, syncOp.DeploymentId), DisplayTitle: syncOp.Message, - Result: convertPhaseToResult(syncOp.Phase), + Result: result, Status: convertPhaseToStatus(syncOp.Phase), OriginalStatus: syncOp.Phase, - OriginalResult: syncOp.Phase, + OriginalResult: originalResult, Environment: detectEnvironment(syncOp, application, data.Options.ScopeConfig, data.RegexEnricher), OriginalEnvironment: syncOp.ApplicationName, TaskDatesInfo: devops.TaskDatesInfo{ @@ -183,6 +185,16 @@ func ConvertSyncOperations(taskCtx plugin.SubTaskContext) errors.Error { return converter.Execute() } +// convertSyncOperationResult returns the deployment result of a sync and the Argo CD +// value it comes from. A sync only reports whether the manifests were applied, so a +// Succeeded sync whose rollout ended up Degraded is a failed deployment. +func convertSyncOperationResult(syncOp *models.ArgocdSyncOperation) (string, string) { + if syncOp.Phase == "Succeeded" && syncOp.HealthStatus == "Degraded" { + return devops.RESULT_FAILURE, syncOp.HealthStatus + } + return convertPhaseToResult(syncOp.Phase), syncOp.Phase +} + func convertPhaseToResult(phase string) string { switch phase { case "Succeeded": diff --git a/backend/plugins/argocd/tasks/sync_operation_convertor_test.go b/backend/plugins/argocd/tasks/sync_operation_convertor_test.go index d905d1c0253..58ecb46c305 100644 --- a/backend/plugins/argocd/tasks/sync_operation_convertor_test.go +++ b/backend/plugins/argocd/tasks/sync_operation_convertor_test.go @@ -94,6 +94,27 @@ func TestConvertPhaseToResult(t *testing.T) { assert.Equal(t, devops.RESULT_DEFAULT, convertPhaseToResult("Unknown")) } +func TestConvertSyncOperationResult(t *testing.T) { + cases := []struct { + phase string + health string + result string + originalResult string + }{ + {"Succeeded", "Degraded", devops.RESULT_FAILURE, "Degraded"}, + {"Succeeded", "Healthy", devops.RESULT_SUCCESS, "Succeeded"}, + {"Succeeded", "Progressing", devops.RESULT_SUCCESS, "Succeeded"}, + {"Succeeded", "", devops.RESULT_SUCCESS, "Succeeded"}, + {"Failed", "Degraded", devops.RESULT_FAILURE, "Failed"}, + {"Running", "Degraded", devops.RESULT_DEFAULT, "Running"}, + } + for _, c := range cases { + result, originalResult := convertSyncOperationResult(&models.ArgocdSyncOperation{Phase: c.phase, HealthStatus: c.health}) + assert.Equal(t, c.result, result, "%s/%s", c.phase, c.health) + assert.Equal(t, c.originalResult, originalResult, "%s/%s", c.phase, c.health) + } +} + func TestConvertPhaseToStatus(t *testing.T) { assert.Equal(t, devops.STATUS_DONE, convertPhaseToStatus("Succeeded")) assert.Equal(t, devops.STATUS_DONE, convertPhaseToStatus("Failed")) diff --git a/backend/plugins/argocd/tasks/sync_operation_extractor.go b/backend/plugins/argocd/tasks/sync_operation_extractor.go index 5138bbb0422..9feef30352c 100644 --- a/backend/plugins/argocd/tasks/sync_operation_extractor.go +++ b/backend/plugins/argocd/tasks/sync_operation_extractor.go @@ -66,6 +66,8 @@ type ArgocdApiSyncOperation struct { } `json:"initiatedBy"` Metadata ArgocdApiSyncOperationMetadata `json:"metadata"` Operation ArgocdApiSyncOperationDetails `json:"operation"` + // ApplicationHealth is not an Argo CD field, see recordHealthOnLatestSync + ApplicationHealth string `json:"applicationHealth"` // For operationState (current operation) Phase string `json:"phase"` // Succeeded, Failed, Error, Running, Terminating @@ -147,6 +149,7 @@ func ExtractSyncOperations(taskCtx plugin.SubTaskContext) errors.Error { } revisionDirty := make(map[string][]string) + lastSeenHealth := make(map[int64]string) extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ RawDataSubTaskArgs: api.RawDataSubTaskArgs{ @@ -246,6 +249,13 @@ func ExtractSyncOperations(taskCtx plugin.SubTaskContext) errors.Error { syncOp.StartedAt = start syncOp.Phase = "Succeeded" + // Raw rows are read oldest first and a sync only carries health while it + // is the newest one, so keep the last value seen for later rows. + if apiOp.ApplicationHealth != "" { + lastSeenHealth[syncOp.DeploymentId] = apiOp.ApplicationHealth + } + syncOp.HealthStatus = lastSeenHealth[syncOp.DeploymentId] + if apiOp.InitiatedBy.Automated { syncOp.InitiatedBy = "automated" } else {