Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions internal/api/handler_history.go
Original file line number Diff line number Diff line change
Expand Up @@ -412,14 +412,25 @@ func executionToHistoryRow(exec config.PurchaseExecution, approver, createdByEma
// fixing the config makes Retry work). Same call and input as the
// retry handler's gate so the two cannot disagree (issue #161).
if reason := purchase.RedriveRefusalReason(&exec); reason != "" {
row.OpsHint = reason
row.RedriveUnsafe = true
// Only a proven pre-send AWS refusal on every rec means nothing was
// bought; for any other outcome the duplicate-purchase warning stays.
// ops_hint stays set either way: the History UI hides Retry on it.
if purchase.IsDefiniteAWSRefusal(exec.Recommendations, exec.Error) {
row.OpsHint = definiteRefusalOpsHint
} else {
row.OpsHint = reason
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
}
}
annotateHistoryRowByStatus(&row, exec, approver)
return row
}

// definiteRefusalOpsHint replaces the duplicate-purchase warning when AWS
// refused the purchase before making it.
const definiteRefusalOpsHint = "AWS refused this purchase before it was made; start a new purchase after fixing the cause"

// annotateHistoryRowByStatus fills in Approver + StatusDescription on the
// row based on exec.Status. Split from executionToHistoryRow to keep the
// switch below under the gocyclo threshold.
Expand All @@ -428,7 +439,7 @@ func annotateHistoryRowByStatus(row *config.PurchaseHistoryRecord, exec config.P
case "pending", "notified":
row.Approver = approver
case "failed":
row.StatusDescription = exec.Error
row.StatusDescription = purchase.SanitizeStoredError(exec.Error)
case "expired":
row.StatusDescription = "approval link expired (not approved within 7 days)"
case config.StatusCanceled, config.LegacyStatusCanceled:
Expand Down Expand Up @@ -482,7 +493,7 @@ func annotateInFlightOrAuditGapRow(row *config.PurchaseHistoryRecord, exec confi
// double-count guard as the audit-gap completed case below.
row.IsAuditGap = true
annotateApproved(row, exec, approver)
row.StatusDescription = "partially completed — some commitments succeeded, others failed: " + exec.Error
row.StatusDescription = "partially completed — some commitments succeeded, others failed: " + purchase.SanitizeStoredError(exec.Error)
case "completed":
// Only audit-gap completed executions reach here (fetchExecutionsAsHistory
// skips clean completed rows). exec.Error carries why the history write
Expand All @@ -494,7 +505,7 @@ func annotateInFlightOrAuditGapRow(row *config.PurchaseHistoryRecord, exec confi
row.IsAuditGap = true
annotateApproved(row, exec, approver)
if exec.Error != "" {
row.StatusDescription = "purchase completed but its history record could not be saved: " + exec.Error
row.StatusDescription = "purchase completed but its history record could not be saved: " + purchase.SanitizeStoredError(exec.Error)
}
}
}
Expand Down
53 changes: 43 additions & 10 deletions internal/api/handler_purchases.go
Original file line number Diff line number Diff line change
Expand Up @@ -527,18 +527,40 @@ func executeFailureError(executionID, action string, final *config.PurchaseExecu
}
switch {
case errors.Is(err, config.ErrAuditLoss):
return NewClientError(500, fmt.Sprintf("execution %s ran but its final status could not be saved: %v", executionID, err))
logRawExecuteFailure(executionID, err)
return NewClientError(500, fmt.Sprintf("execution %s ran but its final status could not be saved (reference: execution %s)", executionID, executionID))
case errors.Is(err, purchase.ErrClaimedRowRefused):
return claimedRowRefusalError(final, err)
case errors.Is(err, purchase.ErrStaleAzurePricing):
return NewClientError(409, err.Error())
return NewClientError(409, purchase.ErrStaleAzurePricing.Error())
case purchase.IsCommittedPartialRun(final, err):
return NewClientErrorWithDetails(502, fmt.Sprintf("execution %s partially completed: some commitments were purchased and can be revoked, others failed: %v", executionID, err),
map[string]any{"execution_id": executionID, "status": final.Status})
category, msg := summarizeExecuteFailure(executionID, err)
return NewClientErrorWithDetails(502, fmt.Sprintf("execution %s partially completed: some commitments were purchased and can be revoked, others failed: %s (reference: execution %s)", executionID, msg, executionID),
map[string]any{"execution_id": executionID, "status": final.Status, "category": category})
default:
return NewClientErrorWithDetails(502, fmt.Sprintf("execution %s failed: %v", executionID, err),
map[string]any{"execution_id": executionID, "status": final.Status})
category, msg := summarizeExecuteFailure(executionID, err)
return NewClientErrorWithDetails(502, fmt.Sprintf("execution %s failed: %s (reference: execution %s)", executionID, msg, executionID),
map[string]any{"execution_id": executionID, "status": final.Status, "category": category})
}
}

// logRawExecuteFailure keeps the provider's own error text in the server log
// only; responses carry a category message and the execution ID.
func logRawExecuteFailure(executionID string, err error) {
logging.Errorf("purchase[%s]: execution failed: %v", executionID, err)
}

func summarizeExecuteFailure(executionID string, err error) (category, message string) {
logRawExecuteFailure(executionID, err)
category, message = purchase.SummarizeProviderError(err)
if category == purchase.ErrCategoryUnknown {
// Only known server-authored refusals (for example "re-fetch the
// recommendation") stay readable; any other text is the category message.
if text, ok := purchase.ServerNoteMessage(err); ok {
message = text
}
}
return category, message
}

// unclaimedExecuteError answers a failure where the funnel returned no row:
Expand Down Expand Up @@ -1059,7 +1081,7 @@ func tokenActionError(err error) error {
case errors.Is(err, purchase.ErrClaimedRowRefused):
return claimedRowRefusalError(nil, err)
case errors.Is(err, purchase.ErrStaleAzurePricing):
return NewClientError(409, err.Error())
return NewClientError(409, purchase.ErrStaleAzurePricing.Error())
case errors.Is(err, purchase.ErrInvalidApprovalToken), errors.Is(err, config.ErrApprovalTokenChanged):
return NewClientError(403, "invalid approval token")
case errors.Is(err, purchase.ErrApprovalTokenExpired):
Expand Down Expand Up @@ -2638,7 +2660,7 @@ func checkRetryEligibilityGates(failedExec *config.PurchaseExecution, req *event
if hint := resolveOpsHint(failedExec.Error); hint != "" {
return NewClientErrorWithDetails(409,
"this failure is operator-fixable; retrying without changing configuration will fail again",
map[string]any{"ops_hint": hint, "failure_reason": failedExec.Error})
map[string]any{"ops_hint": hint, "failure_reason": purchase.SanitizeStoredError(failedExec.Error)})
}

// Threshold soft-block (Q2). force=true (set by the frontend
Expand Down Expand Up @@ -3076,6 +3098,17 @@ func (h *Handler) tryGetSession(ctx context.Context, req *events.LambdaFunctionU
return session
}

// sanitizedRecommendationErrors copies recs with provider text in Error
// replaced, leaving the stored slice untouched.
func sanitizedRecommendationErrors(recs []config.RecommendationRecord) []config.RecommendationRecord {
out := make([]config.RecommendationRecord, len(recs))
copy(out, recs)
for i := range out {
out[i].Error = purchase.SanitizeStoredError(out[i].Error)
}
return out
}

// buildPurchaseDetailsResponse builds the response map for a purchase execution.
func buildPurchaseDetailsResponse(execution *config.PurchaseExecution, planName string) map[string]any {
response := map[string]any{
Expand All @@ -3087,7 +3120,7 @@ func buildPurchaseDetailsResponse(execution *config.PurchaseExecution, planName
"scheduled_date": execution.ScheduledDate.Format("2006-01-02"),
"total_upfront_cost": execution.TotalUpfrontCost,
"estimated_savings": execution.EstimatedSavings,
"recommendations": execution.Recommendations,
"recommendations": sanitizedRecommendationErrors(execution.Recommendations),
}

if execution.NotificationSent != nil {
Expand All @@ -3097,7 +3130,7 @@ func buildPurchaseDetailsResponse(execution *config.PurchaseExecution, planName
response["completed_at"] = execution.CompletedAt.Format("2006-01-02T15:04:05Z")
}
if execution.Error != "" {
response["error"] = execution.Error
response["error"] = purchase.SanitizeStoredError(execution.Error)
}
return response
}
Expand Down
4 changes: 2 additions & 2 deletions internal/api/handler_purchases_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2771,7 +2771,7 @@ func TestHandler_getPurchaseDetails_WithTimestamps(t *testing.T) {
ScheduledDate: scheduledDate,
NotificationSent: &notificationSent,
CompletedAt: &completedAt,
Error: "some error",
Error: "ramp not advanced: some error",
}

mockAuth.On("ValidateSession", ctx, "admin-token").Return(adminSession, nil)
Expand All @@ -2793,7 +2793,7 @@ func TestHandler_getPurchaseDetails_WithTimestamps(t *testing.T) {
assert.Equal(t, "completed", resultMap["status"])
assert.NotNil(t, resultMap["notification_sent"])
assert.NotNil(t, resultMap["completed_at"])
assert.Equal(t, "some error", resultMap["error"])
assert.Equal(t, "ramp not advanced", resultMap["error"])
}

// Tests for executePurchase
Expand Down
53 changes: 53 additions & 0 deletions internal/api/history_duplicate_hint_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package api

import (
"strings"
"testing"

"github.com/LeanerCloud/cloud-commitments-platform/internal/config"
"github.com/stretchr/testify/assert"
)

func TestHistoryRow_DuplicateGuardHint(t *testing.T) {
ec2 := []config.RecommendationRecord{{Provider: "aws", Service: "ec2"}}
denied := executionToHistoryRow(config.PurchaseExecution{Status: "failed", Error: fakeAWSDenied, Recommendations: ec2}, "", "")
assert.False(t, strings.Contains(denied.OpsHint, "duplicate guard"), "definite refusal hides the warning")
unknown := executionToHistoryRow(config.PurchaseExecution{Status: "failed", Error: "purchase outcome unknown", Recommendations: ec2}, "", "")
assert.Contains(t, unknown.OpsHint, "duplicate guard")
assert.True(t, unknown.RedriveUnsafe)

// One refused rec does not prove that another, whose outcome is unknown, bought nothing.
mixed := executionToHistoryRow(config.PurchaseExecution{Status: "failed", Error: "t4g.nano: purchase outcome unknown; t4g.micro: " + fakeAWSDenied, Recommendations: ec2}, "", "")
assert.Contains(t, mixed.OpsHint, "duplicate guard", "a mixed failure keeps the warning")
allDenied := executionToHistoryRow(config.PurchaseExecution{Status: "failed", Error: "t4g.nano: " + fakeAWSDenied + "; t4g.micro: " + fakeAWSDenied, Recommendations: ec2}, "", "")
assert.NotContains(t, allDenied.OpsHint, "duplicate guard", "every rec refused hides the warning")
}

func TestHistoryRow_DuplicateHintKeptOutsideDefiniteAWSRefusals(t *testing.T) {
const gcpRate = "commitment creation failed: googleapi: Error 403: Rate Limit Exceeded, rateLimitExceeded"
const gcpNotFound = "commitment creation failed: googleapi: Error 404: operations/op-1 was not found, notFound"
azure500 := azureHTTPError(t, 500, "/providers/Microsoft.Capacity/reservationOrders/x").Error()
azure429 := "PUT https://management.azure.com/x\nRESPONSE 429: 429 Too Many\nERROR CODE: TooManyRequests"
aws503 := strings.Replace(fakeAWSDenied, "StatusCode: 403", "StatusCode: 503", 1)
ec2 := config.RecommendationRecord{Provider: "aws", Service: "ec2"}
azSP := config.RecommendationRecord{Provider: "azure", Service: "savingsplans"}
gcp := config.RecommendationRecord{Provider: "gcp", Service: "compute"}
cases := []struct {
name string
recs []config.RecommendationRecord
err string
}{
{"gcp rate limit", []config.RecommendationRecord{gcp, ec2}, gcpRate},
{"gcp not found", []config.RecommendationRecord{gcp, ec2}, gcpNotFound},
{"azure 500 on a capacity url", []config.RecommendationRecord{azSP}, azure500},
{"azure 429", []config.RecommendationRecord{azSP}, azure429},
{"aws 5xx", []config.RecommendationRecord{ec2}, aws503},
{"aws without a code", []config.RecommendationRecord{ec2}, "operation error EC2: X, https response error StatusCode: 403"},
{"non-aws rec beside a definite aws refusal", []config.RecommendationRecord{ec2, gcp}, fakeAWSDenied},
}
for _, c := range cases {
row := executionToHistoryRow(config.PurchaseExecution{Status: "failed", Error: c.err, Recommendations: c.recs}, "", "")
assert.True(t, row.RedriveUnsafe, c.name)
assert.Contains(t, row.OpsHint, "duplicate guard", c.name)
}
}
96 changes: 96 additions & 0 deletions internal/api/purchase_failure_notes_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package api

import (
"encoding/json"
"errors"
"fmt"
"strings"
"testing"

"github.com/LeanerCloud/cloud-commitments-platform/internal/config"
"github.com/LeanerCloud/cloud-commitments-platform/internal/purchase"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

const fakeConnectErr = "failed to connect to `host=db.internal.example user=cudly_app database=cudly`: " +
"dial error (dial tcp 10.0.0.5:5432: connect: connection refused)"

func assertNoConnectText(t *testing.T, where, s string) {
t.Helper()
for _, n := range []string{"db.internal.example", "cudly_app", "10.0.0.5", "dial tcp"} {
assert.NotContains(t, s, n, where)
}
}

func TestTailedNotesWithStoreErrors_NeverReachUserSurfaces(t *testing.T) {
notes := []string{
"ramp not advanced: " + fakeConnectErr,
"refusing to execute: ladder tranche lookup failed: " + fakeConnectErr,
"refusing to execute ladder purchase: global config unreadable: " + fakeConnectErr,
purchase.ErrStaleAzurePricing.Error() + ": cannot read the latest collection state to confirm Azure prices are current: " + fakeConnectErr,
}
for _, note := range notes {
name := note[:strings.Index(note, ":")]
t.Run(name, func(t *testing.T) {
row := executionToHistoryRow(config.PurchaseExecution{ExecutionID: "e-1", Status: "failed", Error: note}, "", "")
assertNoConnectText(t, "status_description", row.StatusDescription)
assert.NotEmpty(t, row.StatusDescription)

exec := &config.PurchaseExecution{ExecutionID: "e-1", Status: "failed", Error: note}
body, err := json.Marshal(buildPurchaseDetailsResponse(exec, "plan"))
require.NoError(t, err)
assertNoConnectText(t, "details", string(body))

creator := retryCallerID
failed := &config.PurchaseExecution{ExecutionID: retryExecID, Status: "failed", CreatedByUserID: &creator,
Error: "FROM_EMAIL not configured for this deployment; " + note}
handler, _, _ := buildSessionRetryHandler(t, failed, &Session{UserID: retryCallerID}, false, true)
_, rErr := handler.retryPurchase(t.Context(), sessionRetryReq(), retryExecID)
ce, ok := IsClientError(rErr)
require.True(t, ok)
assert.Equal(t, 409, ce.code)
raw, mErr := json.Marshal(ce.Details())
require.NoError(t, mErr)
assertNoConnectText(t, "retry 409", string(raw)+ce.Error())
assert.NotEmpty(t, ce.Details()["failure_reason"])
})
}
}

func TestExecuteFailure_SentinelWrappingProviderErrorShowsOnlySentinelText(t *testing.T) {
for _, sentinel := range []error{purchase.ErrStaleAzurePricing, purchase.ErrClaimedRowRefused} {
err := fmt.Errorf("%w: %w", sentinel, errors.New(fakeConnectErr))
_, msg := summarizeExecuteFailure("e-1", err)
assert.Equal(t, sentinel.Error(), msg)
out := executeFailureError("e-1", "executed", &config.PurchaseExecution{Status: "failed"}, err)
require.Error(t, out)
// Non-client errors render as a generic 500 (handler.go), so only
// structured client errors can carry text to the caller.
if ce, ok := IsClientError(out); ok {
assertNoConnectText(t, "live", ce.Error())
}
}
}

func TestTokenActionError_StalePricingShowsOnlySentinelText(t *testing.T) {
err := tokenActionError(fmt.Errorf("%w: %w", purchase.ErrStaleAzurePricing, errors.New(fakeConnectErr)))
ce, ok := IsClientError(err)
require.True(t, ok)
assert.Equal(t, purchase.ErrStaleAzurePricing.Error(), ce.Error())
}

// A refused AWS purchase keeps ops_hint (the History UI hides Retry on it) but
// carries no duplicate-purchase warning.
func TestHistoryRow_DefiniteAWSRefusalKeepsOpsHintWithoutDuplicateWarning(t *testing.T) {
denied := "operation error EC2: X, https response error StatusCode: 403, RequestID: r, api error UnauthorizedOperation: msg"
row := executionToHistoryRow(config.PurchaseExecution{
ExecutionID: "e-1", Status: "failed", Error: denied,
Recommendations: []config.RecommendationRecord{{Provider: "aws", Service: "ec2"}},
}, "", "")
assert.True(t, row.RedriveUnsafe)
assert.NotEmpty(t, row.OpsHint)
assert.NotContains(t, strings.ToLower(row.OpsHint), "duplicate guard")
assert.NotEqual(t, purchase.RedriveRefusalReason(&config.PurchaseExecution{
Recommendations: []config.RecommendationRecord{{Provider: "aws", Service: "ec2"}}}), row.OpsHint)
}
Loading
Loading