diff --git a/apps/workspace-engine/pkg/jobagents/argo/application_upserter.go b/apps/workspace-engine/pkg/jobagents/argo/application_upserter.go index 7c02b510d..90ed77710 100644 --- a/apps/workspace-engine/pkg/jobagents/argo/application_upserter.go +++ b/apps/workspace-engine/pkg/jobagents/argo/application_upserter.go @@ -2,6 +2,7 @@ package argo import ( "context" + "errors" "fmt" "log/slog" "strings" @@ -76,6 +77,9 @@ func isRetryableError(err error) bool { if err == nil { return false } + if errors.Is(err, context.DeadlineExceeded) { + return true + } errStr := err.Error() return strings.Contains(errStr, "502") || strings.Contains(errStr, "503") || @@ -85,7 +89,8 @@ func isRetryableError(err error) bool { strings.Contains(errStr, "timeout") || strings.Contains(errStr, "temporarily unavailable") || strings.Contains(errStr, "EOF") || - strings.Contains(errStr, "Unavailable") + strings.Contains(errStr, "Unavailable") || + strings.Contains(errStr, "context deadline exceeded") } // GoApplicationDeleter is the production implementation of diff --git a/apps/workspace-engine/pkg/jobagents/argo/argoapp_test.go b/apps/workspace-engine/pkg/jobagents/argo/argoapp_test.go index 3d5133ca3..62eef6e12 100644 --- a/apps/workspace-engine/pkg/jobagents/argo/argoapp_test.go +++ b/apps/workspace-engine/pkg/jobagents/argo/argoapp_test.go @@ -637,6 +637,12 @@ func TestIsRetryableError(t *testing.T) { {"Unavailable", fmt.Errorf("Unavailable"), true}, {"not found", fmt.Errorf("application not found"), false}, {"permission denied", fmt.Errorf("permission denied"), false}, + {"context deadline exceeded sentinel", context.DeadlineExceeded, true}, + { + "context deadline exceeded grpc status", + status.Error(codes.DeadlineExceeded, "context deadline exceeded"), + true, + }, } for _, tt := range tests {