diff --git a/pkg/api/handlers/stellar/solver.go b/pkg/api/handlers/stellar/solver.go index f97da4e1cb..ccf9bcf987 100644 --- a/pkg/api/handlers/stellar/solver.go +++ b/pkg/api/handlers/stellar/solver.go @@ -1,7 +1,6 @@ package stellar import ( - "github.com/kubestellar/console/pkg/k8s" "context" "fmt" "html" @@ -12,22 +11,10 @@ import ( "github.com/gofiber/fiber/v2" - "encoding/json" - - "github.com/kubestellar/console/pkg/stellar" - "github.com/kubestellar/console/pkg/stellar/scheduler" "github.com/kubestellar/console/pkg/stellar/solver" "github.com/kubestellar/console/pkg/store" ) -// safeAutoActions are the action types autoTriggerSolve may dispatch on its -// own (Phase 3a, before falling through to the AI mission). The list matches -// the legacy autoExecuteAction allowlist — RestartDeployment is the only -// non-destructive action that's almost always safe to attempt. -var safeAutoActions = map[string]bool{ - "RestartDeployment": true, -} - // broadcastSolveProgress emits a structured phase update over SSE. The event // card uses it to render the live progress bar; the activity log uses the // matching kind to record the same step. Phase strings are stable contract. @@ -86,32 +73,6 @@ type solveFullStore interface { GetRecentSolveForWorkload(ctx context.Context, cluster, namespace, workload string, since time.Time) (*store.StellarSolve, error) } -// AutoSolveCooldown is the minimum window between back-to-back Solve attempts -// for the same workload. Only blocks when a solve is still RUNNING — terminal -// solves (resolved/escalated/exhausted) never block a fresh attempt. Cut to -// 5 min so demos don't feel stuck behind stale escalations. -const AutoSolveCooldown = 5 * time.Minute - -func formatBatchTimestamp(ts *time.Time) string { - if ts == nil { - return "unknown" - } - return ts.UTC().Format(time.RFC3339) -} - -func (h *Handler) isLatestEventBatch(ctx context.Context, notif *store.StellarNotification) (bool, *time.Time, error) { - latestBatchTimestamp, err := h.store.GetLatestEventBatchTimestamp(ctx) - if err != nil { - return false, nil, err - } - if notif == nil || notif.BatchTimestamp == nil || latestBatchTimestamp == nil { - return false, latestBatchTimestamp, nil - } - currentBatchTimestamp := notif.BatchTimestamp.UTC() - latestBatch := latestBatchTimestamp.UTC() - return currentBatchTimestamp.Equal(latestBatch), &latestBatch, nil -} - // logActivity is the single write-and-broadcast helper for Stellar's activity // log. UI subscribes via the SSE `activity` channel and renders the entries in // the dedicated StellarActivityPanel — not the chat, not the events column. @@ -157,319 +118,6 @@ func (h *Handler) ListActivity(c *fiber.Ctx) error { return c.JSON(fiber.Map{"items": items}) } -// autoTriggerSolve drives the autonomous-solve narrative for one critical -// event. Phased broadcasts (investigating → root_cause → solving) keep the -// event card's progress bar and the activity log in lockstep, so the operator -// sees Stellar's reasoning unfold in real time. -// -// Cooldown protects against re-firing on the same workload within -// AutoSolveCooldown. Cluster-client availability is *not* gated — the AI -// mission runs through the user's agent connection, not the backend's direct -// client, so demo-mode and partial setups still get a useful narrative. -func (h *Handler) autoTriggerSolve(ctx context.Context, event IncomingEvent, notif *store.StellarNotification, eval *stellar.EvaluationResult) { - full, ok := h.fullStore() - if !ok { - return - } - workload := deploymentNameFromPodName(event.Name) - - cooldownSince := time.Now().Add(-AutoSolveCooldown) - if recent, _ := full.GetRecentSolveForWorkload(ctx, event.Cluster, event.Namespace, workload, cooldownSince); recent != nil && recent.Status == "running" { - // A solve for this workload is RUNNING. Link this new event card to - // it so the operator sees the same progress bar — duplicating mission - // work would be wasteful. Terminal solves (escalated/resolved/ - // exhausted) intentionally fall through so Stellar gets another shot; - // the operator deserves a fresh attempt, not an inherited verdict. - h.broadcastToClients(SSEEvent{Type: "solve_started", Data: map[string]interface{}{ - "userId": notif.UserID, - "solveId": recent.ID, "eventId": notif.ID, - }}) - h.broadcastSolveProgress(notif.UserID, recent.ID, notif.ID, "solving", - fmt.Sprintf("Linked to active solve started %s ago.", - time.Since(recent.StartedAt).Round(time.Second)), 60) - h.logActivity(ctx, &store.StellarActivity{ - Kind: "decided_skip", - EventID: notif.ID, - SolveID: recent.ID, - Cluster: event.Cluster, - Namespace: event.Namespace, - Workload: workload, - Title: fmt.Sprintf("Linked to active solve for %s/%s", event.Namespace, workload), - Detail: fmt.Sprintf("Solve %s started %s ago is still running. Linking this event card so progress is shared.", recent.ID[:8], time.Since(recent.StartedAt).Round(time.Second)), - Severity: "info", - }) - return - } - - // Create the solve record up front so every subsequent broadcast carries - // the same solveID — that lets the frontend correlate progress phases - // with the same UI element instead of spawning new cards per phase. - solve := &store.StellarSolve{ - EventID: notif.ID, - UserID: notif.UserID, - Cluster: event.Cluster, - Namespace: event.Namespace, - Workload: workload, - Status: "running", - Summary: "Autonomous solve in progress.", - StartedAt: time.Now().UTC(), - } - if err := full.CreateSolve(ctx, solve); err != nil { - slog.Warn("stellar: auto-solve CreateSolve failed", "error", err) - return - } - - // Flip the card into "solving" mode immediately so the user never sees a - // dead critical event without status. - h.broadcastToClients(SSEEvent{Type: "solve_started", Data: map[string]interface{}{ - "userId": notif.UserID, - "solveId": solve.ID, "eventId": notif.ID, - }}) - - // PHASE 1 — INVESTIGATING. - // The first user-visible beat: Stellar acknowledges the event and says - // it's looking into it. Activity log and card progress both update. - investigatingMsg := fmt.Sprintf("Investigating %s on %s/%s — pulling logs and pod state.", - event.Reason, event.Namespace, workload) - h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "investigating", investigatingMsg, 20) - h.logActivity(ctx, &store.StellarActivity{ - Kind: "investigating", - EventID: notif.ID, - SolveID: solve.ID, - Cluster: event.Cluster, - Namespace: event.Namespace, - Workload: workload, - Title: fmt.Sprintf("Investigating %s on %s/%s", event.Reason, event.Namespace, workload), - Detail: "Examining pod logs, recent events, and deployment state.", - Severity: "info", - }) - - // PHASE 2 — ROOT CAUSE. - // Use the evaluator's reasoning as the headline if the LLM produced - // something substantive; fall back to the rule-based diagnosis line. - rootCauseHeadline := deriveDiagnosisHeadline(event, "critical", false) - rootCauseDetail := "" - if eval != nil { - rootCauseDetail = strings.TrimSpace(eval.Reasoning) - if eval.RecommendedAction != nil && eval.RecommendedAction.Reasoning != "" { - if rootCauseDetail != "" { - rootCauseDetail += "\n\n" - } - rootCauseDetail += "Recommendation: " + eval.RecommendedAction.Type + " — " + eval.RecommendedAction.Reasoning - } - } - if rootCauseDetail == "" { - rootCauseDetail = fmt.Sprintf("Reason: %s. Message: %s", - event.Reason, truncateString(event.Message, 200)) - } - h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "root_cause", "Root cause: "+rootCauseHeadline, 50) - h.logActivity(ctx, &store.StellarActivity{ - Kind: "root_cause", - EventID: notif.ID, - SolveID: solve.ID, - Cluster: event.Cluster, - Namespace: event.Namespace, - Workload: workload, - Title: "Root cause: " + rootCauseHeadline, - Detail: rootCauseDetail, - Severity: "info", - }) - - autoResolveAllowed, latestBatchTimestamp, batchErr := h.isLatestEventBatch(ctx, notif) - if batchErr != nil { - slog.Warn("stellar: latest batch lookup failed", "event_id", notif.ID, "error", batchErr) - } - if !autoResolveAllowed { - summary := fmt.Sprintf( - "Root cause analysis completed. Automated remediation skipped because this event belongs to batch %s while the latest batch is %s. Manual resolution is still available.", - formatBatchTimestamp(notif.BatchTimestamp), - formatBatchTimestamp(latestBatchTimestamp), - ) - _ = full.UpdateSolveStatus(ctx, solve.ID, "escalated", summary, "", "") - h.logActivity(ctx, &store.StellarActivity{ - Kind: "solve_escalated", - EventID: notif.ID, - SolveID: solve.ID, - Cluster: event.Cluster, - Namespace: event.Namespace, - Workload: workload, - Title: fmt.Sprintf("Escalated: older batch requires manual review for %s/%s", event.Namespace, workload), - Detail: summary, - Severity: "warning", - }) - h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "escalated", summary, 100) - h.broadcastToClients(SSEEvent{Type: "solve_complete", Data: map[string]interface{}{ - "userId": notif.UserID, - "solveId": solve.ID, - "eventId": notif.ID, - "status": "escalated", - "summary": summary, - }}) - return - } - - // PHASE 3a — TRY THE SAFE DETERMINISTIC FIX FIRST. - // If the evaluator recommended a safe action (RestartDeployment today), - // Stellar attempts it directly. This is the same path the legacy - // autoExecuteAction took, but folded into the solve narrative so the card - // and the activity log stay in lockstep. Success here marks the solve - // resolved (green ✓) and skips the AI mission entirely — JARVIS doesn't - // summon a heavy diagnostic when a rollout-restart already worked. - // - // On failure (or when the action isn't on the safe allowlist), we fall - // through to PHASE 3b (mission trigger) so the operator's connected AI - // agent can take a deeper look. - if eval != nil && eval.RecommendedAction != nil && safeAutoActions[eval.RecommendedAction.Type] && h.k8sClient != nil { - h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "solving", - fmt.Sprintf("Trying %s — Stellar's first-line fix.", eval.RecommendedAction.Type), 75) - h.logActivity(ctx, &store.StellarActivity{ - Kind: "solving", - EventID: notif.ID, - SolveID: solve.ID, - Cluster: event.Cluster, - Namespace: event.Namespace, - Workload: workload, - Title: fmt.Sprintf("Trying %s on %s/%s", eval.RecommendedAction.Type, event.Namespace, workload), - Detail: eval.RecommendedAction.Reasoning, - Severity: "info", - }) - - params := map[string]any{ - "namespace": event.Namespace, - "name": workload, - } - paramsJSON, _ := json.Marshal(params) - now := time.Now().UTC() - action := &store.StellarAction{ - UserID: notif.UserID, - Description: fmt.Sprintf("Solve %s: %s on %s/%s", solve.ID[:8], eval.RecommendedAction.Type, event.Namespace, workload), - ActionType: eval.RecommendedAction.Type, - Parameters: string(paramsJSON), - Cluster: event.Cluster, - Namespace: event.Namespace, - Status: "approved", - CreatedBy: "stellar-solver", - ApprovedBy: "stellar-solver", - ApprovedAt: &now, - } - if err := h.store.CreateStellarAction(ctx, action); err != nil { - slog.Warn("[StellarSolver] failed to create action", "solveId", solve.ID, "error", err) - } - if err := h.store.UpdateStellarActionStatus(ctx, action.ID, "running", "", ""); err != nil { - slog.Warn("[StellarSolver] failed to update action status to running", "actionId", action.ID, "error", err) - } - outcome, dispatchErr := scheduler.Dispatch(ctx, h.k8sClient.(*k8s.MultiClusterClient), *action) - status := "completed" - if dispatchErr != nil { - status = "failed" - outcome = dispatchErr.Error() - } - if err := h.store.UpdateStellarActionStatus(ctx, action.ID, status, outcome, ""); err != nil { - slog.Warn("[StellarSolver] failed to update action status", "actionId", action.ID, "status", status, "error", err) - } - - if dispatchErr == nil { - // Restart succeeded. Mark the solve resolved and broadcast green ✓ - // so every card for this workload reflects the fix. - summary := fmt.Sprintf("Tried %s and it worked. %s", eval.RecommendedAction.Type, outcome) - if err := full.UpdateSolveStatus(ctx, solve.ID, "resolved", summary, "", ""); err != nil { - slog.Warn("[StellarSolver] failed to update solve status", "solveId", solve.ID, "error", err) - } - h.logActivity(ctx, &store.StellarActivity{ - Kind: "solve_resolved", - EventID: notif.ID, - SolveID: solve.ID, - Cluster: event.Cluster, - Namespace: event.Namespace, - Workload: workload, - Title: fmt.Sprintf("Resolved: %s succeeded on %s/%s", eval.RecommendedAction.Type, event.Namespace, workload), - Detail: summary, - Severity: "info", - }) - h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "resolved", summary, 100) - h.broadcastToClients(SSEEvent{Type: "solve_complete", Data: map[string]interface{}{ - "userId": notif.UserID, - "solveId": solve.ID, - "eventId": notif.ID, - "status": "resolved", - "summary": summary, - }}) - // Mission not triggered — first-line fix was sufficient. - return - } - // Restart attempt failed — fall through to the AI mission for a - // deeper diagnose+act loop. Log the failed attempt so the operator - // sees the journey. - h.logActivity(ctx, &store.StellarActivity{ - Kind: "auto_fix_failed", - EventID: notif.ID, - SolveID: solve.ID, - Cluster: event.Cluster, - Namespace: event.Namespace, - Workload: workload, - Title: fmt.Sprintf("First-line fix failed for %s/%s", event.Namespace, workload), - Detail: fmt.Sprintf("%s — escalating to AI mission. Error: %s", eval.RecommendedAction.Type, dispatchErr.Error()), - Severity: "warning", - }) - } - - // PHASE 3b — AI MISSION. - // Broadcast the mission trigger. The frontend bridge consumes this and - // drives MissionContext.startMission with the user's connected agent + - // LLM — same machinery as the ConsoleIssuesCard "Repair" button. - safeEventCluster := renderUntrustedPromptData("k8s-event-cluster", event.Cluster) - safeEventNamespace := renderUntrustedPromptData("k8s-event-namespace", event.Namespace) - safeEventKind := renderUntrustedPromptData("k8s-event-kind", event.Kind) - safeEventName := renderUntrustedPromptData("k8s-event-name", event.Name) - safeEventReason := renderUntrustedPromptData("k8s-event-reason", event.Reason) - safeEventMessage := renderUntrustedPromptData("k8s-event-message", event.Message) - safeRootCauseHeadline := renderUntrustedPromptData("stellar-root-cause-headline", rootCauseHeadline) - missionPrompt := fmt.Sprintf(`I'm a Kubernetes operator and Stellar (your assistant peer) just flagged a critical event. Diagnose and fix it. - -Cluster: %s -Namespace: %s -Resource: %s/%s -Reason: %s -Message: %s -Suspected root cause: %s - -Please: -1. Pull the pod logs and 'describe' output for the affected resource. -2. Identify the root cause from those signals. -3. Apply the safest single action to fix it (rollout restart, scale, env/configmap edit, or rollback). -4. Verify the fix landed by re-checking pod status after ~15 seconds. -5. Report what you did, the outcome, and any follow-up I should know about. - -Don't ask me first — act. If you genuinely can't fix it safely, tell me what's blocking you.`, - safeEventCluster, safeEventNamespace, safeEventKind, safeEventName, safeEventReason, safeEventMessage, safeRootCauseHeadline) - - h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "solving", - "Applying fix via AI mission — using your connected agent.", 75) - h.logActivity(ctx, &store.StellarActivity{ - Kind: "solving", - EventID: notif.ID, - SolveID: solve.ID, - Cluster: event.Cluster, - Namespace: event.Namespace, - Workload: workload, - Title: fmt.Sprintf("Solving: applying fix to %s/%s", event.Namespace, workload), - Detail: "Triggering an AI mission through your connected agent. Watch the mission sidebar for live actions.", - Severity: "info", - }) - h.broadcastToClients(SSEEvent{Type: "mission_trigger", Data: map[string]interface{}{ - "userId": notif.UserID, - "solveId": solve.ID, - "eventId": notif.ID, - "cluster": event.Cluster, - "namespace": event.Namespace, - "workload": workload, - "reason": event.Reason, - "message": event.Message, - "title": fmt.Sprintf("Stellar: fix %s on %s/%s", event.Reason, event.Namespace, workload), - "prompt": missionPrompt, - }}) -} - // CompleteAutoMissionRequest is the body the frontend sends after a Stellar- // triggered AI mission ends, so the activity log + solve record reflect the // real-world outcome. Sent by StellarMissionBridge. diff --git a/pkg/api/handlers/stellar/solver_autotrigger.go b/pkg/api/handlers/stellar/solver_autotrigger.go new file mode 100644 index 0000000000..93e34c1c01 --- /dev/null +++ b/pkg/api/handlers/stellar/solver_autotrigger.go @@ -0,0 +1,362 @@ +package stellar + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "strings" + "time" + + "github.com/kubestellar/console/pkg/k8s" + "github.com/kubestellar/console/pkg/stellar" + "github.com/kubestellar/console/pkg/stellar/scheduler" + "github.com/kubestellar/console/pkg/store" +) + +// safeAutoActions are the action types autoTriggerSolve may dispatch on its +// own (Phase 3a, before falling through to the AI mission). The list matches +// the legacy autoExecuteAction allowlist — RestartDeployment is the only +// non-destructive action that's almost always safe to attempt. +var safeAutoActions = map[string]bool{ + "RestartDeployment": true, +} + +// AutoSolveCooldown is the minimum window between back-to-back Solve attempts +// for the same workload. Only blocks when a solve is still RUNNING — terminal +// solves (resolved/escalated/exhausted) never block a fresh attempt. Cut to +// 5 min so demos don't feel stuck behind stale escalations. +const AutoSolveCooldown = 5 * time.Minute + +func formatBatchTimestamp(ts *time.Time) string { + if ts == nil { + return "unknown" + } + return ts.UTC().Format(time.RFC3339) +} + +func (h *Handler) isLatestEventBatch(ctx context.Context, notif *store.StellarNotification) (bool, *time.Time, error) { + latestBatchTimestamp, err := h.store.GetLatestEventBatchTimestamp(ctx) + if err != nil { + return false, nil, err + } + if notif == nil || notif.BatchTimestamp == nil || latestBatchTimestamp == nil { + return false, latestBatchTimestamp, nil + } + currentBatchTimestamp := notif.BatchTimestamp.UTC() + latestBatch := latestBatchTimestamp.UTC() + return currentBatchTimestamp.Equal(latestBatch), &latestBatch, nil +} + +// autoTriggerSolve drives the autonomous-solve narrative for one critical +// event. Phased broadcasts (investigating → root_cause → solving) keep the +// event card's progress bar and the activity log in lockstep, so the operator +// sees Stellar's reasoning unfold in real time. +// +// Cooldown protects against re-firing on the same workload within +// AutoSolveCooldown. Cluster-client availability is *not* gated — the AI +// mission runs through the user's agent connection, not the backend's direct +// client, so demo-mode and partial setups still get a useful narrative. +func (h *Handler) autoTriggerSolve(ctx context.Context, event IncomingEvent, notif *store.StellarNotification, eval *stellar.EvaluationResult) { + full, ok := h.fullStore() + if !ok { + return + } + workload := deploymentNameFromPodName(event.Name) + + cooldownSince := time.Now().Add(-AutoSolveCooldown) + if recent, _ := full.GetRecentSolveForWorkload(ctx, event.Cluster, event.Namespace, workload, cooldownSince); recent != nil && recent.Status == "running" { + // A solve for this workload is RUNNING. Link this new event card to + // it so the operator sees the same progress bar — duplicating mission + // work would be wasteful. Terminal solves (escalated/resolved/ + // exhausted) intentionally fall through so Stellar gets another shot; + // the operator deserves a fresh attempt, not an inherited verdict. + h.broadcastToClients(SSEEvent{Type: "solve_started", Data: map[string]interface{}{ + "userId": notif.UserID, + "solveId": recent.ID, "eventId": notif.ID, + }}) + h.broadcastSolveProgress(notif.UserID, recent.ID, notif.ID, "solving", + fmt.Sprintf("Linked to active solve started %s ago.", + time.Since(recent.StartedAt).Round(time.Second)), 60) + h.logActivity(ctx, &store.StellarActivity{ + Kind: "decided_skip", + EventID: notif.ID, + SolveID: recent.ID, + Cluster: event.Cluster, + Namespace: event.Namespace, + Workload: workload, + Title: fmt.Sprintf("Linked to active solve for %s/%s", event.Namespace, workload), + Detail: fmt.Sprintf("Solve %s started %s ago is still running. Linking this event card so progress is shared.", recent.ID[:8], time.Since(recent.StartedAt).Round(time.Second)), + Severity: "info", + }) + return + } + + // Create the solve record up front so every subsequent broadcast carries + // the same solveID — that lets the frontend correlate progress phases + // with the same UI element instead of spawning new cards per phase. + solve := &store.StellarSolve{ + EventID: notif.ID, + UserID: notif.UserID, + Cluster: event.Cluster, + Namespace: event.Namespace, + Workload: workload, + Status: "running", + Summary: "Autonomous solve in progress.", + StartedAt: time.Now().UTC(), + } + if err := full.CreateSolve(ctx, solve); err != nil { + slog.Warn("stellar: auto-solve CreateSolve failed", "error", err) + return + } + + // Flip the card into "solving" mode immediately so the user never sees a + // dead critical event without status. + h.broadcastToClients(SSEEvent{Type: "solve_started", Data: map[string]interface{}{ + "userId": notif.UserID, + "solveId": solve.ID, "eventId": notif.ID, + }}) + + // PHASE 1 — INVESTIGATING. + // The first user-visible beat: Stellar acknowledges the event and says + // it's looking into it. Activity log and card progress both update. + investigatingMsg := fmt.Sprintf("Investigating %s on %s/%s — pulling logs and pod state.", + event.Reason, event.Namespace, workload) + h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "investigating", investigatingMsg, 20) + h.logActivity(ctx, &store.StellarActivity{ + Kind: "investigating", + EventID: notif.ID, + SolveID: solve.ID, + Cluster: event.Cluster, + Namespace: event.Namespace, + Workload: workload, + Title: fmt.Sprintf("Investigating %s on %s/%s", event.Reason, event.Namespace, workload), + Detail: "Examining pod logs, recent events, and deployment state.", + Severity: "info", + }) + + // PHASE 2 — ROOT CAUSE. + // Use the evaluator's reasoning as the headline if the LLM produced + // something substantive; fall back to the rule-based diagnosis line. + rootCauseHeadline := deriveDiagnosisHeadline(event, "critical", false) + rootCauseDetail := "" + if eval != nil { + rootCauseDetail = strings.TrimSpace(eval.Reasoning) + if eval.RecommendedAction != nil && eval.RecommendedAction.Reasoning != "" { + if rootCauseDetail != "" { + rootCauseDetail += "\n\n" + } + rootCauseDetail += "Recommendation: " + eval.RecommendedAction.Type + " — " + eval.RecommendedAction.Reasoning + } + } + if rootCauseDetail == "" { + rootCauseDetail = fmt.Sprintf("Reason: %s. Message: %s", + event.Reason, truncateString(event.Message, 200)) + } + h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "root_cause", "Root cause: "+rootCauseHeadline, 50) + h.logActivity(ctx, &store.StellarActivity{ + Kind: "root_cause", + EventID: notif.ID, + SolveID: solve.ID, + Cluster: event.Cluster, + Namespace: event.Namespace, + Workload: workload, + Title: "Root cause: " + rootCauseHeadline, + Detail: rootCauseDetail, + Severity: "info", + }) + + autoResolveAllowed, latestBatchTimestamp, batchErr := h.isLatestEventBatch(ctx, notif) + if batchErr != nil { + slog.Warn("stellar: latest batch lookup failed", "event_id", notif.ID, "error", batchErr) + } + if !autoResolveAllowed { + summary := fmt.Sprintf( + "Root cause analysis completed. Automated remediation skipped because this event belongs to batch %s while the latest batch is %s. Manual resolution is still available.", + formatBatchTimestamp(notif.BatchTimestamp), + formatBatchTimestamp(latestBatchTimestamp), + ) + _ = full.UpdateSolveStatus(ctx, solve.ID, "escalated", summary, "", "") + h.logActivity(ctx, &store.StellarActivity{ + Kind: "solve_escalated", + EventID: notif.ID, + SolveID: solve.ID, + Cluster: event.Cluster, + Namespace: event.Namespace, + Workload: workload, + Title: fmt.Sprintf("Escalated: older batch requires manual review for %s/%s", event.Namespace, workload), + Detail: summary, + Severity: "warning", + }) + h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "escalated", summary, 100) + h.broadcastToClients(SSEEvent{Type: "solve_complete", Data: map[string]interface{}{ + "userId": notif.UserID, + "solveId": solve.ID, + "eventId": notif.ID, + "status": "escalated", + "summary": summary, + }}) + return + } + + // PHASE 3a — TRY THE SAFE DETERMINISTIC FIX FIRST. + // If the evaluator recommended a safe action (RestartDeployment today), + // Stellar attempts it directly. This is the same path the legacy + // autoExecuteAction took, but folded into the solve narrative so the card + // and the activity log stay in lockstep. Success here marks the solve + // resolved (green ✓) and skips the AI mission entirely — JARVIS doesn't + // summon a heavy diagnostic when a rollout-restart already worked. + // + // On failure (or when the action isn't on the safe allowlist), we fall + // through to PHASE 3b (mission trigger) so the operator's connected AI + // agent can take a deeper look. + if eval != nil && eval.RecommendedAction != nil && safeAutoActions[eval.RecommendedAction.Type] && h.k8sClient != nil { + h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "solving", + fmt.Sprintf("Trying %s — Stellar's first-line fix.", eval.RecommendedAction.Type), 75) + h.logActivity(ctx, &store.StellarActivity{ + Kind: "solving", + EventID: notif.ID, + SolveID: solve.ID, + Cluster: event.Cluster, + Namespace: event.Namespace, + Workload: workload, + Title: fmt.Sprintf("Trying %s on %s/%s", eval.RecommendedAction.Type, event.Namespace, workload), + Detail: eval.RecommendedAction.Reasoning, + Severity: "info", + }) + + params := map[string]any{ + "namespace": event.Namespace, + "name": workload, + } + paramsJSON, _ := json.Marshal(params) + now := time.Now().UTC() + action := &store.StellarAction{ + UserID: notif.UserID, + Description: fmt.Sprintf("Solve %s: %s on %s/%s", solve.ID[:8], eval.RecommendedAction.Type, event.Namespace, workload), + ActionType: eval.RecommendedAction.Type, + Parameters: string(paramsJSON), + Cluster: event.Cluster, + Namespace: event.Namespace, + Status: "approved", + CreatedBy: "stellar-solver", + ApprovedBy: "stellar-solver", + ApprovedAt: &now, + } + if err := h.store.CreateStellarAction(ctx, action); err != nil { + slog.Warn("[StellarSolver] failed to create action", "solveId", solve.ID, "error", err) + } + if err := h.store.UpdateStellarActionStatus(ctx, action.ID, "running", "", ""); err != nil { + slog.Warn("[StellarSolver] failed to update action status to running", "actionId", action.ID, "error", err) + } + outcome, dispatchErr := scheduler.Dispatch(ctx, h.k8sClient.(*k8s.MultiClusterClient), *action) + status := "completed" + if dispatchErr != nil { + status = "failed" + outcome = dispatchErr.Error() + } + if err := h.store.UpdateStellarActionStatus(ctx, action.ID, status, outcome, ""); err != nil { + slog.Warn("[StellarSolver] failed to update action status", "actionId", action.ID, "status", status, "error", err) + } + + if dispatchErr == nil { + // Restart succeeded. Mark the solve resolved and broadcast green ✓ + // so every card for this workload reflects the fix. + summary := fmt.Sprintf("Tried %s and it worked. %s", eval.RecommendedAction.Type, outcome) + if err := full.UpdateSolveStatus(ctx, solve.ID, "resolved", summary, "", ""); err != nil { + slog.Warn("[StellarSolver] failed to update solve status", "solveId", solve.ID, "error", err) + } + h.logActivity(ctx, &store.StellarActivity{ + Kind: "solve_resolved", + EventID: notif.ID, + SolveID: solve.ID, + Cluster: event.Cluster, + Namespace: event.Namespace, + Workload: workload, + Title: fmt.Sprintf("Resolved: %s succeeded on %s/%s", eval.RecommendedAction.Type, event.Namespace, workload), + Detail: summary, + Severity: "info", + }) + h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "resolved", summary, 100) + h.broadcastToClients(SSEEvent{Type: "solve_complete", Data: map[string]interface{}{ + "userId": notif.UserID, + "solveId": solve.ID, + "eventId": notif.ID, + "status": "resolved", + "summary": summary, + }}) + // Mission not triggered — first-line fix was sufficient. + return + } + // Restart attempt failed — fall through to the AI mission for a + // deeper diagnose+act loop. Log the failed attempt so the operator + // sees the journey. + h.logActivity(ctx, &store.StellarActivity{ + Kind: "auto_fix_failed", + EventID: notif.ID, + SolveID: solve.ID, + Cluster: event.Cluster, + Namespace: event.Namespace, + Workload: workload, + Title: fmt.Sprintf("First-line fix failed for %s/%s", event.Namespace, workload), + Detail: fmt.Sprintf("%s — escalating to AI mission. Error: %s", eval.RecommendedAction.Type, dispatchErr.Error()), + Severity: "warning", + }) + } + + // PHASE 3b — AI MISSION. + // Broadcast the mission trigger. The frontend bridge consumes this and + // drives MissionContext.startMission with the user's connected agent + + // LLM — same machinery as the ConsoleIssuesCard "Repair" button. + safeEventCluster := renderUntrustedPromptData("k8s-event-cluster", event.Cluster) + safeEventNamespace := renderUntrustedPromptData("k8s-event-namespace", event.Namespace) + safeEventKind := renderUntrustedPromptData("k8s-event-kind", event.Kind) + safeEventName := renderUntrustedPromptData("k8s-event-name", event.Name) + safeEventReason := renderUntrustedPromptData("k8s-event-reason", event.Reason) + safeEventMessage := renderUntrustedPromptData("k8s-event-message", event.Message) + safeRootCauseHeadline := renderUntrustedPromptData("stellar-root-cause-headline", rootCauseHeadline) + missionPrompt := fmt.Sprintf(`I'm a Kubernetes operator and Stellar (your assistant peer) just flagged a critical event. Diagnose and fix it. + +Cluster: %s +Namespace: %s +Resource: %s/%s +Reason: %s +Message: %s +Suspected root cause: %s + +Please: +1. Pull the pod logs and 'describe' output for the affected resource. +2. Identify the root cause from those signals. +3. Apply the safest single action to fix it (rollout restart, scale, env/configmap edit, or rollback). +4. Verify the fix landed by re-checking pod status after ~15 seconds. +5. Report what you did, the outcome, and any follow-up I should know about. + +Don't ask me first — act. If you genuinely can't fix it safely, tell me what's blocking you.`, + safeEventCluster, safeEventNamespace, safeEventKind, safeEventName, safeEventReason, safeEventMessage, safeRootCauseHeadline) + + h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "solving", + "Applying fix via AI mission — using your connected agent.", 75) + h.logActivity(ctx, &store.StellarActivity{ + Kind: "solving", + EventID: notif.ID, + SolveID: solve.ID, + Cluster: event.Cluster, + Namespace: event.Namespace, + Workload: workload, + Title: fmt.Sprintf("Solving: applying fix to %s/%s", event.Namespace, workload), + Detail: "Triggering an AI mission through your connected agent. Watch the mission sidebar for live actions.", + Severity: "info", + }) + h.broadcastToClients(SSEEvent{Type: "mission_trigger", Data: map[string]interface{}{ + "userId": notif.UserID, + "solveId": solve.ID, + "eventId": notif.ID, + "cluster": event.Cluster, + "namespace": event.Namespace, + "workload": workload, + "reason": event.Reason, + "message": event.Message, + "title": fmt.Sprintf("Stellar: fix %s on %s/%s", event.Reason, event.Namespace, workload), + "prompt": missionPrompt, + }}) +}