Skip to content

refactor: split solver.go autoTriggerSolve pipeline into sibling file #24038

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hivecommons-hive merged 1 commit into from
Oct 6, 2026
Merged

refactor: split solver.go autoTriggerSolve pipeline into sibling file #24038

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
352 changes: 0 additions & 352 deletions pkg/api/handlers/stellar/solver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package stellar

import (
"github.com/kubestellar/console/pkg/k8s"
"context"
"fmt"
"html"
Expand All @@ -12,22 +11,10 @@ import (

"github.com/gofiber/fiber/v2"

"encoding/json"

"github.com/kubestellar/console/pkg/stellar"
"github.com/kubestellar/console/pkg/stellar/scheduler"
"github.com/kubestellar/console/pkg/stellar/solver"
"github.com/kubestellar/console/pkg/store"
)

// safeAutoActions are the action types autoTriggerSolve may dispatch on its
// own (Phase 3a, before falling through to the AI mission). The list matches
// the legacy autoExecuteAction allowlist — RestartDeployment is the only
// non-destructive action that's almost always safe to attempt.
var safeAutoActions = map[string]bool{
"RestartDeployment": true,
}

// broadcastSolveProgress emits a structured phase update over SSE. The event
// card uses it to render the live progress bar; the activity log uses the
// matching kind to record the same step. Phase strings are stable contract.
Expand Down Expand Up @@ -86,32 +73,6 @@ type solveFullStore interface {
GetRecentSolveForWorkload(ctx context.Context, cluster, namespace, workload string, since time.Time) (*store.StellarSolve, error)
}

// AutoSolveCooldown is the minimum window between back-to-back Solve attempts
// for the same workload. Only blocks when a solve is still RUNNING — terminal
// solves (resolved/escalated/exhausted) never block a fresh attempt. Cut to
// 5 min so demos don't feel stuck behind stale escalations.
const AutoSolveCooldown = 5 * time.Minute

func formatBatchTimestamp(ts *time.Time) string {
if ts == nil {
return "unknown"
}
return ts.UTC().Format(time.RFC3339)
}

func (h *Handler) isLatestEventBatch(ctx context.Context, notif *store.StellarNotification) (bool, *time.Time, error) {
latestBatchTimestamp, err := h.store.GetLatestEventBatchTimestamp(ctx)
if err != nil {
return false, nil, err
}
if notif == nil || notif.BatchTimestamp == nil || latestBatchTimestamp == nil {
return false, latestBatchTimestamp, nil
}
currentBatchTimestamp := notif.BatchTimestamp.UTC()
latestBatch := latestBatchTimestamp.UTC()
return currentBatchTimestamp.Equal(latestBatch), &latestBatch, nil
}

// logActivity is the single write-and-broadcast helper for Stellar's activity
// log. UI subscribes via the SSE `activity` channel and renders the entries in
// the dedicated StellarActivityPanel — not the chat, not the events column.
Expand Down Expand Up @@ -157,319 +118,6 @@ func (h *Handler) ListActivity(c *fiber.Ctx) error {
return c.JSON(fiber.Map{"items": items})
}

// autoTriggerSolve drives the autonomous-solve narrative for one critical
// event. Phased broadcasts (investigating → root_cause → solving) keep the
// event card's progress bar and the activity log in lockstep, so the operator
// sees Stellar's reasoning unfold in real time.
//
// Cooldown protects against re-firing on the same workload within
// AutoSolveCooldown. Cluster-client availability is *not* gated — the AI
// mission runs through the user's agent connection, not the backend's direct
// client, so demo-mode and partial setups still get a useful narrative.
func (h *Handler) autoTriggerSolve(ctx context.Context, event IncomingEvent, notif *store.StellarNotification, eval *stellar.EvaluationResult) {
full, ok := h.fullStore()
if !ok {
return
}
workload := deploymentNameFromPodName(event.Name)

cooldownSince := time.Now().Add(-AutoSolveCooldown)
if recent, _ := full.GetRecentSolveForWorkload(ctx, event.Cluster, event.Namespace, workload, cooldownSince); recent != nil && recent.Status == "running" {
// A solve for this workload is RUNNING. Link this new event card to
// it so the operator sees the same progress bar — duplicating mission
// work would be wasteful. Terminal solves (escalated/resolved/
// exhausted) intentionally fall through so Stellar gets another shot;
// the operator deserves a fresh attempt, not an inherited verdict.
h.broadcastToClients(SSEEvent{Type: "solve_started", Data: map[string]interface{}{
"userId": notif.UserID,
"solveId": recent.ID, "eventId": notif.ID,
}})
h.broadcastSolveProgress(notif.UserID, recent.ID, notif.ID, "solving",
fmt.Sprintf("Linked to active solve started %s ago.",
time.Since(recent.StartedAt).Round(time.Second)), 60)
h.logActivity(ctx, &store.StellarActivity{
Kind: "decided_skip",
EventID: notif.ID,
SolveID: recent.ID,
Cluster: event.Cluster,
Namespace: event.Namespace,
Workload: workload,
Title: fmt.Sprintf("Linked to active solve for %s/%s", event.Namespace, workload),
Detail: fmt.Sprintf("Solve %s started %s ago is still running. Linking this event card so progress is shared.", recent.ID[:8], time.Since(recent.StartedAt).Round(time.Second)),
Severity: "info",
})
return
}

// Create the solve record up front so every subsequent broadcast carries
// the same solveID — that lets the frontend correlate progress phases
// with the same UI element instead of spawning new cards per phase.
solve := &store.StellarSolve{
EventID: notif.ID,
UserID: notif.UserID,
Cluster: event.Cluster,
Namespace: event.Namespace,
Workload: workload,
Status: "running",
Summary: "Autonomous solve in progress.",
StartedAt: time.Now().UTC(),
}
if err := full.CreateSolve(ctx, solve); err != nil {
slog.Warn("stellar: auto-solve CreateSolve failed", "error", err)
return
}

// Flip the card into "solving" mode immediately so the user never sees a
// dead critical event without status.
h.broadcastToClients(SSEEvent{Type: "solve_started", Data: map[string]interface{}{
"userId": notif.UserID,
"solveId": solve.ID, "eventId": notif.ID,
}})

// PHASE 1 — INVESTIGATING.
// The first user-visible beat: Stellar acknowledges the event and says
// it's looking into it. Activity log and card progress both update.
investigatingMsg := fmt.Sprintf("Investigating %s on %s/%s — pulling logs and pod state.",
event.Reason, event.Namespace, workload)
h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "investigating", investigatingMsg, 20)
h.logActivity(ctx, &store.StellarActivity{
Kind: "investigating",
EventID: notif.ID,
SolveID: solve.ID,
Cluster: event.Cluster,
Namespace: event.Namespace,
Workload: workload,
Title: fmt.Sprintf("Investigating %s on %s/%s", event.Reason, event.Namespace, workload),
Detail: "Examining pod logs, recent events, and deployment state.",
Severity: "info",
})

// PHASE 2 — ROOT CAUSE.
// Use the evaluator's reasoning as the headline if the LLM produced
// something substantive; fall back to the rule-based diagnosis line.
rootCauseHeadline := deriveDiagnosisHeadline(event, "critical", false)
rootCauseDetail := ""
if eval != nil {
rootCauseDetail = strings.TrimSpace(eval.Reasoning)
if eval.RecommendedAction != nil && eval.RecommendedAction.Reasoning != "" {
if rootCauseDetail != "" {
rootCauseDetail += "\n\n"
}
rootCauseDetail += "Recommendation: " + eval.RecommendedAction.Type + " — " + eval.RecommendedAction.Reasoning
}
}
if rootCauseDetail == "" {
rootCauseDetail = fmt.Sprintf("Reason: %s. Message: %s",
event.Reason, truncateString(event.Message, 200))
}
h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "root_cause", "Root cause: "+rootCauseHeadline, 50)
h.logActivity(ctx, &store.StellarActivity{
Kind: "root_cause",
EventID: notif.ID,
SolveID: solve.ID,
Cluster: event.Cluster,
Namespace: event.Namespace,
Workload: workload,
Title: "Root cause: " + rootCauseHeadline,
Detail: rootCauseDetail,
Severity: "info",
})

autoResolveAllowed, latestBatchTimestamp, batchErr := h.isLatestEventBatch(ctx, notif)
if batchErr != nil {
slog.Warn("stellar: latest batch lookup failed", "event_id", notif.ID, "error", batchErr)
}
if !autoResolveAllowed {
summary := fmt.Sprintf(
"Root cause analysis completed. Automated remediation skipped because this event belongs to batch %s while the latest batch is %s. Manual resolution is still available.",
formatBatchTimestamp(notif.BatchTimestamp),
formatBatchTimestamp(latestBatchTimestamp),
)
_ = full.UpdateSolveStatus(ctx, solve.ID, "escalated", summary, "", "")
h.logActivity(ctx, &store.StellarActivity{
Kind: "solve_escalated",
EventID: notif.ID,
SolveID: solve.ID,
Cluster: event.Cluster,
Namespace: event.Namespace,
Workload: workload,
Title: fmt.Sprintf("Escalated: older batch requires manual review for %s/%s", event.Namespace, workload),
Detail: summary,
Severity: "warning",
})
h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "escalated", summary, 100)
h.broadcastToClients(SSEEvent{Type: "solve_complete", Data: map[string]interface{}{
"userId": notif.UserID,
"solveId": solve.ID,
"eventId": notif.ID,
"status": "escalated",
"summary": summary,
}})
return
}

// PHASE 3a — TRY THE SAFE DETERMINISTIC FIX FIRST.
// If the evaluator recommended a safe action (RestartDeployment today),
// Stellar attempts it directly. This is the same path the legacy
// autoExecuteAction took, but folded into the solve narrative so the card
// and the activity log stay in lockstep. Success here marks the solve
// resolved (green ✓) and skips the AI mission entirely — JARVIS doesn't
// summon a heavy diagnostic when a rollout-restart already worked.
//
// On failure (or when the action isn't on the safe allowlist), we fall
// through to PHASE 3b (mission trigger) so the operator's connected AI
// agent can take a deeper look.
if eval != nil && eval.RecommendedAction != nil && safeAutoActions[eval.RecommendedAction.Type] && h.k8sClient != nil {
h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "solving",
fmt.Sprintf("Trying %s — Stellar's first-line fix.", eval.RecommendedAction.Type), 75)
h.logActivity(ctx, &store.StellarActivity{
Kind: "solving",
EventID: notif.ID,
SolveID: solve.ID,
Cluster: event.Cluster,
Namespace: event.Namespace,
Workload: workload,
Title: fmt.Sprintf("Trying %s on %s/%s", eval.RecommendedAction.Type, event.Namespace, workload),
Detail: eval.RecommendedAction.Reasoning,
Severity: "info",
})

params := map[string]any{
"namespace": event.Namespace,
"name": workload,
}
paramsJSON, _ := json.Marshal(params)
now := time.Now().UTC()
action := &store.StellarAction{
UserID: notif.UserID,
Description: fmt.Sprintf("Solve %s: %s on %s/%s", solve.ID[:8], eval.RecommendedAction.Type, event.Namespace, workload),
ActionType: eval.RecommendedAction.Type,
Parameters: string(paramsJSON),
Cluster: event.Cluster,
Namespace: event.Namespace,
Status: "approved",
CreatedBy: "stellar-solver",
ApprovedBy: "stellar-solver",
ApprovedAt: &now,
}
if err := h.store.CreateStellarAction(ctx, action); err != nil {
slog.Warn("[StellarSolver] failed to create action", "solveId", solve.ID, "error", err)
}
if err := h.store.UpdateStellarActionStatus(ctx, action.ID, "running", "", ""); err != nil {
slog.Warn("[StellarSolver] failed to update action status to running", "actionId", action.ID, "error", err)
}
outcome, dispatchErr := scheduler.Dispatch(ctx, h.k8sClient.(*k8s.MultiClusterClient), *action)
status := "completed"
if dispatchErr != nil {
status = "failed"
outcome = dispatchErr.Error()
}
if err := h.store.UpdateStellarActionStatus(ctx, action.ID, status, outcome, ""); err != nil {
slog.Warn("[StellarSolver] failed to update action status", "actionId", action.ID, "status", status, "error", err)
}

if dispatchErr == nil {
// Restart succeeded. Mark the solve resolved and broadcast green ✓
// so every card for this workload reflects the fix.
summary := fmt.Sprintf("Tried %s and it worked. %s", eval.RecommendedAction.Type, outcome)
if err := full.UpdateSolveStatus(ctx, solve.ID, "resolved", summary, "", ""); err != nil {
slog.Warn("[StellarSolver] failed to update solve status", "solveId", solve.ID, "error", err)
}
h.logActivity(ctx, &store.StellarActivity{
Kind: "solve_resolved",
EventID: notif.ID,
SolveID: solve.ID,
Cluster: event.Cluster,
Namespace: event.Namespace,
Workload: workload,
Title: fmt.Sprintf("Resolved: %s succeeded on %s/%s", eval.RecommendedAction.Type, event.Namespace, workload),
Detail: summary,
Severity: "info",
})
h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "resolved", summary, 100)
h.broadcastToClients(SSEEvent{Type: "solve_complete", Data: map[string]interface{}{
"userId": notif.UserID,
"solveId": solve.ID,
"eventId": notif.ID,
"status": "resolved",
"summary": summary,
}})
// Mission not triggered — first-line fix was sufficient.
return
}
// Restart attempt failed — fall through to the AI mission for a
// deeper diagnose+act loop. Log the failed attempt so the operator
// sees the journey.
h.logActivity(ctx, &store.StellarActivity{
Kind: "auto_fix_failed",
EventID: notif.ID,
SolveID: solve.ID,
Cluster: event.Cluster,
Namespace: event.Namespace,
Workload: workload,
Title: fmt.Sprintf("First-line fix failed for %s/%s", event.Namespace, workload),
Detail: fmt.Sprintf("%s — escalating to AI mission. Error: %s", eval.RecommendedAction.Type, dispatchErr.Error()),
Severity: "warning",
})
}

// PHASE 3b — AI MISSION.
// Broadcast the mission trigger. The frontend bridge consumes this and
// drives MissionContext.startMission with the user's connected agent +
// LLM — same machinery as the ConsoleIssuesCard "Repair" button.
safeEventCluster := renderUntrustedPromptData("k8s-event-cluster", event.Cluster)
safeEventNamespace := renderUntrustedPromptData("k8s-event-namespace", event.Namespace)
safeEventKind := renderUntrustedPromptData("k8s-event-kind", event.Kind)
safeEventName := renderUntrustedPromptData("k8s-event-name", event.Name)
safeEventReason := renderUntrustedPromptData("k8s-event-reason", event.Reason)
safeEventMessage := renderUntrustedPromptData("k8s-event-message", event.Message)
safeRootCauseHeadline := renderUntrustedPromptData("stellar-root-cause-headline", rootCauseHeadline)
missionPrompt := fmt.Sprintf(`I'm a Kubernetes operator and Stellar (your assistant peer) just flagged a critical event. Diagnose and fix it.

Cluster: %s
Namespace: %s
Resource: %s/%s
Reason: %s
Message: %s
Suspected root cause: %s

Please:
1. Pull the pod logs and 'describe' output for the affected resource.
2. Identify the root cause from those signals.
3. Apply the safest single action to fix it (rollout restart, scale, env/configmap edit, or rollback).
4. Verify the fix landed by re-checking pod status after ~15 seconds.
5. Report what you did, the outcome, and any follow-up I should know about.

Don't ask me first — act. If you genuinely can't fix it safely, tell me what's blocking you.`,
safeEventCluster, safeEventNamespace, safeEventKind, safeEventName, safeEventReason, safeEventMessage, safeRootCauseHeadline)

h.broadcastSolveProgress(notif.UserID, solve.ID, notif.ID, "solving",
"Applying fix via AI mission — using your connected agent.", 75)
h.logActivity(ctx, &store.StellarActivity{
Kind: "solving",
EventID: notif.ID,
SolveID: solve.ID,
Cluster: event.Cluster,
Namespace: event.Namespace,
Workload: workload,
Title: fmt.Sprintf("Solving: applying fix to %s/%s", event.Namespace, workload),
Detail: "Triggering an AI mission through your connected agent. Watch the mission sidebar for live actions.",
Severity: "info",
})
h.broadcastToClients(SSEEvent{Type: "mission_trigger", Data: map[string]interface{}{
"userId": notif.UserID,
"solveId": solve.ID,
"eventId": notif.ID,
"cluster": event.Cluster,
"namespace": event.Namespace,
"workload": workload,
"reason": event.Reason,
"message": event.Message,
"title": fmt.Sprintf("Stellar: fix %s on %s/%s", event.Reason, event.Namespace, workload),
"prompt": missionPrompt,
}})
}

// CompleteAutoMissionRequest is the body the frontend sends after a Stellar-
// triggered AI mission ends, so the activity log + solve record reflect the
// real-world outcome. Sent by StellarMissionBridge.
Expand Down
Loading
Loading