From e41aa032cd380e969c8b5a9ada3f56c170e613f2 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 13:14:03 +0100 Subject: [PATCH 01/46] feat(cluster): Add live setup and scoped peer access --- internal/api/cluster_handlers.go | 180 ++++++++++++++++++++++---- internal/api/cluster_handlers_test.go | 99 +++++++++++++- internal/api/server.go | 2 + internal/cluster/capabilities.go | 21 +++ internal/orchestrator/provider.go | 60 +++++++++ internal/routing/provider.go | 33 +++++ 6 files changed, 367 insertions(+), 28 deletions(-) create mode 100644 internal/cluster/capabilities.go create mode 100644 internal/orchestrator/provider.go create mode 100644 internal/routing/provider.go diff --git a/internal/api/cluster_handlers.go b/internal/api/cluster_handlers.go index ae5e9d1..1ee95bf 100644 --- a/internal/api/cluster_handlers.go +++ b/internal/api/cluster_handlers.go @@ -1,6 +1,7 @@ package api import ( + "context" "crypto/rand" "database/sql" "encoding/base64" @@ -8,43 +9,139 @@ import ( "fmt" "io" "net/http" + "net/url" + "strings" "time" "github.com/flatrun/agent/internal/auth" "github.com/flatrun/agent/internal/cluster" + "github.com/flatrun/agent/pkg/config" "github.com/flatrun/agent/pkg/version" "github.com/gin-gonic/gin" ) func (s *Server) clusterStatus(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusOK, gin.H{ "enabled": false, }) return } - peers := s.clusterManager.ListPeers() + peers := mgr.ListPeers() c.JSON(http.StatusOK, gin.H{ "enabled": true, - "server_name": s.clusterManager.ServerName(), + "server_name": mgr.ServerName(), "peer_count": len(peers), "version": version.Get(), }) } +type clusterSetupRequest struct { + ServerName string `json:"server_name" binding:"required"` + AdvertiseURL string `json:"advertise_url" binding:"required"` +} + +func (s *Server) clusterSetup(c *gin.Context) { + var req clusterSetupRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + req.ServerName = strings.TrimSpace(req.ServerName) + req.AdvertiseURL = strings.TrimRight(strings.TrimSpace(req.AdvertiseURL), "/") + if req.ServerName == "" || strings.ContainsAny(req.ServerName, "/\\") { + c.JSON(http.StatusBadRequest, gin.H{"error": "Server name must not be empty or contain slashes"}) + return + } + parsedURL, err := url.ParseRequestURI(req.AdvertiseURL) + if err != nil || parsedURL.Host == "" || (parsedURL.Scheme != "http" && parsedURL.Scheme != "https") { + c.JSON(http.StatusBadRequest, gin.H{"error": "Advertise URL must be a valid HTTP or HTTPS URL"}) + return + } + + s.clusterMu.Lock() + defer s.clusterMu.Unlock() + if s.clusterManager != nil { + c.JSON(http.StatusConflict, gin.H{"error": "Cluster is already enabled"}) + return + } + + clusterDB, err := cluster.NewDB(s.config.DeploymentsPath) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to initialize cluster database: %v", err)}) + return + } + previous := s.config.Cluster + s.config.Cluster.Enabled = true + s.config.Cluster.ServerName = req.ServerName + s.config.Cluster.AdvertiseURL = req.AdvertiseURL + if s.config.Cluster.HealthInterval == "" { + s.config.Cluster.HealthInterval = "30s" + } + if s.config.Cluster.RequestTimeout == "" { + s.config.Cluster.RequestTimeout = "10s" + } + if s.configPath != "" { + if err := config.Save(s.config, s.configPath); err != nil { + s.config.Cluster = previous + _ = clusterDB.Close() + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to save cluster configuration: %v", err)}) + return + } + } + + healthInterval, _ := time.ParseDuration(s.config.Cluster.HealthInterval) + requestTimeout, _ := time.ParseDuration(s.config.Cluster.RequestTimeout) + if healthInterval <= 0 { + healthInterval = 30 * time.Second + } + if requestTimeout <= 0 { + requestTimeout = 10 * time.Second + } + mgr := cluster.NewManager(clusterDB, req.ServerName, healthInterval, requestTimeout, s.config.Auth.JWTSecret) + if err := mgr.Start(context.Background()); err != nil { + _ = clusterDB.Close() + s.config.Cluster = previous + if s.configPath != "" { + _ = config.Save(s.config, s.configPath) + } + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to start cluster: %v", err)}) + return + } + s.clusterManager = mgr + + c.JSON(http.StatusOK, gin.H{ + "enabled": true, + "server_name": req.ServerName, + "advertise_url": req.AdvertiseURL, + "peer_count": 0, + "version": version.Get(), + }) +} + +func (s *Server) getClusterManager() *cluster.Manager { + s.clusterMu.RLock() + defer s.clusterMu.RUnlock() + return s.clusterManager +} + func (s *Server) clusterListPeers(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } - peers := s.clusterManager.ListPeers() + peers := mgr.ListPeers() c.JSON(http.StatusOK, gin.H{"peers": peers}) } func (s *Server) clusterInvite(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } @@ -70,7 +167,7 @@ func (s *Server) clusterInvite(c *gin.Context) { ExpiresAt: time.Now().Add(1 * time.Hour), } - if _, err := s.clusterManager.DB().CreateInvite(invite); err != nil { + if _, err := mgr.DB().CreateInvite(invite); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create invite"}) return } @@ -82,7 +179,8 @@ func (s *Server) clusterInvite(c *gin.Context) { } func (s *Server) clusterAccept(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } @@ -119,7 +217,7 @@ func (s *Server) clusterAccept(c *gin.Context) { InviteToken: req.InviteToken, URL: callbackURL, APIKey: ourAPIKeyForThem, - Name: s.clusterManager.ServerName(), + Name: mgr.ServerName(), } body, err := json.Marshal(exchangeReq) @@ -145,7 +243,7 @@ func (s *Server) clusterAccept(c *gin.Context) { return } - if err := s.clusterManager.AddPeer(exchangeResp.Name, req.PeerURL, exchangeResp.APIKey); err != nil { + if err := mgr.AddPeer(exchangeResp.Name, req.PeerURL, exchangeResp.APIKey); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to store peer: %v", err)}) return } @@ -174,7 +272,8 @@ type exchangeResponse struct { } func (s *Server) clusterExchange(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } @@ -186,7 +285,7 @@ func (s *Server) clusterExchange(c *gin.Context) { } tokenHash := cluster.HashToken(req.InviteToken) - invite, err := s.clusterManager.DB().GetInviteByHash(tokenHash) + invite, err := mgr.DB().GetInviteByHash(tokenHash) if err != nil { if err == sql.ErrNoRows { c.JSON(http.StatusNotFound, gin.H{"error": "Invalid or expired invite token"}) @@ -206,7 +305,7 @@ func (s *Server) clusterExchange(c *gin.Context) { return } - if err := s.clusterManager.DB().ConsumeInvite(tokenHash, req.Name); err != nil { + if err := mgr.DB().ConsumeInvite(tokenHash, req.Name); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to consume invite"}) return } @@ -218,7 +317,7 @@ func (s *Server) clusterExchange(c *gin.Context) { } ourAPIKeyForThem := base64.URLEncoding.EncodeToString(apiKeyBytes) - if err := s.clusterManager.AddPeer(req.Name, req.URL, req.APIKey); err != nil { + if err := mgr.AddPeer(req.Name, req.URL, req.APIKey); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to store peer: %v", err)}) return } @@ -229,7 +328,7 @@ func (s *Server) clusterExchange(c *gin.Context) { c.JSON(http.StatusOK, exchangeResponse{ APIKey: ourAPIKeyForThem, - Name: s.clusterManager.ServerName(), + Name: mgr.ServerName(), }) } @@ -242,30 +341,57 @@ func (s *Server) createClusterAPIKey(rawKey, peerName string) { 1, fmt.Sprintf("cluster-peer-%s", peerName), fmt.Sprintf("Auto-generated API key for cluster peer %s", peerName), - auth.RoleAdmin, - nil, + auth.Role(""), + []string{ + auth.PermClusterRead.String(), + auth.PermDeploymentsRead.String(), + auth.PermDeploymentsWrite.String(), + auth.PermContainersRead.String(), + auth.PermContainersWrite.String(), + auth.PermSystemRead.String(), + auth.PermTrafficRead.String(), + }, nil, time.Time{}, ) } func (s *Server) clusterRemovePeer(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } name := c.Param("name") - if err := s.clusterManager.RemovePeer(name); err != nil { + if err := mgr.RemovePeer(name); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } + s.revokeClusterAPIKey(name) c.JSON(http.StatusOK, gin.H{"status": "removed", "peer": name}) } +func (s *Server) revokeClusterAPIKey(peerName string) { + if s.authManager == nil { + return + } + keys, err := s.authManager.GetAllAPIKeys() + if err != nil { + return + } + name := fmt.Sprintf("cluster-peer-%s", peerName) + for _, key := range keys { + if key.Name == name { + _ = s.authManager.DeactivateAPIKey(key.ID) + } + } +} + func (s *Server) clusterProxy(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } @@ -273,7 +399,7 @@ func (s *Server) clusterProxy(c *gin.Context) { name := c.Param("name") path := c.Param("path") - client, err := s.clusterManager.GetPeer(name) + client, err := mgr.GetPeer(name) if err != nil { c.JSON(http.StatusNotFound, gin.H{"error": err.Error()}) return @@ -299,7 +425,8 @@ func (s *Server) clusterProxy(c *gin.Context) { } func (s *Server) clusterAggregateDeployments(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } @@ -310,18 +437,19 @@ func (s *Server) clusterAggregateDeployments(c *gin.Context) { return } - localData, err := json.Marshal(deployments) + localData, err := json.Marshal(NewList(deployments, "deployments").Also("path", s.manager.BasePath())) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to marshal local deployments"}) return } - result := cluster.AggregateFromPeers(c.Request.Context(), localData, s.clusterManager, "/api/deployments") + result := cluster.AggregateFromPeers(c.Request.Context(), localData, mgr, "/api/deployments") c.JSON(http.StatusOK, result) } func (s *Server) clusterAggregateStats(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } @@ -337,6 +465,6 @@ func (s *Server) clusterAggregateStats(c *gin.Context) { return } - result := cluster.AggregateFromPeers(c.Request.Context(), localData, s.clusterManager, "/api/health") + result := cluster.AggregateFromPeers(c.Request.Context(), localData, mgr, "/api/health") c.JSON(http.StatusOK, result) } diff --git a/internal/api/cluster_handlers_test.go b/internal/api/cluster_handlers_test.go index 6194760..7be1bc9 100644 --- a/internal/api/cluster_handlers_test.go +++ b/internal/api/cluster_handlers_test.go @@ -77,6 +77,7 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool server := &Server{ config: cfg, + configPath: tmpDir + "/config.yml", authManager: authManager, clusterManager: clusterManager, } @@ -96,6 +97,7 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool clusterGroup.Use(authMiddleware.RequirePermission(auth.PermClusterRead)) { clusterGroup.GET("/status", server.clusterStatus) + clusterGroup.POST("/setup", authMiddleware.RequirePermission(auth.PermClusterWrite), server.clusterSetup) clusterGroup.GET("/peers", server.clusterListPeers) clusterGroup.POST("/invite", authMiddleware.RequirePermission(auth.PermClusterWrite), server.clusterInvite) clusterGroup.POST("/accept", authMiddleware.RequirePermission(auth.PermClusterWrite), server.clusterAccept) @@ -107,8 +109,8 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool } cleanup := func() { - if clusterManager != nil { - clusterManager.Stop() + if manager := server.getClusterManager(); manager != nil { + manager.Stop() } authManager.Close() os.RemoveAll(tmpDir) @@ -122,6 +124,99 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool } } +func TestClusterSetupEnablesClusterWithoutRestart(t *testing.T) { + env := setupClusterTestServer(t, "", false) + defer env.cleanup() + + token := clusterLogin(t, env.router) + body, _ := json.Marshal(map[string]string{ + "server_name": "server-a", + "advertise_url": "https://server-a.example.com/", + }) + req := httptest.NewRequest(http.MethodPost, "/api/cluster/setup", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("Expected 200, got %d: %s", w.Code, w.Body.String()) + } + + statusReq := httptest.NewRequest(http.MethodGet, "/api/cluster/status", nil) + statusReq.Header.Set("Authorization", "Bearer "+token) + statusWriter := httptest.NewRecorder() + env.router.ServeHTTP(statusWriter, statusReq) + if statusWriter.Code != http.StatusOK { + t.Fatalf("Expected status 200, got %d: %s", statusWriter.Code, statusWriter.Body.String()) + } + + var status map[string]interface{} + if err := json.Unmarshal(statusWriter.Body.Bytes(), &status); err != nil { + t.Fatal(err) + } + if status["enabled"] != true || status["server_name"] != "server-a" { + t.Fatalf("Unexpected status: %s", statusWriter.Body.String()) + } + if env.server.config.Cluster.AdvertiseURL != "https://server-a.example.com" { + t.Fatalf("AdvertiseURL = %q", env.server.config.Cluster.AdvertiseURL) + } +} + +func TestClusterSetupRejectsInvalidAdvertiseURL(t *testing.T) { + env := setupClusterTestServer(t, "", false) + defer env.cleanup() + + token := clusterLogin(t, env.router) + body, _ := json.Marshal(map[string]string{"server_name": "server-a", "advertise_url": "server-a"}) + req := httptest.NewRequest(http.MethodPost, "/api/cluster/setup", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("Expected 400, got %d: %s", w.Code, w.Body.String()) + } + if env.server.getClusterManager() != nil { + t.Fatal("Cluster manager started after invalid setup") + } +} + +func TestClusterAPIKeyUsesExplicitPermissions(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + + env.server.createClusterAPIKey("peer-key-for-test", "server-b") + keys, err := env.server.authManager.GetAllAPIKeys() + if err != nil { + t.Fatal(err) + } + var peerKey *auth.APIKey + for i := range keys { + if keys[i].Name == "cluster-peer-server-b" { + peerKey = &keys[i] + break + } + } + if peerKey == nil { + t.Fatal("Cluster API key was not created") + } + if peerKey.Role == auth.RoleAdmin { + t.Fatal("Cluster API key has administrator role") + } + permissions := make(map[string]bool, len(peerKey.Permissions)) + for _, permission := range peerKey.Permissions { + permissions[permission] = true + } + if !permissions[auth.PermDeploymentsRead.String()] || !permissions[auth.PermDeploymentsWrite.String()] { + t.Fatalf("permissions = %#v", peerKey.Permissions) + } + if permissions[auth.PermUsersWrite.String()] || permissions[auth.PermConfigWrite.String()] { + t.Fatalf("permissions include administrative access: %#v", peerKey.Permissions) + } +} + func clusterLogin(t *testing.T, router *gin.Engine) string { t.Helper() body, _ := json.Marshal(map[string]string{ diff --git a/internal/api/server.go b/internal/api/server.go index 554e4f7..6095d8d 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -98,6 +98,7 @@ type Server struct { auditManager *audit.Manager auditMiddleware *audit.Middleware powerDNSManager *dns.PowerDNSManager + clusterMu sync.RWMutex clusterManager *cluster.Manager setupManager *setup.Manager setupHandlers *setup.Handlers @@ -872,6 +873,7 @@ func (s *Server) setupRoutes() { clusterGroup.Use(s.authMiddleware.RequirePermission(auth.PermClusterRead)) { clusterGroup.GET("/status", s.clusterStatus) + clusterGroup.POST("/setup", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.clusterSetup) clusterGroup.GET("/peers", s.clusterListPeers) clusterGroup.POST("/invite", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.clusterInvite) clusterGroup.POST("/accept", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.clusterAccept) diff --git a/internal/cluster/capabilities.go b/internal/cluster/capabilities.go new file mode 100644 index 0000000..cf9bfa3 --- /dev/null +++ b/internal/cluster/capabilities.go @@ -0,0 +1,21 @@ +package cluster + +type Capability string + +const ( + CapabilityFleetRead Capability = "fleet.read" + CapabilityDeploymentsRead Capability = "deployments.read" + CapabilityDeploymentsRun Capability = "deployments.run" + CapabilityCapacityRead Capability = "capacity.read" + CapabilityCapacityOffer Capability = "capacity.offer" + CapabilityEventsPublish Capability = "events.publish" + CapabilityRoutingManage Capability = "routing.manage" +) + +type Grant struct { + Capability Capability `json:"capability"` + Deployments []string `json:"deployments,omitempty"` + MaxCPU float64 `json:"max_cpu,omitempty"` + MaxMemory uint64 `json:"max_memory,omitempty"` + MaxReplicas int `json:"max_replicas,omitempty"` +} diff --git a/internal/orchestrator/provider.go b/internal/orchestrator/provider.go new file mode 100644 index 0000000..a3ad228 --- /dev/null +++ b/internal/orchestrator/provider.go @@ -0,0 +1,60 @@ +package orchestrator + +import "context" + +type ProviderID string + +const ( + ProviderStandalone ProviderID = "standalone" + ProviderSwarm ProviderID = "swarm" + ProviderK3s ProviderID = "k3s" +) + +type Resources struct { + CPURequest float64 `json:"cpu_request,omitempty"` + CPULimit float64 `json:"cpu_limit,omitempty"` + MemoryRequest uint64 `json:"memory_request,omitempty"` + MemoryLimit uint64 `json:"memory_limit,omitempty"` +} + +type Health struct { + Path string `json:"path,omitempty"` + IntervalSeconds int `json:"interval_seconds,omitempty"` + TimeoutSeconds int `json:"timeout_seconds,omitempty"` + HealthyThreshold int `json:"healthy_threshold,omitempty"` +} + +type Workload struct { + ID string `json:"id"` + Image string `json:"image"` + Replicas int `json:"replicas"` + Resources Resources `json:"resources"` + Health Health `json:"health"` + Labels map[string]string `json:"labels,omitempty"` + Stateful bool `json:"stateful"` +} + +type Instance struct { + ID string `json:"id"` + Node string `json:"node"` + Address string `json:"address"` + Healthy bool `json:"healthy"` + Ready bool `json:"ready"` +} + +type Status struct { + Workload string `json:"workload"` + Desired int `json:"desired"` + Available int `json:"available"` + Instances []Instance `json:"instances"` +} + +type Provider interface { + ID() ProviderID + Validate(context.Context, Workload) error + Apply(context.Context, Workload) (Status, error) + Resize(context.Context, string, Resources) (Status, error) + Scale(context.Context, string, int) (Status, error) + Status(context.Context, string) (Status, error) + Remove(context.Context, string) error +} diff --git a/internal/routing/provider.go b/internal/routing/provider.go new file mode 100644 index 0000000..31d0c59 --- /dev/null +++ b/internal/routing/provider.go @@ -0,0 +1,33 @@ +package routing + +import "context" + +type ProviderID string + +const ( + ProviderNginx ProviderID = "nginx" + ProviderTraefik ProviderID = "traefik" +) + +type Backend struct { + ID string `json:"id"` + Address string `json:"address"` + Healthy bool `json:"healthy"` + Weight int `json:"weight,omitempty"` +} + +type Route struct { + ID string `json:"id"` + Domain string `json:"domain"` + Path string `json:"path,omitempty"` + Protocol string `json:"protocol"` + Backends []Backend `json:"backends"` +} + +type Provider interface { + ID() ProviderID + Validate(context.Context, Route) error + Reconcile(context.Context, Route) error + Drain(context.Context, string, string) error + Remove(context.Context, string) error +} From 2f24bb516f74c25aa041cae13b3b972f25d5e5b0 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 13:14:33 +0100 Subject: [PATCH 02/46] feat(capacity): Add resource pressure diagnosis --- internal/api/resource_handlers.go | 47 ++++++++++ internal/api/server.go | 1 + internal/capacity/capacity.go | 132 +++++++++++++++++++++++++++++ internal/capacity/capacity_test.go | 38 +++++++++ 4 files changed, 218 insertions(+) create mode 100644 internal/capacity/capacity.go create mode 100644 internal/capacity/capacity_test.go diff --git a/internal/api/resource_handlers.go b/internal/api/resource_handlers.go index 70d6f63..88e80ec 100644 --- a/internal/api/resource_handlers.go +++ b/internal/api/resource_handlers.go @@ -4,7 +4,9 @@ import ( "net/http" "github.com/flatrun/agent/internal/auth" + "github.com/flatrun/agent/internal/capacity" "github.com/flatrun/agent/internal/docker" + "github.com/flatrun/agent/internal/system" "github.com/gin-gonic/gin" ) @@ -27,6 +29,51 @@ func (s *Server) getContainerResources(c *gin.Context) { }) } +func (s *Server) diagnoseContainerCapacity(c *gin.Context) { + id := c.Param("id") + if !s.requireContainerAccess(c, id, auth.AccessLevelRead) { + return + } + + stats, err := docker.GetContainerStats(id) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) + return + } + limits, err := docker.GetContainerResources(id) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) + return + } + hostStats, err := system.GetSystemStats() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + host := capacity.Host{ + CPUCores: float64(hostStats.CPU.Cores), + CPUUsagePercent: hostStats.CPU.UsagePercent, + MemoryTotal: hostStats.Memory.Total, + MemoryAvailable: hostStats.Memory.Available, + } + container := capacity.Container{ + ID: stats.ContainerID, + Name: stats.Name, + CPUPercent: stats.CPUPercent, + CPULimit: limits.CPUs, + MemoryUsage: stats.MemoryUsage, + MemoryLimit: uint64(max(limits.MemoryLimit, 0)), + } + policy := capacity.DefaultPolicy() + c.JSON(http.StatusOK, gin.H{ + "host": host, + "container": container, + "policy": policy, + "diagnosis": capacity.Diagnose(host, container, policy), + }) +} + func (s *Server) updateContainerResources(c *gin.Context) { id := c.Param("id") diff --git a/internal/api/server.go b/internal/api/server.go index 6095d8d..a732dd7 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -606,6 +606,7 @@ func (s *Server) setupRoutes() { protected.GET("/containers/stats", s.authMiddleware.RequirePermission(auth.PermContainersRead), s.getAllContainerStats) protected.POST("/containers/:id/exec", s.authMiddleware.RequirePermission(auth.PermContainersWrite), s.containerExecHTTP) protected.GET("/containers/:id/resources", s.authMiddleware.RequirePermission(auth.PermContainersRead), s.getContainerResources) + protected.GET("/containers/:id/capacity", s.authMiddleware.RequirePermission(auth.PermContainersRead), s.diagnoseContainerCapacity) protected.PUT("/containers/:id/resources", s.authMiddleware.RequirePermission(auth.PermContainersWrite), s.updateContainerResources) protected.GET("/deployments/:name/stats", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentContainerStats) protected.GET("/deployments/:name/resources", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentResources) diff --git a/internal/capacity/capacity.go b/internal/capacity/capacity.go new file mode 100644 index 0000000..94781d6 --- /dev/null +++ b/internal/capacity/capacity.go @@ -0,0 +1,132 @@ +package capacity + +import "math" + +type Pressure string + +const ( + PressureNone Pressure = "none" + PressureAllocation Pressure = "allocation" + PressureHost Pressure = "host" +) + +type Action string + +const ( + ActionNone Action = "none" + ActionIncreaseMemory Action = "increase_memory" + ActionIncreaseCPU Action = "increase_cpu" + ActionAddReplica Action = "add_replica" + ActionNotify Action = "notify" +) + +type Host struct { + CPUCores float64 `json:"cpu_cores"` + CPUUsagePercent float64 `json:"cpu_usage_percent"` + MemoryTotal uint64 `json:"memory_total"` + MemoryAvailable uint64 `json:"memory_available"` +} + +type Container struct { + ID string `json:"id"` + Name string `json:"name"` + CPUPercent float64 `json:"cpu_percent"` + CPULimit float64 `json:"cpu_limit"` + MemoryUsage uint64 `json:"memory_usage"` + MemoryLimit uint64 `json:"memory_limit"` +} + +type Policy struct { + AllocationThresholdPercent float64 `json:"allocation_threshold_percent"` + HostThresholdPercent float64 `json:"host_threshold_percent"` + HostMemoryReserve uint64 `json:"host_memory_reserve"` + HostCPUReserve float64 `json:"host_cpu_reserve"` + MemoryStepPercent float64 `json:"memory_step_percent"` + CPUStepPercent float64 `json:"cpu_step_percent"` + MaxMemory uint64 `json:"max_memory"` + MaxCPU float64 `json:"max_cpu"` + AllowVertical bool `json:"allow_vertical"` + AllowHorizontal bool `json:"allow_horizontal"` +} + +type Diagnosis struct { + Pressure Pressure `json:"pressure"` + Resource string `json:"resource,omitempty"` + Action Action `json:"action"` + CurrentLimit float64 `json:"current_limit,omitempty"` + RecommendedLimit float64 `json:"recommended_limit,omitempty"` + Reason string `json:"reason"` +} + +func DefaultPolicy() Policy { + return Policy{ + AllocationThresholdPercent: 90, + HostThresholdPercent: 85, + HostMemoryReserve: 512 * 1024 * 1024, + HostCPUReserve: 0.25, + MemoryStepPercent: 50, + CPUStepPercent: 50, + AllowVertical: true, + AllowHorizontal: true, + } +} + +func Diagnose(host Host, container Container, policy Policy) Diagnosis { + if policy.AllocationThresholdPercent <= 0 { + policy = DefaultPolicy() + } + + if container.MemoryLimit > 0 { + utilization := float64(container.MemoryUsage) / float64(container.MemoryLimit) * 100 + if utilization >= policy.AllocationThresholdPercent { + next := grow(float64(container.MemoryLimit), policy.MemoryStepPercent, float64(policy.MaxMemory)) + required := uint64(math.Max(0, next-float64(container.MemoryLimit))) + if policy.AllowVertical && next > float64(container.MemoryLimit) && host.MemoryAvailable >= policy.HostMemoryReserve+required { + return Diagnosis{Pressure: PressureAllocation, Resource: "memory", Action: ActionIncreaseMemory, CurrentLimit: float64(container.MemoryLimit), RecommendedLimit: next, Reason: "Container memory is constrained while the host has reserved headroom"} + } + return exhaustedAction("memory", policy) + } + } + + if container.CPULimit > 0 { + utilization := container.CPUPercent / (container.CPULimit * 100) * 100 + if utilization >= policy.AllocationThresholdPercent { + next := grow(container.CPULimit, policy.CPUStepPercent, policy.MaxCPU) + available := host.CPUCores * math.Max(0, 100-host.CPUUsagePercent) / 100 + if policy.AllowVertical && next > container.CPULimit && available >= policy.HostCPUReserve+(next-container.CPULimit) { + return Diagnosis{Pressure: PressureAllocation, Resource: "cpu", Action: ActionIncreaseCPU, CurrentLimit: container.CPULimit, RecommendedLimit: next, Reason: "Container CPU is constrained while the host has reserved headroom"} + } + return exhaustedAction("cpu", policy) + } + } + + memoryPressure := host.MemoryTotal > 0 && float64(host.MemoryTotal-host.MemoryAvailable)/float64(host.MemoryTotal)*100 >= policy.HostThresholdPercent + if memoryPressure || host.CPUUsagePercent >= policy.HostThresholdPercent { + resource := "cpu" + if memoryPressure { + resource = "memory" + } + return Diagnosis{Pressure: PressureHost, Resource: resource, Action: horizontalOrNotify(policy), Reason: "Host capacity is below its configured safety reserve"} + } + + return Diagnosis{Pressure: PressureNone, Action: ActionNone, Reason: "Container allocation and host capacity are within policy"} +} + +func grow(current, percent, maximum float64) float64 { + next := current * (1 + percent/100) + if maximum > 0 && next > maximum { + return maximum + } + return next +} + +func exhaustedAction(resource string, policy Policy) Diagnosis { + return Diagnosis{Pressure: PressureHost, Resource: resource, Action: horizontalOrNotify(policy), Reason: "Container allocation is constrained and the host cannot safely increase it"} +} + +func horizontalOrNotify(policy Policy) Action { + if policy.AllowHorizontal { + return ActionAddReplica + } + return ActionNotify +} diff --git a/internal/capacity/capacity_test.go b/internal/capacity/capacity_test.go new file mode 100644 index 0000000..e94384a --- /dev/null +++ b/internal/capacity/capacity_test.go @@ -0,0 +1,38 @@ +package capacity + +import "testing" + +func TestDiagnoseIncreasesContainerMemoryBeforeScaling(t *testing.T) { + policy := DefaultPolicy() + host := Host{MemoryTotal: 16 << 30, MemoryAvailable: 8 << 30, CPUCores: 8, CPUUsagePercent: 20} + container := Container{MemoryUsage: 950 << 20, MemoryLimit: 1 << 30} + + diagnosis := Diagnose(host, container, policy) + if diagnosis.Pressure != PressureAllocation || diagnosis.Action != ActionIncreaseMemory { + t.Fatalf("diagnosis = %#v", diagnosis) + } + if diagnosis.RecommendedLimit != float64(1536<<20) { + t.Fatalf("recommended memory = %.0f", diagnosis.RecommendedLimit) + } +} + +func TestDiagnoseAddsReplicaWhenHostCannotResize(t *testing.T) { + policy := DefaultPolicy() + host := Host{MemoryTotal: 4 << 30, MemoryAvailable: 600 << 20, CPUCores: 2, CPUUsagePercent: 80} + container := Container{MemoryUsage: 950 << 20, MemoryLimit: 1 << 30} + + diagnosis := Diagnose(host, container, policy) + if diagnosis.Pressure != PressureHost || diagnosis.Action != ActionAddReplica { + t.Fatalf("diagnosis = %#v", diagnosis) + } +} + +func TestDiagnoseReportsUnlimitedContainerAsHealthyWhenHostHasHeadroom(t *testing.T) { + policy := DefaultPolicy() + host := Host{MemoryTotal: 16 << 30, MemoryAvailable: 10 << 30, CPUCores: 8, CPUUsagePercent: 10} + + diagnosis := Diagnose(host, Container{MemoryUsage: 6 << 30, CPUPercent: 200}, policy) + if diagnosis.Pressure != PressureNone || diagnosis.Action != ActionNone { + t.Fatalf("diagnosis = %#v", diagnosis) + } +} From a84d202b5a8879b3d323d7b741a33201d0119d68 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 13:15:07 +0100 Subject: [PATCH 03/46] feat(notifications): Correlate infrastructure incidents --- internal/api/events_handlers.go | 38 +++++++ internal/api/notifications_test.go | 29 ++++++ internal/api/server.go | 2 + internal/events/events.go | 156 +++++++++++++++++++++++++++++ internal/events/events_test.go | 45 +++++++++ internal/notify/events_test.go | 62 ++++++++++++ internal/notify/notify.go | 105 +++++++++++++++++-- 7 files changed, 429 insertions(+), 8 deletions(-) create mode 100644 internal/api/events_handlers.go create mode 100644 internal/events/events.go create mode 100644 internal/events/events_test.go create mode 100644 internal/notify/events_test.go diff --git a/internal/api/events_handlers.go b/internal/api/events_handlers.go new file mode 100644 index 0000000..a7dcdff --- /dev/null +++ b/internal/api/events_handlers.go @@ -0,0 +1,38 @@ +package api + +import ( + "net/http" + "time" + + "github.com/flatrun/agent/internal/events" + "github.com/gin-gonic/gin" +) + +func (s *Server) listNotificationIncidents(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{"incidents": s.notify.Incidents()}) +} + +func (s *Server) emitEvent(c *gin.Context) { + if s.pluginToken == "" || c.GetHeader("X-Plugin-Token") != s.pluginToken { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"}) + return + } + var event events.Event + if err := c.ShouldBindJSON(&event); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request"}) + return + } + if event.Source == "" || event.Type == "" || event.Title == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "source, type, and title are required"}) + return + } + if event.OccurredAt.IsZero() { + event.OccurredAt = time.Now().UTC() + } + result, err := s.notify.Publish(event) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error(), "result": result}) + return + } + c.JSON(http.StatusAccepted, result) +} diff --git a/internal/api/notifications_test.go b/internal/api/notifications_test.go index 60b927c..66e9b12 100644 --- a/internal/api/notifications_test.go +++ b/internal/api/notifications_test.go @@ -23,11 +23,40 @@ func setupNotifyTest(t *testing.T) (*Server, *gin.Engine) { } r := gin.New() r.GET("/notifications/targets", s.getNotificationTargets) + r.GET("/notifications/incidents", s.listNotificationIncidents) r.PUT("/notifications/targets", s.updateNotificationTargets) r.POST("/internal/notify/emit", s.emitNotification) + r.POST("/internal/events", s.emitEvent) return s, r } +func TestEmitEventCorrelatesIncidentThroughHTTP(t *testing.T) { + _, r := setupNotifyTest(t) + emit := func(payload string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/internal/events", bytes.NewBufferString(payload)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Plugin-Token", "plugin-token") + r.ServeHTTP(w, req) + return w + } + + first := emit(`{"source":"fleet","type":"node.unavailable","severity":"critical","title":"prod2 unavailable","scope":{"node":"prod2"}}`) + if first.Code != http.StatusAccepted { + t.Fatalf("status = %d, body = %s", first.Code, first.Body.String()) + } + second := emit(`{"source":"capacity","type":"deployment.unavailable","severity":"critical","title":"app unavailable","scope":{"node":"prod2","deployment":"app"},"correlation_key":"node:prod2"}`) + if second.Code != http.StatusAccepted { + t.Fatalf("status = %d, body = %s", second.Code, second.Body.String()) + } + + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/notifications/incidents", nil)) + if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"event_count":2`) { + t.Fatalf("incidents status = %d, body = %s", w.Code, w.Body.String()) + } +} + func TestGetNotificationTargetsMasksSecret(t *testing.T) { s, r := setupNotifyTest(t) if err := s.notify.Save(notify.Config{Targets: []notify.Target{ diff --git a/internal/api/server.go b/internal/api/server.go index a732dd7..a72027f 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -524,6 +524,7 @@ func (s *Server) setupRoutes() { protected.GET("/agent/update", s.authMiddleware.RequirePermission(auth.PermSettingsRead), s.getAgentUpdate) protected.POST("/agent/update", s.authMiddleware.RequirePermission(auth.PermSettingsWrite), s.triggerAgentUpdate) protected.GET("/notifications/targets", s.authMiddleware.RequirePermission(auth.PermSettingsRead), s.getNotificationTargets) + protected.GET("/notifications/incidents", s.authMiddleware.RequirePermission(auth.PermSettingsRead), s.listNotificationIncidents) protected.PUT("/notifications/targets", s.authMiddleware.RequirePermission(auth.PermSettingsWrite), s.updateNotificationTargets) protected.POST("/notifications/test", s.authMiddleware.RequirePermission(auth.PermSettingsWrite), s.testNotification) protected.GET("/config", s.authMiddleware.RequirePermission(auth.PermConfigRead), s.listConfig) @@ -890,6 +891,7 @@ func (s *Server) setupRoutes() { // Plugin-emitted notifications (authenticated by the per-run plugin token). api.POST("/internal/notify/emit", s.emitNotification) + api.POST("/internal/events", s.emitEvent) // Log lines and triage for built-in apps, on the same plugin token. Both keep one // implementation in the agent rather than a second one inside every app. api.GET("/internal/logs/stream", s.streamInternalLogs) diff --git a/internal/events/events.go b/internal/events/events.go new file mode 100644 index 0000000..d5f1cf0 --- /dev/null +++ b/internal/events/events.go @@ -0,0 +1,156 @@ +package events + +import ( + "fmt" + "sync" + "time" +) + +type Severity string + +const ( + SeverityInfo Severity = "info" + SeverityWarning Severity = "warning" + SeverityCritical Severity = "critical" +) + +type Scope struct { + Node string `json:"node,omitempty"` + Deployment string `json:"deployment,omitempty"` + Container string `json:"container,omitempty"` +} + +type Event struct { + ID string `json:"id"` + Source string `json:"source"` + Type string `json:"type"` + Severity Severity `json:"severity"` + Title string `json:"title"` + Message string `json:"message"` + Scope Scope `json:"scope"` + CorrelationKey string `json:"correlation_key,omitempty"` + OccurredAt time.Time `json:"occurred_at"` + Attributes map[string]any `json:"attributes,omitempty"` + Resolved bool `json:"resolved,omitempty"` +} + +type NotificationAction string + +const ( + NotificationNone NotificationAction = "none" + NotificationOpened NotificationAction = "opened" + NotificationUpdated NotificationAction = "updated" + NotificationResolved NotificationAction = "resolved" +) + +type IngestResult struct { + Incident Incident `json:"incident"` + Notification NotificationAction `json:"notification"` +} + +type Correlator struct { + mu sync.Mutex + incidents map[string]Incident + lastNotifiedAt map[string]time.Time + updateInterval time.Duration +} + +func NewCorrelator(updateInterval time.Duration) *Correlator { + return &Correlator{ + incidents: make(map[string]Incident), + lastNotifiedAt: make(map[string]time.Time), + updateInterval: updateInterval, + } +} + +func (c *Correlator) Ingest(event Event) IngestResult { + c.mu.Lock() + defer c.mu.Unlock() + + key := CorrelationKey(event) + incident, exists := c.incidents[key] + action := NotificationNone + if !exists || incident.Status == IncidentResolved { + incident = Incident{ + ID: fmt.Sprintf("%s:%d", key, event.OccurredAt.UnixNano()), + CorrelationKey: key, + Status: IncidentOpen, + Severity: event.Severity, + Title: event.Title, + FirstEventAt: event.OccurredAt, + } + action = NotificationOpened + } + + incident.EventCount++ + incident.LastEventAt = event.OccurredAt + incident.LastEvent = event + if severityRank(event.Severity) > severityRank(incident.Severity) { + incident.Severity = event.Severity + } + if event.Resolved { + incident.Status = IncidentResolved + action = NotificationResolved + } else if action == NotificationNone && c.updateInterval > 0 && event.OccurredAt.Sub(c.lastNotifiedAt[key]) >= c.updateInterval { + action = NotificationUpdated + } + + c.incidents[key] = incident + if action != NotificationNone { + c.lastNotifiedAt[key] = event.OccurredAt + } + return IngestResult{Incident: incident, Notification: action} +} + +func (c *Correlator) List() []Incident { + c.mu.Lock() + defer c.mu.Unlock() + result := make([]Incident, 0, len(c.incidents)) + for _, incident := range c.incidents { + result = append(result, incident) + } + return result +} + +func severityRank(severity Severity) int { + switch severity { + case SeverityCritical: + return 3 + case SeverityWarning: + return 2 + default: + return 1 + } +} + +type IncidentStatus string + +const ( + IncidentOpen IncidentStatus = "open" + IncidentResolved IncidentStatus = "resolved" +) + +type Incident struct { + ID string `json:"id"` + CorrelationKey string `json:"correlation_key"` + Status IncidentStatus `json:"status"` + Severity Severity `json:"severity"` + Title string `json:"title"` + EventCount int `json:"event_count"` + FirstEventAt time.Time `json:"first_event_at"` + LastEventAt time.Time `json:"last_event_at"` + LastEvent Event `json:"last_event"` +} + +func CorrelationKey(event Event) string { + if event.CorrelationKey != "" { + return event.CorrelationKey + } + if event.Scope.Node != "" { + return "node:" + event.Scope.Node + } + if event.Scope.Deployment != "" { + return "deployment:" + event.Scope.Deployment + } + return event.Source + ":" + event.Type +} diff --git a/internal/events/events_test.go b/internal/events/events_test.go new file mode 100644 index 0000000..4edecf0 --- /dev/null +++ b/internal/events/events_test.go @@ -0,0 +1,45 @@ +package events + +import ( + "testing" + "time" +) + +func TestCorrelatorSuppressesDeploymentFloodDuringNodeIncident(t *testing.T) { + correlator := NewCorrelator(15 * time.Minute) + started := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) + opened := correlator.Ingest(Event{ + Source: "fleet", Type: "node.unavailable", Severity: SeverityCritical, + Title: "prod2 unavailable", Scope: Scope{Node: "prod2"}, OccurredAt: started, + }) + if opened.Notification != NotificationOpened { + t.Fatalf("opened notification = %q", opened.Notification) + } + + for i := 0; i < 14; i++ { + result := correlator.Ingest(Event{ + Source: "capacity", Type: "deployment.unavailable", Severity: SeverityCritical, + Title: "Deployment unavailable", Scope: Scope{Node: "prod2", Deployment: "app"}, + CorrelationKey: "node:prod2", OccurredAt: started.Add(time.Duration(i+1) * 10 * time.Second), + }) + if result.Notification != NotificationNone { + t.Fatalf("event %d notification = %q", i, result.Notification) + } + } + + incidents := correlator.List() + if len(incidents) != 1 || incidents[0].EventCount != 15 { + t.Fatalf("incidents = %#v", incidents) + } +} + +func TestCorrelatorSendsOneRecoveryNotification(t *testing.T) { + correlator := NewCorrelator(15 * time.Minute) + started := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) + correlator.Ingest(Event{Source: "fleet", Type: "node.unavailable", Severity: SeverityCritical, Title: "prod2 unavailable", Scope: Scope{Node: "prod2"}, OccurredAt: started}) + + resolved := correlator.Ingest(Event{Source: "fleet", Type: "node.available", Severity: SeverityInfo, Title: "prod2 recovered", Scope: Scope{Node: "prod2"}, OccurredAt: started.Add(10 * time.Minute), Resolved: true}) + if resolved.Notification != NotificationResolved || resolved.Incident.Status != IncidentResolved { + t.Fatalf("resolved = %#v", resolved) + } +} diff --git a/internal/notify/events_test.go b/internal/notify/events_test.go new file mode 100644 index 0000000..6361dfb --- /dev/null +++ b/internal/notify/events_test.go @@ -0,0 +1,62 @@ +package notify + +import ( + "strings" + "testing" + "time" + + "github.com/flatrun/agent/internal/events" +) + +func TestPublishSendsOneNotificationForCorrelatedFailure(t *testing.T) { + service := NewService(t.TempDir()) + if err := service.Save(Config{Targets: []Target{{ID: "email", Name: "Email", URL: "smtp://test", Enabled: true}}}); err != nil { + t.Fatal(err) + } + var deliveries []string + service.send = func(_, message string) error { + deliveries = append(deliveries, message) + return nil + } + started := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) + + _, err := service.Publish(events.Event{Source: "fleet", Type: "node.unavailable", Severity: events.SeverityCritical, Title: "prod2 unavailable", Message: "The node stopped responding.", Scope: events.Scope{Node: "prod2"}, OccurredAt: started}) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 20; i++ { + _, err = service.Publish(events.Event{Source: "capacity", Type: "deployment.unavailable", Severity: events.SeverityCritical, Title: "Deployment unavailable", Message: "A dependent deployment is unavailable.", Scope: events.Scope{Node: "prod2", Deployment: "app"}, CorrelationKey: "node:prod2", OccurredAt: started.Add(time.Duration(i+1) * 10 * time.Second)}) + if err != nil { + t.Fatal(err) + } + } + if len(deliveries) != 1 { + t.Fatalf("deliveries = %d", len(deliveries)) + } + if !strings.Contains(deliveries[0], "prod2 unavailable") { + t.Fatalf("delivery = %q", deliveries[0]) + } +} + +func TestPublishFiltersTargetsByTopicAndNode(t *testing.T) { + service := NewService(t.TempDir()) + if err := service.Save(Config{Targets: []Target{ + {ID: "prod", URL: "generic+https://prod.example.test", Enabled: true, Topics: []string{"fleet"}, Nodes: []string{"prod2"}}, + {ID: "dev", URL: "generic+https://dev.example.test", Enabled: true, Topics: []string{"fleet"}, Nodes: []string{"dev1"}}, + }}); err != nil { + t.Fatal(err) + } + var targets []string + service.send = func(target, _ string) error { + targets = append(targets, target) + return nil + } + + _, err := service.Publish(events.Event{Source: "fleet", Type: "node.unavailable", Severity: events.SeverityCritical, Title: "prod2 unavailable", Scope: events.Scope{Node: "prod2"}, OccurredAt: time.Now()}) + if err != nil { + t.Fatal(err) + } + if len(targets) != 1 || !strings.Contains(targets[0], "prod.example.test") { + t.Fatalf("targets = %#v", targets) + } +} diff --git a/internal/notify/notify.go b/internal/notify/notify.go index d1d5c00..c5f360a 100644 --- a/internal/notify/notify.go +++ b/internal/notify/notify.go @@ -12,7 +12,9 @@ import ( "path/filepath" "strings" "sync" + "time" + "github.com/flatrun/agent/internal/events" "github.com/nicholas-fedor/shoutrrr" "github.com/nicholas-fedor/shoutrrr/pkg/router" "gopkg.in/yaml.v3" @@ -26,10 +28,14 @@ const MaskedURL = "********" // Target is one delivery destination. URL is a shoutrrr service URL, e.g. // "smtp://user:pass@host:587/?from=x&to=y" or "generic+https://example.com/hook". type Target struct { - ID string `yaml:"id" json:"id"` - Name string `yaml:"name" json:"name"` - URL string `yaml:"url" json:"url"` - Enabled bool `yaml:"enabled" json:"enabled"` + ID string `yaml:"id" json:"id"` + Name string `yaml:"name" json:"name"` + URL string `yaml:"url" json:"url"` + Enabled bool `yaml:"enabled" json:"enabled"` + Topics []string `yaml:"topics,omitempty" json:"topics,omitempty"` + Severities []events.Severity `yaml:"severities,omitempty" json:"severities,omitempty"` + Nodes []string `yaml:"nodes,omitempty" json:"nodes,omitempty"` + Deployments []string `yaml:"deployments,omitempty" json:"deployments,omitempty"` } // MarshalJSON masks the credential-bearing URL. YAML persistence does not use @@ -50,17 +56,100 @@ type Config struct { // Service loads/saves targets and delivers messages. type Service struct { - path string - mu sync.RWMutex - send func(url, message string) error // overridable in tests + path string + mu sync.RWMutex + send func(url, message string) error // overridable in tests + events *events.Correlator } func NewService(basePath string) *Service { return &Service{ - path: filepath.Join(basePath, ".flatrun", "notifications.yml"), + path: filepath.Join(basePath, ".flatrun", "notifications.yml"), + events: events.NewCorrelator(15 * time.Minute), } } +func (s *Service) Publish(event events.Event) (events.IngestResult, error) { + result := s.events.Ingest(event) + if result.Notification == events.NotificationNone { + return result, nil + } + + kind := KindNegative + message := event.Message + switch result.Notification { + case events.NotificationResolved: + kind = KindPositive + message = fmt.Sprintf("Resolved after %d related events. %s", result.Incident.EventCount, event.Message) + case events.NotificationUpdated: + message = fmt.Sprintf("%d related events are grouped in this incident. %s", result.Incident.EventCount, event.Message) + } + if event.Severity == events.SeverityInfo && result.Notification != events.NotificationResolved { + kind = KindGeneric + } + + notification := Notification{ + Kind: kind, + Title: event.Title, + Message: message, + Panels: []Panel{{ + Title: "Incident ID", + Value: result.Incident.ID, + Detail: fmt.Sprintf("Source: %s", event.Source), + }}, + } + return result, s.deliverEvent(event, notification) +} + +func (s *Service) Incidents() []events.Incident { + return s.events.List() +} + +func (s *Service) deliverEvent(event events.Event, notification Notification) error { + cfg := s.Load() + var firstErr error + for _, target := range cfg.Targets { + if !target.Enabled || target.URL == "" || !targetMatches(target, event) { + continue + } + if err := s.deliver(target.URL, notification); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} + +func targetMatches(target Target, event events.Event) bool { + return matchesString(target.Topics, event.Source) && + matchesSeverity(target.Severities, event.Severity) && + matchesString(target.Nodes, event.Scope.Node) && + matchesString(target.Deployments, event.Scope.Deployment) +} + +func matchesString(filter []string, value string) bool { + if len(filter) == 0 { + return true + } + for _, candidate := range filter { + if candidate == value { + return true + } + } + return false +} + +func matchesSeverity(filter []events.Severity, value events.Severity) bool { + if len(filter) == 0 { + return true + } + for _, candidate := range filter { + if candidate == value { + return true + } + } + return false +} + func (s *Service) Load() Config { s.mu.RLock() defer s.mu.RUnlock() From 85d7807a1afd742db5735b3e264d22458e96e2d8 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 13:22:10 +0100 Subject: [PATCH 04/46] feat(notifications): Persist correlated incidents --- internal/events/events.go | 9 +++ internal/events/store.go | 100 +++++++++++++++++++++++++++++++++ internal/notify/events_test.go | 26 +++++++++ internal/notify/notify.go | 41 ++++++++++++-- 4 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 internal/events/store.go diff --git a/internal/events/events.go b/internal/events/events.go index d5f1cf0..c908224 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -112,6 +112,15 @@ func (c *Correlator) List() []Incident { return result } +func (c *Correlator) Restore(incidents []Incident) { + c.mu.Lock() + defer c.mu.Unlock() + for _, incident := range incidents { + c.incidents[incident.CorrelationKey] = incident + c.lastNotifiedAt[incident.CorrelationKey] = incident.LastEventAt + } +} + func severityRank(severity Severity) int { switch severity { case SeverityCritical: diff --git a/internal/events/store.go b/internal/events/store.go new file mode 100644 index 0000000..4b314bf --- /dev/null +++ b/internal/events/store.go @@ -0,0 +1,100 @@ +package events + +import ( + "database/sql" + "encoding/json" + "fmt" + "os" + "path/filepath" + + _ "modernc.org/sqlite" +) + +type Store struct { + db *sql.DB +} + +func NewStore(basePath string) (*Store, error) { + dir := filepath.Join(basePath, ".flatrun") + if err := os.MkdirAll(dir, 0700); err != nil { + return nil, err + } + db, err := sql.Open("sqlite", "file:"+filepath.Join(dir, "events.db")+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + if err != nil { + return nil, err + } + store := &Store{db: db} + if err := store.migrate(); err != nil { + _ = db.Close() + return nil, err + } + return store, nil +} + +func (s *Store) migrate() error { + _, err := s.db.Exec(` + CREATE TABLE IF NOT EXISTS incidents ( + correlation_key TEXT PRIMARY KEY, + payload BLOB NOT NULL, + last_event_at DATETIME NOT NULL + ); + CREATE TABLE IF NOT EXISTS events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + incident_id TEXT NOT NULL, + payload BLOB NOT NULL, + occurred_at DATETIME NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_events_incident_id ON events(incident_id); + CREATE INDEX IF NOT EXISTS idx_events_occurred_at ON events(occurred_at); + `) + return err +} + +func (s *Store) Record(event Event, incident Incident) error { + eventPayload, err := json.Marshal(event) + if err != nil { + return err + } + incidentPayload, err := json.Marshal(incident) + if err != nil { + return err + } + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.Exec(`INSERT INTO events (incident_id, payload, occurred_at) VALUES (?, ?, ?)`, incident.ID, eventPayload, event.OccurredAt); err != nil { + return err + } + if _, err := tx.Exec(`INSERT INTO incidents (correlation_key, payload, last_event_at) VALUES (?, ?, ?) + ON CONFLICT(correlation_key) DO UPDATE SET payload = excluded.payload, last_event_at = excluded.last_event_at`, incident.CorrelationKey, incidentPayload, incident.LastEventAt); err != nil { + return err + } + return tx.Commit() +} + +func (s *Store) ListIncidents() ([]Incident, error) { + rows, err := s.db.Query(`SELECT payload FROM incidents ORDER BY last_event_at DESC`) + if err != nil { + return nil, err + } + defer rows.Close() + var incidents []Incident + for rows.Next() { + var payload []byte + if err := rows.Scan(&payload); err != nil { + return nil, err + } + var incident Incident + if err := json.Unmarshal(payload, &incident); err != nil { + return nil, fmt.Errorf("decode incident: %w", err) + } + incidents = append(incidents, incident) + } + return incidents, rows.Err() +} + +func (s *Store) Close() error { + return s.db.Close() +} diff --git a/internal/notify/events_test.go b/internal/notify/events_test.go index 6361dfb..b99905c 100644 --- a/internal/notify/events_test.go +++ b/internal/notify/events_test.go @@ -60,3 +60,29 @@ func TestPublishFiltersTargetsByTopicAndNode(t *testing.T) { t.Fatalf("targets = %#v", targets) } } + +func TestIncidentsSurviveServiceRestart(t *testing.T) { + basePath := t.TempDir() + service := NewService(basePath) + started := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) + if _, err := service.Publish(events.Event{Source: "fleet", Type: "node.unavailable", Severity: events.SeverityCritical, Title: "prod2 unavailable", Scope: events.Scope{Node: "prod2"}, OccurredAt: started}); err != nil { + t.Fatal(err) + } + if err := service.Close(); err != nil { + t.Fatal(err) + } + + restarted := NewService(basePath) + defer restarted.Close() + incidents := restarted.Incidents() + if len(incidents) != 1 || incidents[0].CorrelationKey != "node:prod2" { + t.Fatalf("incidents = %#v", incidents) + } + result, err := restarted.Publish(events.Event{Source: "capacity", Type: "deployment.unavailable", Severity: events.SeverityCritical, Title: "app unavailable", Scope: events.Scope{Node: "prod2", Deployment: "app"}, CorrelationKey: "node:prod2", OccurredAt: started.Add(time.Minute)}) + if err != nil { + t.Fatal(err) + } + if result.Incident.EventCount != 2 || result.Notification != events.NotificationNone { + t.Fatalf("result = %#v", result) + } +} diff --git a/internal/notify/notify.go b/internal/notify/notify.go index c5f360a..c22ad0e 100644 --- a/internal/notify/notify.go +++ b/internal/notify/notify.go @@ -56,21 +56,39 @@ type Config struct { // Service loads/saves targets and delivers messages. type Service struct { - path string - mu sync.RWMutex - send func(url, message string) error // overridable in tests - events *events.Correlator + path string + mu sync.RWMutex + send func(url, message string) error // overridable in tests + events *events.Correlator + store *events.Store + storeErr error } func NewService(basePath string) *Service { - return &Service{ + service := &Service{ path: filepath.Join(basePath, ".flatrun", "notifications.yml"), events: events.NewCorrelator(15 * time.Minute), } + service.store, service.storeErr = events.NewStore(basePath) + if service.storeErr == nil { + incidents, err := service.store.ListIncidents() + if err != nil { + service.storeErr = err + } else { + service.events.Restore(incidents) + } + } + return service } func (s *Service) Publish(event events.Event) (events.IngestResult, error) { + if s.storeErr != nil { + return events.IngestResult{}, s.storeErr + } result := s.events.Ingest(event) + if err := s.store.Record(event, result.Incident); err != nil { + return result, err + } if result.Notification == events.NotificationNone { return result, nil } @@ -102,7 +120,18 @@ func (s *Service) Publish(event events.Event) (events.IngestResult, error) { } func (s *Service) Incidents() []events.Incident { - return s.events.List() + incidents, err := s.store.ListIncidents() + if err != nil { + return s.events.List() + } + return incidents +} + +func (s *Service) Close() error { + if s.store == nil { + return nil + } + return s.store.Close() } func (s *Service) deliverEvent(event events.Event, notification Notification) error { From aaa57985844ffcf766db60388c2a8c4d70531d54 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 13:23:25 +0100 Subject: [PATCH 05/46] feat(notifications): Add event delivery rules --- internal/api/events_handlers.go | 20 +++++++++++ internal/api/notifications_test.go | 20 +++++++++++ internal/api/server.go | 2 ++ internal/notify/events_test.go | 27 +++++++++++++++ internal/notify/notify.go | 55 ++++++++++++++++++++++++++++-- 5 files changed, 121 insertions(+), 3 deletions(-) diff --git a/internal/api/events_handlers.go b/internal/api/events_handlers.go index a7dcdff..d2028ce 100644 --- a/internal/api/events_handlers.go +++ b/internal/api/events_handlers.go @@ -5,9 +5,29 @@ import ( "time" "github.com/flatrun/agent/internal/events" + "github.com/flatrun/agent/internal/notify" "github.com/gin-gonic/gin" ) +func (s *Server) listNotificationRules(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{"rules": s.notify.Load().Rules}) +} + +func (s *Server) updateNotificationRules(c *gin.Context) { + var req struct { + Rules []notify.Rule `json:"rules"` + } + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request"}) + return + } + if err := s.notify.UpdateRules(req.Rules); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, gin.H{"rules": s.notify.Load().Rules}) +} + func (s *Server) listNotificationIncidents(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"incidents": s.notify.Incidents()}) } diff --git a/internal/api/notifications_test.go b/internal/api/notifications_test.go index 66e9b12..c39932e 100644 --- a/internal/api/notifications_test.go +++ b/internal/api/notifications_test.go @@ -24,12 +24,32 @@ func setupNotifyTest(t *testing.T) (*Server, *gin.Engine) { r := gin.New() r.GET("/notifications/targets", s.getNotificationTargets) r.GET("/notifications/incidents", s.listNotificationIncidents) + r.GET("/notifications/rules", s.listNotificationRules) + r.PUT("/notifications/rules", s.updateNotificationRules) r.PUT("/notifications/targets", s.updateNotificationTargets) r.POST("/internal/notify/emit", s.emitNotification) r.POST("/internal/events", s.emitEvent) return s, r } +func TestNotificationRulesRoundTripThroughHTTP(t *testing.T) { + _, r := setupNotifyTest(t) + payload := `{"rules":[{"id":"critical-fleet","name":"Critical fleet incidents","enabled":true,"topics":["fleet"],"severities":["critical"],"target_ids":["email"]}]}` + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPut, "/notifications/rules", bytes.NewBufferString(payload)) + req.Header.Set("Content-Type", "application/json") + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) + } + + w = httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/notifications/rules", nil)) + if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"id":"critical-fleet"`) { + t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) + } +} + func TestEmitEventCorrelatesIncidentThroughHTTP(t *testing.T) { _, r := setupNotifyTest(t) emit := func(payload string) *httptest.ResponseRecorder { diff --git a/internal/api/server.go b/internal/api/server.go index a72027f..ebb338a 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -525,6 +525,8 @@ func (s *Server) setupRoutes() { protected.POST("/agent/update", s.authMiddleware.RequirePermission(auth.PermSettingsWrite), s.triggerAgentUpdate) protected.GET("/notifications/targets", s.authMiddleware.RequirePermission(auth.PermSettingsRead), s.getNotificationTargets) protected.GET("/notifications/incidents", s.authMiddleware.RequirePermission(auth.PermSettingsRead), s.listNotificationIncidents) + protected.GET("/notifications/rules", s.authMiddleware.RequirePermission(auth.PermSettingsRead), s.listNotificationRules) + protected.PUT("/notifications/rules", s.authMiddleware.RequirePermission(auth.PermSettingsWrite), s.updateNotificationRules) protected.PUT("/notifications/targets", s.authMiddleware.RequirePermission(auth.PermSettingsWrite), s.updateNotificationTargets) protected.POST("/notifications/test", s.authMiddleware.RequirePermission(auth.PermSettingsWrite), s.testNotification) protected.GET("/config", s.authMiddleware.RequirePermission(auth.PermConfigRead), s.listConfig) diff --git a/internal/notify/events_test.go b/internal/notify/events_test.go index b99905c..f86b6a1 100644 --- a/internal/notify/events_test.go +++ b/internal/notify/events_test.go @@ -86,3 +86,30 @@ func TestIncidentsSurviveServiceRestart(t *testing.T) { t.Fatalf("result = %#v", result) } } + +func TestPublishUsesMatchingRules(t *testing.T) { + service := NewService(t.TempDir()) + defer service.Close() + if err := service.Save(Config{ + Targets: []Target{ + {ID: "email", URL: "generic+https://email.example.test", Enabled: true}, + {ID: "webhook", URL: "generic+https://hook.example.test", Enabled: true}, + }, + Rules: []Rule{{ID: "critical", Enabled: true, Topics: []string{"fleet"}, Severities: []events.Severity{events.SeverityCritical}, TargetIDs: []string{"email"}}}, + }); err != nil { + t.Fatal(err) + } + var targets []string + service.send = func(target, _ string) error { + targets = append(targets, target) + return nil + } + + _, err := service.Publish(events.Event{Source: "fleet", Type: "node.unavailable", Severity: events.SeverityCritical, Title: "prod2 unavailable", Scope: events.Scope{Node: "prod2"}, OccurredAt: time.Now()}) + if err != nil { + t.Fatal(err) + } + if len(targets) != 1 || !strings.Contains(targets[0], "email.example.test") { + t.Fatalf("targets = %#v", targets) + } +} diff --git a/internal/notify/notify.go b/internal/notify/notify.go index c22ad0e..e5a9be2 100644 --- a/internal/notify/notify.go +++ b/internal/notify/notify.go @@ -52,6 +52,20 @@ func (t Target) MarshalJSON() ([]byte, error) { // Config is the persisted notification settings. type Config struct { Targets []Target `yaml:"targets" json:"targets"` + Rules []Rule `yaml:"rules,omitempty" json:"rules,omitempty"` +} + +type Rule struct { + ID string `yaml:"id" json:"id"` + Name string `yaml:"name" json:"name"` + Enabled bool `yaml:"enabled" json:"enabled"` + Topics []string `yaml:"topics,omitempty" json:"topics,omitempty"` + EventTypes []string `yaml:"event_types,omitempty" json:"event_types,omitempty"` + Severities []events.Severity `yaml:"severities,omitempty" json:"severities,omitempty"` + Nodes []string `yaml:"nodes,omitempty" json:"nodes,omitempty"` + Deployments []string `yaml:"deployments,omitempty" json:"deployments,omitempty"` + Notifications []events.NotificationAction `yaml:"notifications,omitempty" json:"notifications,omitempty"` + TargetIDs []string `yaml:"target_ids" json:"target_ids"` } // Service loads/saves targets and delivers messages. @@ -116,7 +130,7 @@ func (s *Service) Publish(event events.Event) (events.IngestResult, error) { Detail: fmt.Sprintf("Source: %s", event.Source), }}, } - return result, s.deliverEvent(event, notification) + return result, s.deliverEvent(event, result.Notification, notification) } func (s *Service) Incidents() []events.Incident { @@ -134,11 +148,12 @@ func (s *Service) Close() error { return s.store.Close() } -func (s *Service) deliverEvent(event events.Event, notification Notification) error { +func (s *Service) deliverEvent(event events.Event, action events.NotificationAction, notification Notification) error { cfg := s.Load() + selected := matchingRuleTargets(cfg.Rules, event, action) var firstErr error for _, target := range cfg.Targets { - if !target.Enabled || target.URL == "" || !targetMatches(target, event) { + if !target.Enabled || target.URL == "" || !targetMatches(target, event) || (len(cfg.Rules) > 0 && !selected[target.ID]) { continue } if err := s.deliver(target.URL, notification); err != nil && firstErr == nil { @@ -148,6 +163,33 @@ func (s *Service) deliverEvent(event events.Event, notification Notification) er return firstErr } +func matchingRuleTargets(rules []Rule, event events.Event, action events.NotificationAction) map[string]bool { + selected := make(map[string]bool) + for _, rule := range rules { + if !rule.Enabled || !matchesString(rule.Topics, event.Source) || !matchesString(rule.EventTypes, event.Type) || + !matchesSeverity(rule.Severities, event.Severity) || !matchesString(rule.Nodes, event.Scope.Node) || + !matchesString(rule.Deployments, event.Scope.Deployment) || !matchesAction(rule.Notifications, action) { + continue + } + for _, id := range rule.TargetIDs { + selected[id] = true + } + } + return selected +} + +func matchesAction(filter []events.NotificationAction, value events.NotificationAction) bool { + if len(filter) == 0 { + return true + } + for _, candidate := range filter { + if candidate == value { + return true + } + } + return false +} + func targetMatches(target Target, event events.Event) bool { return matchesString(target.Topics, event.Source) && matchesSeverity(target.Severities, event.Severity) && @@ -209,6 +251,7 @@ func (s *Service) Save(cfg Config) error { // unchanged must not overwrite the real URL with the mask. func (s *Service) Update(cfg Config) error { stored := s.Load() + cfg.Rules = stored.Rules byID := make(map[string]string, len(stored.Targets)) for _, t := range stored.Targets { byID[t.ID] = t.URL @@ -221,6 +264,12 @@ func (s *Service) Update(cfg Config) error { return s.Save(cfg) } +func (s *Service) UpdateRules(rules []Rule) error { + cfg := s.Load() + cfg.Rules = rules + return s.Save(cfg) +} + // Test sends a message to a single URL, so an admin can verify a target before saving it. func (s *Service) Test(url string) error { if url == "" { From c3fd3a576ef168465cb483abe4f1576c7a536531 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 13:39:15 +0100 Subject: [PATCH 06/46] feat(capacity): Persist scaling policy --- internal/api/openapi.json | 244 ++++++++++++++++++++++++++++- internal/api/resource_handlers.go | 2 +- internal/capacity/capacity.go | 37 ++++- internal/capacity/capacity_test.go | 26 ++- pkg/config/capacity_test.go | 28 ++++ pkg/config/config.go | 40 +++++ 6 files changed, 372 insertions(+), 5 deletions(-) create mode 100644 pkg/config/capacity_test.go diff --git a/internal/api/openapi.json b/internal/api/openapi.json index 7f9182e..af33c5e 100644 --- a/internal/api/openapi.json +++ b/internal/api/openapi.json @@ -1612,6 +1612,30 @@ "x-permission": "cluster:write" } }, + "/api/cluster/setup": { + "post": { + "operationId": "post-cluster-setup", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/api.clusterSetupRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "cluster" + ], + "x-permission": "cluster:write" + } + }, "/api/cluster/stats": { "get": { "operationId": "get-cluster-stats", @@ -1799,6 +1823,30 @@ "x-permission": "containers:delete" } }, + "/api/containers/{id}/capacity": { + "get": { + "operationId": "get-containers-by-id-capacity", + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "containers" + ], + "x-permission": "containers:read" + } + }, "/api/containers/{id}/exec": { "get": { "operationId": "get-containers-by-id-exec", @@ -6485,6 +6533,67 @@ "x-permission": "networks:write" } }, + "/api/notifications/incidents": { + "get": { + "operationId": "get-notifications-incidents", + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "notifications" + ], + "x-permission": "settings:read" + } + }, + "/api/notifications/rules": { + "get": { + "operationId": "get-notifications-rules", + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "notifications" + ], + "x-permission": "settings:read" + }, + "put": { + "operationId": "put-notifications-rules", + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "rules": { + "items": { + "$ref": "#/components/schemas/notify.Rule" + }, + "type": "array" + } + }, + "type": "object", + "x-property-order": [ + "rules" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "notifications" + ], + "x-permission": "settings:write" + } + }, "/api/notifications/targets": { "get": { "operationId": "get-notifications-targets", @@ -11006,6 +11115,29 @@ "tail" ] }, + "api.clusterSetupRequest": { + "type": "object", + "properties": { + "advertise_url": { + "type": "string" + }, + "server_name": { + "type": "string" + } + }, + "x-property-order": [ + "server_name", + "advertise_url" + ], + "x-columns": [ + "server_name", + "advertise_url" + ], + "required": [ + "server_name", + "advertise_url" + ] + }, "api.deploymentSource": { "type": "object", "properties": { @@ -13189,6 +13321,12 @@ "notify.Config": { "type": "object", "properties": { + "rules": { + "type": "array", + "items": { + "$ref": "#/components/schemas/notify.Rule" + } + }, "targets": { "type": "array", "items": { @@ -13197,12 +13335,92 @@ } }, "x-property-order": [ - "targets" + "targets", + "rules" + ] + }, + "notify.Rule": { + "type": "object", + "properties": { + "deployments": { + "type": "array", + "items": { + "type": "string" + } + }, + "enabled": { + "type": "boolean" + }, + "event_types": { + "type": "array", + "items": { + "type": "string" + } + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "nodes": { + "type": "array", + "items": { + "type": "string" + } + }, + "notifications": { + "type": "array", + "items": { + "type": "string" + } + }, + "severities": { + "type": "array", + "items": { + "type": "string" + } + }, + "target_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "topics": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "x-property-order": [ + "id", + "name", + "enabled", + "topics", + "event_types", + "severities", + "nodes", + "deployments", + "notifications", + "target_ids" + ], + "x-columns": [ + "id", + "name", + "enabled" ] }, "notify.Target": { "type": "object", "properties": { + "deployments": { + "type": "array", + "items": { + "type": "string" + } + }, "enabled": { "type": "boolean" }, @@ -13212,6 +13430,24 @@ "name": { "type": "string" }, + "nodes": { + "type": "array", + "items": { + "type": "string" + } + }, + "severities": { + "type": "array", + "items": { + "type": "string" + } + }, + "topics": { + "type": "array", + "items": { + "type": "string" + } + }, "url": { "type": "string" } @@ -13220,7 +13456,11 @@ "id", "name", "url", - "enabled" + "enabled", + "topics", + "severities", + "nodes", + "deployments" ], "x-columns": [ "id", diff --git a/internal/api/resource_handlers.go b/internal/api/resource_handlers.go index 88e80ec..4a6b8b2 100644 --- a/internal/api/resource_handlers.go +++ b/internal/api/resource_handlers.go @@ -65,7 +65,7 @@ func (s *Server) diagnoseContainerCapacity(c *gin.Context) { MemoryUsage: stats.MemoryUsage, MemoryLimit: uint64(max(limits.MemoryLimit, 0)), } - policy := capacity.DefaultPolicy() + policy := capacity.PolicyFromConfig(s.config.Capacity) c.JSON(http.StatusOK, gin.H{ "host": host, "container": container, diff --git a/internal/capacity/capacity.go b/internal/capacity/capacity.go index 94781d6..d80d426 100644 --- a/internal/capacity/capacity.go +++ b/internal/capacity/capacity.go @@ -1,6 +1,10 @@ package capacity -import "math" +import ( + "math" + + "github.com/flatrun/agent/pkg/config" +) type Pressure string @@ -71,6 +75,37 @@ func DefaultPolicy() Policy { } } +func PolicyFromConfig(cfg config.CapacityConfig) Policy { + policy := DefaultPolicy() + if cfg.AllocationThresholdPercent > 0 { + policy.AllocationThresholdPercent = cfg.AllocationThresholdPercent + } + if cfg.HostThresholdPercent > 0 { + policy.HostThresholdPercent = cfg.HostThresholdPercent + } + if cfg.HostMemoryReserve > 0 { + policy.HostMemoryReserve = cfg.HostMemoryReserve + } + if cfg.HostCPUReserve > 0 { + policy.HostCPUReserve = cfg.HostCPUReserve + } + if cfg.MemoryStepPercent > 0 { + policy.MemoryStepPercent = cfg.MemoryStepPercent + } + if cfg.CPUStepPercent > 0 { + policy.CPUStepPercent = cfg.CPUStepPercent + } + policy.MaxMemory = cfg.MaxMemory + policy.MaxCPU = cfg.MaxCPU + if cfg.AllowVertical != nil { + policy.AllowVertical = *cfg.AllowVertical + } + if cfg.AllowHorizontal != nil { + policy.AllowHorizontal = *cfg.AllowHorizontal + } + return policy +} + func Diagnose(host Host, container Container, policy Policy) Diagnosis { if policy.AllocationThresholdPercent <= 0 { policy = DefaultPolicy() diff --git a/internal/capacity/capacity_test.go b/internal/capacity/capacity_test.go index e94384a..aa64a29 100644 --- a/internal/capacity/capacity_test.go +++ b/internal/capacity/capacity_test.go @@ -1,6 +1,10 @@ package capacity -import "testing" +import ( + "testing" + + "github.com/flatrun/agent/pkg/config" +) func TestDiagnoseIncreasesContainerMemoryBeforeScaling(t *testing.T) { policy := DefaultPolicy() @@ -36,3 +40,23 @@ func TestDiagnoseReportsUnlimitedContainerAsHealthyWhenHostHasHeadroom(t *testin t.Fatalf("diagnosis = %#v", diagnosis) } } + +func TestPolicyFromConfigPreservesExplicitScalingChoices(t *testing.T) { + disabled := false + policy := PolicyFromConfig(config.CapacityConfig{ + AllocationThresholdPercent: 75, + HostThresholdPercent: 80, + AllowVertical: &disabled, + AllowHorizontal: &disabled, + }) + + if policy.AllocationThresholdPercent != 75 || policy.HostThresholdPercent != 80 { + t.Fatalf("thresholds = %.0f, %.0f", policy.AllocationThresholdPercent, policy.HostThresholdPercent) + } + if policy.AllowVertical || policy.AllowHorizontal { + t.Fatalf("scaling choices = vertical:%t horizontal:%t", policy.AllowVertical, policy.AllowHorizontal) + } + if policy.HostMemoryReserve == 0 || policy.MemoryStepPercent == 0 { + t.Fatalf("defaults were not retained: %#v", policy) + } +} diff --git a/pkg/config/capacity_test.go b/pkg/config/capacity_test.go new file mode 100644 index 0000000..5e4cf9f --- /dev/null +++ b/pkg/config/capacity_test.go @@ -0,0 +1,28 @@ +package config + +import "testing" + +func TestCapacityDefaultsAllowVerticalAndHorizontalScaling(t *testing.T) { + cfg := &Config{} + setDefaults(cfg) + + if cfg.Capacity.AllocationThresholdPercent != 90 || cfg.Capacity.HostThresholdPercent != 85 { + t.Fatalf("thresholds = %.0f, %.0f", cfg.Capacity.AllocationThresholdPercent, cfg.Capacity.HostThresholdPercent) + } + if cfg.Capacity.AllowVertical == nil || !*cfg.Capacity.AllowVertical { + t.Fatal("vertical scaling should default to enabled") + } + if cfg.Capacity.AllowHorizontal == nil || !*cfg.Capacity.AllowHorizontal { + t.Fatal("horizontal scaling should default to enabled") + } +} + +func TestCapacityDefaultsPreserveExplicitDisabledScaling(t *testing.T) { + disabled := false + cfg := &Config{Capacity: CapacityConfig{AllowVertical: &disabled, AllowHorizontal: &disabled}} + setDefaults(cfg) + + if *cfg.Capacity.AllowVertical || *cfg.Capacity.AllowHorizontal { + t.Fatal("explicit scaling choices were overwritten") + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index b7578ec..e0c5f5b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -21,6 +21,19 @@ type ClusterConfig struct { RequestTimeout string `yaml:"request_timeout"` } +type CapacityConfig struct { + AllocationThresholdPercent float64 `yaml:"allocation_threshold_percent" json:"allocation_threshold_percent"` + HostThresholdPercent float64 `yaml:"host_threshold_percent" json:"host_threshold_percent"` + HostMemoryReserve uint64 `yaml:"host_memory_reserve" json:"host_memory_reserve"` + HostCPUReserve float64 `yaml:"host_cpu_reserve" json:"host_cpu_reserve"` + MemoryStepPercent float64 `yaml:"memory_step_percent" json:"memory_step_percent"` + CPUStepPercent float64 `yaml:"cpu_step_percent" json:"cpu_step_percent"` + MaxMemory uint64 `yaml:"max_memory" json:"max_memory"` + MaxCPU float64 `yaml:"max_cpu" json:"max_cpu"` + AllowVertical *bool `yaml:"allow_vertical" json:"allow_vertical"` + AllowHorizontal *bool `yaml:"allow_horizontal" json:"allow_horizontal"` +} + type Config struct { DeploymentsPath string `yaml:"deployments_path"` SystemFilesRoot string `yaml:"system_files_root"` @@ -37,6 +50,7 @@ type Config struct { Security SecurityConfig `yaml:"security"` Audit AuditConfig `yaml:"audit"` Cluster ClusterConfig `yaml:"cluster"` + Capacity CapacityConfig `yaml:"capacity"` SystemTerminal SystemTerminalConfig `yaml:"system_terminal"` Cleanup CleanupConfig `yaml:"cleanup"` Plans PlansConfig `yaml:"plans"` @@ -386,6 +400,32 @@ func setDefaults(cfg *Config) { if cfg.DefaultTimeout == 0 { cfg.DefaultTimeout = 2 * time.Minute } + if cfg.Capacity.AllocationThresholdPercent == 0 { + cfg.Capacity.AllocationThresholdPercent = 90 + } + if cfg.Capacity.HostThresholdPercent == 0 { + cfg.Capacity.HostThresholdPercent = 85 + } + if cfg.Capacity.HostMemoryReserve == 0 { + cfg.Capacity.HostMemoryReserve = 512 * 1024 * 1024 + } + if cfg.Capacity.HostCPUReserve == 0 { + cfg.Capacity.HostCPUReserve = 0.25 + } + if cfg.Capacity.MemoryStepPercent == 0 { + cfg.Capacity.MemoryStepPercent = 50 + } + if cfg.Capacity.CPUStepPercent == 0 { + cfg.Capacity.CPUStepPercent = 50 + } + if cfg.Capacity.AllowVertical == nil { + enabled := true + cfg.Capacity.AllowVertical = &enabled + } + if cfg.Capacity.AllowHorizontal == nil { + enabled := true + cfg.Capacity.AllowHorizontal = &enabled + } if cfg.Cleanup.Timeout == 0 { cfg.Cleanup.Timeout = cfg.DefaultTimeout } From 41d6729976bf717fb1acd068704ce7a75d53f1f6 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 13:42:06 +0100 Subject: [PATCH 07/46] feat(capacity): Publish permitted fleet headroom --- internal/api/cluster_handlers.go | 35 ++++++++++++++++++++++++++ internal/api/cluster_handlers_test.go | 36 +++++++++++++++++++++++++++ internal/api/openapi.json | 34 +++++++++++++++++++++++++ internal/api/resource_handlers.go | 20 +++++++++++++++ internal/api/server.go | 2 ++ internal/capacity/capacity.go | 28 +++++++++++++++++++++ internal/capacity/capacity_test.go | 26 +++++++++++++++++++ pkg/config/config.go | 1 + 8 files changed, 182 insertions(+) diff --git a/internal/api/cluster_handlers.go b/internal/api/cluster_handlers.go index 1ee95bf..af198da 100644 --- a/internal/api/cluster_handlers.go +++ b/internal/api/cluster_handlers.go @@ -14,7 +14,9 @@ import ( "time" "github.com/flatrun/agent/internal/auth" + "github.com/flatrun/agent/internal/capacity" "github.com/flatrun/agent/internal/cluster" + "github.com/flatrun/agent/internal/system" "github.com/flatrun/agent/pkg/config" "github.com/flatrun/agent/pkg/version" "github.com/gin-gonic/gin" @@ -468,3 +470,36 @@ func (s *Server) clusterAggregateStats(c *gin.Context) { result := cluster.AggregateFromPeers(c.Request.Context(), localData, mgr, "/api/health") c.JSON(http.StatusOK, result) } + +func (s *Server) clusterAggregateCapacity(c *gin.Context) { + mgr := s.getClusterManager() + if mgr == nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) + return + } + + hostStats, err := system.GetSystemStats() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + host := capacity.Host{ + CPUCores: float64(hostStats.CPU.Cores), + CPUUsagePercent: hostStats.CPU.UsagePercent, + MemoryTotal: hostStats.Memory.Total, + MemoryAvailable: hostStats.Memory.Available, + } + policy := capacity.PolicyFromConfig(s.config.Capacity) + localData, err := json.Marshal(gin.H{ + "host": host, + "policy": policy, + "offer": capacity.FleetOffer(host, policy, s.config.Capacity.OfferToFleet), + }) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to marshal local capacity"}) + return + } + + result := cluster.AggregateFromPeers(c.Request.Context(), localData, mgr, "/api/capacity") + c.JSON(http.StatusOK, result) +} diff --git a/internal/api/cluster_handlers_test.go b/internal/api/cluster_handlers_test.go index 7be1bc9..02ab399 100644 --- a/internal/api/cluster_handlers_test.go +++ b/internal/api/cluster_handlers_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/flatrun/agent/internal/auth" + "github.com/flatrun/agent/internal/capacity" "github.com/flatrun/agent/internal/cluster" "github.com/flatrun/agent/pkg/config" "github.com/gin-gonic/gin" @@ -93,6 +94,7 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool protected := api.Group("") protected.Use(authMiddleware.RequireAuth()) { + protected.GET("/capacity", authMiddleware.RequirePermission(auth.PermSystemRead), server.getCapacityStatus) clusterGroup := protected.Group("/cluster") clusterGroup.Use(authMiddleware.RequirePermission(auth.PermClusterRead)) { @@ -105,6 +107,7 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool clusterGroup.Any("/peers/:name/proxy/*path", authMiddleware.RequirePermission(auth.PermClusterWrite), server.clusterProxy) clusterGroup.GET("/deployments", server.clusterAggregateDeployments) clusterGroup.GET("/stats", server.clusterAggregateStats) + clusterGroup.GET("/capacity", server.clusterAggregateCapacity) } } @@ -124,6 +127,39 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool } } +func TestClusterCapacityIncludesLocalOfferPolicy(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + + token := clusterLogin(t, env.router) + req := httptest.NewRequest(http.MethodGet, "/api/cluster/capacity", nil) + req.Header.Set("Authorization", "Bearer "+token) + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + var response struct { + Servers map[string]struct { + Online bool `json:"online"` + Data struct { + Offer capacity.Offer `json:"offer"` + } `json:"data"` + } `json:"servers"` + } + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatalf("decode response: %v", err) + } + local, ok := response.Servers["server-a"] + if !ok || !local.Online { + t.Fatalf("local server = %#v", local) + } + if local.Data.Offer.Enabled { + t.Fatal("fleet capacity should require explicit permission") + } +} + func TestClusterSetupEnablesClusterWithoutRestart(t *testing.T) { env := setupClusterTestServer(t, "", false) defer env.cleanup() diff --git a/internal/api/openapi.json b/internal/api/openapi.json index af33c5e..066b1ad 100644 --- a/internal/api/openapi.json +++ b/internal/api/openapi.json @@ -1257,6 +1257,20 @@ "x-permission": "backups:write" } }, + "/api/capacity": { + "get": { + "operationId": "get-capacity", + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "capacity" + ], + "x-permission": "system:read" + } + }, "/api/certificates": { "get": { "operationId": "get-certificates", @@ -1511,6 +1525,26 @@ "x-permission": "cluster:write" } }, + "/api/cluster/capacity": { + "get": { + "operationId": "get-cluster-capacity", + "tags": [ + "cluster" + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/cluster.AggregatedResponse" + } + } + } + } + } + } + }, "/api/cluster/deployments": { "get": { "operationId": "get-cluster-deployments", diff --git a/internal/api/resource_handlers.go b/internal/api/resource_handlers.go index 4a6b8b2..5885a9b 100644 --- a/internal/api/resource_handlers.go +++ b/internal/api/resource_handlers.go @@ -10,6 +10,26 @@ import ( "github.com/gin-gonic/gin" ) +func (s *Server) getCapacityStatus(c *gin.Context) { + hostStats, err := system.GetSystemStats() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + host := capacity.Host{ + CPUCores: float64(hostStats.CPU.Cores), + CPUUsagePercent: hostStats.CPU.UsagePercent, + MemoryTotal: hostStats.Memory.Total, + MemoryAvailable: hostStats.Memory.Available, + } + policy := capacity.PolicyFromConfig(s.config.Capacity) + c.JSON(http.StatusOK, gin.H{ + "host": host, + "policy": policy, + "offer": capacity.FleetOffer(host, policy, s.config.Capacity.OfferToFleet), + }) +} + func (s *Server) getContainerResources(c *gin.Context) { id := c.Param("id") if !s.requireContainerAccess(c, id, auth.AccessLevelRead) { diff --git a/internal/api/server.go b/internal/api/server.go index ebb338a..6bc6a07 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -610,6 +610,7 @@ func (s *Server) setupRoutes() { protected.POST("/containers/:id/exec", s.authMiddleware.RequirePermission(auth.PermContainersWrite), s.containerExecHTTP) protected.GET("/containers/:id/resources", s.authMiddleware.RequirePermission(auth.PermContainersRead), s.getContainerResources) protected.GET("/containers/:id/capacity", s.authMiddleware.RequirePermission(auth.PermContainersRead), s.diagnoseContainerCapacity) + protected.GET("/capacity", s.authMiddleware.RequirePermission(auth.PermSystemRead), s.getCapacityStatus) protected.PUT("/containers/:id/resources", s.authMiddleware.RequirePermission(auth.PermContainersWrite), s.updateContainerResources) protected.GET("/deployments/:name/stats", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentContainerStats) protected.GET("/deployments/:name/resources", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentResources) @@ -885,6 +886,7 @@ func (s *Server) setupRoutes() { clusterGroup.Any("/peers/:name/proxy/*path", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.clusterProxy) clusterGroup.GET("/deployments", s.clusterAggregateDeployments) clusterGroup.GET("/stats", s.clusterAggregateStats) + clusterGroup.GET("/capacity", s.clusterAggregateCapacity) } } diff --git a/internal/capacity/capacity.go b/internal/capacity/capacity.go index d80d426..7cca183 100644 --- a/internal/capacity/capacity.go +++ b/internal/capacity/capacity.go @@ -62,6 +62,13 @@ type Diagnosis struct { Reason string `json:"reason"` } +type Offer struct { + Enabled bool `json:"enabled"` + AvailableCPU float64 `json:"available_cpu"` + AvailableMemory uint64 `json:"available_memory"` + Reason string `json:"reason"` +} + func DefaultPolicy() Policy { return Policy{ AllocationThresholdPercent: 90, @@ -147,6 +154,27 @@ func Diagnose(host Host, container Container, policy Policy) Diagnosis { return Diagnosis{Pressure: PressureNone, Action: ActionNone, Reason: "Container allocation and host capacity are within policy"} } +func FleetOffer(host Host, policy Policy, enabled bool) Offer { + if !enabled { + return Offer{Reason: "This server has not permitted fleet workloads"} + } + + availableCPU := host.CPUCores*math.Max(0, 100-host.CPUUsagePercent)/100 - policy.HostCPUReserve + if availableCPU < 0 { + availableCPU = 0 + } + availableMemory := uint64(0) + if host.MemoryAvailable > policy.HostMemoryReserve { + availableMemory = host.MemoryAvailable - policy.HostMemoryReserve + } + enabled = availableCPU > 0 && availableMemory > 0 + reason := "CPU and memory are available above the host safety reserve" + if !enabled { + reason = "Host capacity is at or below its safety reserve" + } + return Offer{Enabled: enabled, AvailableCPU: availableCPU, AvailableMemory: availableMemory, Reason: reason} +} + func grow(current, percent, maximum float64) float64 { next := current * (1 + percent/100) if maximum > 0 && next > maximum { diff --git a/internal/capacity/capacity_test.go b/internal/capacity/capacity_test.go index aa64a29..43b5e53 100644 --- a/internal/capacity/capacity_test.go +++ b/internal/capacity/capacity_test.go @@ -60,3 +60,29 @@ func TestPolicyFromConfigPreservesExplicitScalingChoices(t *testing.T) { t.Fatalf("defaults were not retained: %#v", policy) } } + +func TestFleetOfferKeepsHostReserve(t *testing.T) { + policy := DefaultPolicy() + offer := FleetOffer(Host{ + CPUCores: 8, + CPUUsagePercent: 25, + MemoryAvailable: 8 << 30, + }, policy, true) + + if !offer.Enabled { + t.Fatalf("offer = %#v", offer) + } + if offer.AvailableCPU != 5.75 { + t.Fatalf("available CPU = %.2f", offer.AvailableCPU) + } + if offer.AvailableMemory != 7680<<20 { + t.Fatalf("available memory = %d", offer.AvailableMemory) + } +} + +func TestFleetOfferRequiresOperatorPermission(t *testing.T) { + offer := FleetOffer(Host{CPUCores: 8, MemoryAvailable: 8 << 30}, DefaultPolicy(), false) + if offer.Enabled || offer.AvailableCPU != 0 || offer.AvailableMemory != 0 { + t.Fatalf("offer = %#v", offer) + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index e0c5f5b..8fbe965 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -32,6 +32,7 @@ type CapacityConfig struct { MaxCPU float64 `yaml:"max_cpu" json:"max_cpu"` AllowVertical *bool `yaml:"allow_vertical" json:"allow_vertical"` AllowHorizontal *bool `yaml:"allow_horizontal" json:"allow_horizontal"` + OfferToFleet bool `yaml:"offer_to_fleet" json:"offer_to_fleet"` } type Config struct { From 23232b38e59cb591e0dcd2b77191faedf3b05053 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 14:15:24 +0100 Subject: [PATCH 08/46] feat(cluster): Version peer access policies --- go.mod | 4 + go.sum | 8 ++ internal/api/cluster_handlers.go | 57 ++++++++ internal/api/cluster_handlers_test.go | 37 +++++ internal/api/openapi.json | 126 ++++++++++++++++++ internal/api/server.go | 2 + internal/cluster/capabilities.go | 24 ++++ internal/cluster/db.go | 104 ++++++++++----- internal/cluster/db_test.go | 62 +++++++++ internal/cluster/migrations/00001_initial.sql | 30 +++++ .../migrations/00002_peer_policies.sql | 9 ++ 11 files changed, 433 insertions(+), 30 deletions(-) create mode 100644 internal/cluster/migrations/00001_initial.sql create mode 100644 internal/cluster/migrations/00002_peer_policies.sql diff --git a/go.mod b/go.mod index 18c3104..fdac7ee 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/moby/moby/client v0.2.2 github.com/modelcontextprotocol/go-sdk v1.6.1 github.com/nicholas-fedor/shoutrrr v0.16.1 + github.com/pressly/goose/v3 v3.26.0 github.com/robfig/cron/v3 v3.0.1 github.com/spf13/cobra v1.10.2 github.com/testcontainers/testcontainers-go v0.41.0 @@ -137,6 +138,7 @@ require ( github.com/mattn/go-isatty v0.0.22 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mattn/go-shellwords v1.0.12 // indirect + github.com/mfridman/interpolate v0.0.2 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/moby/buildkit v0.27.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -170,6 +172,7 @@ require ( github.com/secure-systems-lab/go-securesystemslib v0.9.1 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/segmentio/encoding v0.5.4 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect github.com/shibumi/go-pathspec v1.3.0 // indirect github.com/shirou/gopsutil/v4 v4.26.2 // indirect github.com/sigstore/sigstore v1.10.4 // indirect @@ -200,6 +203,7 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0 // indirect go.opentelemetry.io/otel/trace v1.41.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect + go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect go.yaml.in/yaml/v4 v4.0.0-rc.4 // indirect golang.org/x/arch v0.18.0 // indirect diff --git a/go.sum b/go.sum index 4ac740c..a4711ad 100644 --- a/go.sum +++ b/go.sum @@ -343,6 +343,8 @@ github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6T github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= +github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/moby/buildkit v0.27.1 h1:qlIWpnZzqCkrYiGkctM1gBD/YZPOJTjtUdRBlI0oBOU= @@ -420,6 +422,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM= +github.com/pressly/goose/v3 v3.26.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -446,6 +450,8 @@ github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI= @@ -570,6 +576,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= diff --git a/internal/api/cluster_handlers.go b/internal/api/cluster_handlers.go index af198da..51de04d 100644 --- a/internal/api/cluster_handlers.go +++ b/internal/api/cluster_handlers.go @@ -141,6 +141,63 @@ func (s *Server) clusterListPeers(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"peers": peers}) } +func (s *Server) clusterPeerPolicy(c *gin.Context) { + mgr := s.getClusterManager() + if mgr == nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) + return + } + policy, err := mgr.DB().GetPeerPolicy(c.Param("name")) + if err != nil { + if err == sql.ErrNoRows { + c.JSON(http.StatusNotFound, gin.H{"error": "Peer not found"}) + return + } + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, policy) +} + +func (s *Server) updateClusterPeerPolicy(c *gin.Context) { + mgr := s.getClusterManager() + if mgr == nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) + return + } + policy := cluster.PeerPolicy{Peer: c.Param("name")} + if err := c.ShouldBindJSON(&policy); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + policy.Peer = c.Param("name") + seen := make(map[cluster.Capability]bool, len(policy.Grants)) + for _, grant := range policy.Grants { + if !cluster.ValidCapability(grant.Capability) { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Unknown capability %q", grant.Capability)}) + return + } + if seen[grant.Capability] { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Duplicate capability %q", grant.Capability)}) + return + } + if grant.MaxCPU < 0 || grant.MaxReplicas < 0 { + c.JSON(http.StatusBadRequest, gin.H{"error": "Resource limits cannot be negative"}) + return + } + seen[grant.Capability] = true + } + if err := mgr.DB().SetPeerPolicy(policy); err != nil { + if err == sql.ErrNoRows { + c.JSON(http.StatusNotFound, gin.H{"error": "Peer not found"}) + return + } + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, policy) +} + func (s *Server) clusterInvite(c *gin.Context) { mgr := s.getClusterManager() if mgr == nil { diff --git a/internal/api/cluster_handlers_test.go b/internal/api/cluster_handlers_test.go index 02ab399..659c685 100644 --- a/internal/api/cluster_handlers_test.go +++ b/internal/api/cluster_handlers_test.go @@ -101,6 +101,8 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool clusterGroup.GET("/status", server.clusterStatus) clusterGroup.POST("/setup", authMiddleware.RequirePermission(auth.PermClusterWrite), server.clusterSetup) clusterGroup.GET("/peers", server.clusterListPeers) + clusterGroup.GET("/peers/:name/policy", server.clusterPeerPolicy) + clusterGroup.PUT("/peers/:name/policy", authMiddleware.RequirePermission(auth.PermClusterWrite), server.updateClusterPeerPolicy) clusterGroup.POST("/invite", authMiddleware.RequirePermission(auth.PermClusterWrite), server.clusterInvite) clusterGroup.POST("/accept", authMiddleware.RequirePermission(auth.PermClusterWrite), server.clusterAccept) clusterGroup.DELETE("/peers/:name", authMiddleware.RequirePermission(auth.PermClusterWrite), server.clusterRemovePeer) @@ -160,6 +162,41 @@ func TestClusterCapacityIncludesLocalOfferPolicy(t *testing.T) { } } +func TestUpdateClusterPeerPolicyThroughHTTP(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + if err := env.server.clusterManager.AddPeer("server-b", "https://server-b.example.com", "peer-key"); err != nil { + t.Fatalf("AddPeer failed: %v", err) + } + + token := clusterLogin(t, env.router) + body := []byte(`{"grants":[{"capability":"capacity.offer","max_cpu":2,"max_memory":2147483648,"max_replicas":2}]}`) + req := httptest.NewRequest(http.MethodPut, "/api/cluster/peers/server-b/policy", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + + getReq := httptest.NewRequest(http.MethodGet, "/api/cluster/peers/server-b/policy", nil) + getReq.Header.Set("Authorization", "Bearer "+token) + getWriter := httptest.NewRecorder() + env.router.ServeHTTP(getWriter, getReq) + if getWriter.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", getWriter.Code, getWriter.Body.String()) + } + var policy cluster.PeerPolicy + if err := json.Unmarshal(getWriter.Body.Bytes(), &policy); err != nil { + t.Fatalf("decode policy: %v", err) + } + if len(policy.Grants) != 1 || policy.Grants[0].MaxCPU != 2 || policy.Grants[0].MaxReplicas != 2 { + t.Fatalf("policy = %#v", policy) + } +} + func TestClusterSetupEnablesClusterWithoutRestart(t *testing.T) { env := setupClusterTestServer(t, "", false) defer env.cleanup() diff --git a/internal/api/openapi.json b/internal/api/openapi.json index 066b1ad..151fd02 100644 --- a/internal/api/openapi.json +++ b/internal/api/openapi.json @@ -1646,6 +1646,75 @@ "x-permission": "cluster:write" } }, + "/api/cluster/peers/{name}/policy": { + "get": { + "operationId": "get-cluster-peers-by-name-policy", + "tags": [ + "cluster" + ], + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/cluster.PeerPolicy" + } + } + } + } + } + }, + "put": { + "operationId": "put-cluster-peers-by-name-policy", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/cluster.PeerPolicy" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/cluster.PeerPolicy" + } + } + }, + "description": "Success" + } + }, + "tags": [ + "cluster" + ], + "x-permission": "cluster:write" + } + }, "/api/cluster/setup": { "post": { "operationId": "post-cluster-setup", @@ -11777,6 +11846,63 @@ "servers" ] }, + "cluster.Grant": { + "type": "object", + "properties": { + "capability": { + "type": "string" + }, + "deployments": { + "type": "array", + "items": { + "type": "string" + } + }, + "max_cpu": { + "type": "number" + }, + "max_memory": { + "type": "integer" + }, + "max_replicas": { + "type": "integer" + } + }, + "x-property-order": [ + "capability", + "deployments", + "max_cpu", + "max_memory", + "max_replicas" + ], + "x-columns": [ + "capability", + "max_cpu", + "max_memory", + "max_replicas" + ] + }, + "cluster.PeerPolicy": { + "type": "object", + "properties": { + "grants": { + "type": "array", + "items": { + "$ref": "#/components/schemas/cluster.Grant" + } + }, + "peer": { + "type": "string" + } + }, + "x-property-order": [ + "peer", + "grants" + ], + "x-columns": [ + "peer" + ] + }, "cluster.ServerResult": { "type": "object", "properties": { diff --git a/internal/api/server.go b/internal/api/server.go index 6bc6a07..b3dc4ca 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -880,6 +880,8 @@ func (s *Server) setupRoutes() { clusterGroup.GET("/status", s.clusterStatus) clusterGroup.POST("/setup", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.clusterSetup) clusterGroup.GET("/peers", s.clusterListPeers) + clusterGroup.GET("/peers/:name/policy", s.clusterPeerPolicy) + clusterGroup.PUT("/peers/:name/policy", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.updateClusterPeerPolicy) clusterGroup.POST("/invite", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.clusterInvite) clusterGroup.POST("/accept", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.clusterAccept) clusterGroup.DELETE("/peers/:name", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.clusterRemovePeer) diff --git a/internal/cluster/capabilities.go b/internal/cluster/capabilities.go index cf9bfa3..87ff1fc 100644 --- a/internal/cluster/capabilities.go +++ b/internal/cluster/capabilities.go @@ -19,3 +19,27 @@ type Grant struct { MaxMemory uint64 `json:"max_memory,omitempty"` MaxReplicas int `json:"max_replicas,omitempty"` } + +type PeerPolicy struct { + Peer string `json:"peer"` + Grants []Grant `json:"grants"` +} + +func DefaultPeerGrants() []Grant { + return []Grant{ + {Capability: CapabilityFleetRead}, + {Capability: CapabilityDeploymentsRead}, + {Capability: CapabilityDeploymentsRun}, + {Capability: CapabilityCapacityRead}, + } +} + +func ValidCapability(value Capability) bool { + switch value { + case CapabilityFleetRead, CapabilityDeploymentsRead, CapabilityDeploymentsRun, + CapabilityCapacityRead, CapabilityCapacityOffer, CapabilityEventsPublish, CapabilityRoutingManage: + return true + default: + return false + } +} diff --git a/internal/cluster/db.go b/internal/cluster/db.go index 87a658c..9ed4f72 100644 --- a/internal/cluster/db.go +++ b/internal/cluster/db.go @@ -1,17 +1,25 @@ package cluster import ( + "context" "crypto/sha256" "database/sql" + "embed" "encoding/hex" + "encoding/json" + "io/fs" "os" "path/filepath" "sync" "time" + "github.com/pressly/goose/v3" _ "modernc.org/sqlite" ) +//go:embed migrations/*.sql +var migrationFiles embed.FS + type Peer struct { ID int64 `json:"id"` Name string `json:"name"` @@ -71,35 +79,20 @@ func (db *DB) Close() error { } func (db *DB) migrate() error { - schema := ` - CREATE TABLE IF NOT EXISTS peers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT UNIQUE NOT NULL, - url TEXT NOT NULL, - api_key_hash TEXT NOT NULL, - api_key_encrypted TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'active', - created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - last_seen_at DATETIME - ); - - CREATE TABLE IF NOT EXISTS invites ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - token_hash TEXT UNIQUE NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - created_by INTEGER NOT NULL, - accepted_peer TEXT, - expires_at DATETIME NOT NULL, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - - CREATE INDEX IF NOT EXISTS idx_peers_name ON peers(name); - CREATE INDEX IF NOT EXISTS idx_peers_status ON peers(status); - CREATE INDEX IF NOT EXISTS idx_invites_token_hash ON invites(token_hash); - CREATE INDEX IF NOT EXISTS idx_invites_status ON invites(status); - ` - - _, err := db.conn.Exec(schema) + migrations, err := fs.Sub(migrationFiles, "migrations") + if err != nil { + return err + } + provider, err := goose.NewProvider( + goose.DialectSQLite3, + db.conn, + migrations, + goose.WithTableName("cluster_schema_version"), + ) + if err != nil { + return err + } + _, err = provider.Up(context.Background()) return err } @@ -120,7 +113,18 @@ func (db *DB) CreatePeer(peer *Peer) (int64, error) { if err != nil { return 0, err } - return result.LastInsertId() + id, err := result.LastInsertId() + if err != nil { + return 0, err + } + grants, err := json.Marshal(DefaultPeerGrants()) + if err != nil { + return 0, err + } + if _, err := db.conn.Exec(`INSERT OR IGNORE INTO peer_policies (peer_name, grants_json) VALUES (?, ?)`, peer.Name, grants); err != nil { + return 0, err + } + return id, nil } func (db *DB) GetPeer(name string) (*Peer, error) { @@ -180,10 +184,50 @@ func (db *DB) DeletePeer(name string) error { db.mu.Lock() defer db.mu.Unlock() + if _, err := db.conn.Exec(`DELETE FROM peer_policies WHERE peer_name = ?`, name); err != nil { + return err + } _, err := db.conn.Exec(`DELETE FROM peers WHERE name = ?`, name) return err } +func (db *DB) GetPeerPolicy(name string) (*PeerPolicy, error) { + db.mu.RLock() + defer db.mu.RUnlock() + + var raw string + if err := db.conn.QueryRow(`SELECT grants_json FROM peer_policies WHERE peer_name = ?`, name).Scan(&raw); err != nil { + return nil, err + } + var grants []Grant + if err := json.Unmarshal([]byte(raw), &grants); err != nil { + return nil, err + } + return &PeerPolicy{Peer: name, Grants: grants}, nil +} + +func (db *DB) SetPeerPolicy(policy PeerPolicy) error { + db.mu.Lock() + defer db.mu.Unlock() + + grants, err := json.Marshal(policy.Grants) + if err != nil { + return err + } + result, err := db.conn.Exec(`UPDATE peer_policies SET grants_json = ?, updated_at = ? WHERE peer_name = ?`, grants, time.Now(), policy.Peer) + if err != nil { + return err + } + rows, err := result.RowsAffected() + if err != nil { + return err + } + if rows == 0 { + return sql.ErrNoRows + } + return nil +} + func (db *DB) UpdateLastSeen(name string) error { db.mu.Lock() defer db.mu.Unlock() diff --git a/internal/cluster/db_test.go b/internal/cluster/db_test.go index 3e195d8..0ed845d 100644 --- a/internal/cluster/db_test.go +++ b/internal/cluster/db_test.go @@ -40,6 +40,19 @@ func TestNewDB(t *testing.T) { } } +func TestNewDBRecordsSchemaVersion(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + var version int64 + if err := db.conn.QueryRow(`SELECT MAX(version_id) FROM cluster_schema_version WHERE is_applied = 1`).Scan(&version); err != nil { + t.Fatalf("read schema version: %v", err) + } + if version != 2 { + t.Fatalf("schema version = %d, want 2", version) + } +} + func TestDBPath(t *testing.T) { tmpDir, err := os.MkdirTemp("", "cluster_test") if err != nil { @@ -188,6 +201,55 @@ func TestDeletePeer(t *testing.T) { } } +func TestPeerPolicyDefaultsAndUpdates(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + _, err := db.CreatePeer(&Peer{ + Name: "policy-peer", URL: "https://peer.example.com", + APIKeyHash: "h", APIKeyEncrypted: "e", Status: "active", + }) + if err != nil { + t.Fatalf("CreatePeer failed: %v", err) + } + + policy, err := db.GetPeerPolicy("policy-peer") + if err != nil { + t.Fatalf("GetPeerPolicy failed: %v", err) + } + if len(policy.Grants) != len(DefaultPeerGrants()) { + t.Fatalf("default grants = %#v", policy.Grants) + } + + policy.Grants = []Grant{{Capability: CapabilityCapacityOffer, MaxCPU: 2, MaxMemory: 2 << 30, MaxReplicas: 3}} + if err := db.SetPeerPolicy(*policy); err != nil { + t.Fatalf("SetPeerPolicy failed: %v", err) + } + updated, err := db.GetPeerPolicy("policy-peer") + if err != nil { + t.Fatalf("GetPeerPolicy after update failed: %v", err) + } + if len(updated.Grants) != 1 || updated.Grants[0].MaxReplicas != 3 { + t.Fatalf("updated policy = %#v", updated) + } +} + +func TestDeletePeerDeletesPolicy(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + _, _ = db.CreatePeer(&Peer{ + Name: "policy-delete", URL: "https://peer.example.com", + APIKeyHash: "h", APIKeyEncrypted: "e", Status: "active", + }) + if err := db.DeletePeer("policy-delete"); err != nil { + t.Fatalf("DeletePeer failed: %v", err) + } + if _, err := db.GetPeerPolicy("policy-delete"); err == nil { + t.Fatal("policy should be deleted with its peer") + } +} + func TestUpdateLastSeen(t *testing.T) { db, cleanup := setupTestDB(t) defer cleanup() diff --git a/internal/cluster/migrations/00001_initial.sql b/internal/cluster/migrations/00001_initial.sql new file mode 100644 index 0000000..f72d808 --- /dev/null +++ b/internal/cluster/migrations/00001_initial.sql @@ -0,0 +1,30 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS peers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT UNIQUE NOT NULL, + url TEXT NOT NULL, + api_key_hash TEXT NOT NULL, + api_key_encrypted TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + last_seen_at DATETIME +); + +CREATE TABLE IF NOT EXISTS invites ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + token_hash TEXT UNIQUE NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + created_by INTEGER NOT NULL, + accepted_peer TEXT, + expires_at DATETIME NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_peers_name ON peers(name); +CREATE INDEX IF NOT EXISTS idx_peers_status ON peers(status); +CREATE INDEX IF NOT EXISTS idx_invites_token_hash ON invites(token_hash); +CREATE INDEX IF NOT EXISTS idx_invites_status ON invites(status); + +-- +goose Down +DROP TABLE IF EXISTS invites; +DROP TABLE IF EXISTS peers; diff --git a/internal/cluster/migrations/00002_peer_policies.sql b/internal/cluster/migrations/00002_peer_policies.sql new file mode 100644 index 0000000..bca1e44 --- /dev/null +++ b/internal/cluster/migrations/00002_peer_policies.sql @@ -0,0 +1,9 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS peer_policies ( + peer_name TEXT PRIMARY KEY, + grants_json TEXT NOT NULL, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- +goose Down +DROP TABLE IF EXISTS peer_policies; From 91556c5737bb018cd50c296cd2aec8965f20b73a Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 14:19:11 +0100 Subject: [PATCH 09/46] fix(cluster): Enforce peer access policies --- internal/api/cluster_handlers.go | 141 +++++++++++++++++++++++--- internal/api/cluster_handlers_test.go | 84 ++++++++++++++- internal/auth/models.go | 3 +- 3 files changed, 210 insertions(+), 18 deletions(-) diff --git a/internal/api/cluster_handlers.go b/internal/api/cluster_handlers.go index 51de04d..8e45800 100644 --- a/internal/api/cluster_handlers.go +++ b/internal/api/cluster_handlers.go @@ -10,6 +10,7 @@ import ( "io" "net/http" "net/url" + "slices" "strings" "time" @@ -187,6 +188,15 @@ func (s *Server) updateClusterPeerPolicy(c *gin.Context) { } seen[grant.Capability] = true } + previous, err := mgr.DB().GetPeerPolicy(policy.Peer) + if err != nil { + if err == sql.ErrNoRows { + c.JSON(http.StatusNotFound, gin.H{"error": "Peer not found"}) + return + } + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } if err := mgr.DB().SetPeerPolicy(policy); err != nil { if err == sql.ErrNoRows { c.JSON(http.StatusNotFound, gin.H{"error": "Peer not found"}) @@ -195,6 +205,11 @@ func (s *Server) updateClusterPeerPolicy(c *gin.Context) { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } + if err := s.applyClusterPeerPolicy(policy); err != nil { + _ = mgr.DB().SetPeerPolicy(*previous) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to apply peer policy: " + err.Error()}) + return + } c.JSON(http.StatusOK, policy) } @@ -308,7 +323,11 @@ func (s *Server) clusterAccept(c *gin.Context) { } if s.authManager != nil { - s.createClusterAPIKey(ourAPIKeyForThem, exchangeResp.Name) + if err := s.createClusterAPIKey(ourAPIKeyForThem, exchangeResp.Name); err != nil { + _ = mgr.RemovePeer(exchangeResp.Name) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create peer credential"}) + return + } } c.JSON(http.StatusOK, gin.H{ @@ -382,7 +401,11 @@ func (s *Server) clusterExchange(c *gin.Context) { } if s.authManager != nil { - s.createClusterAPIKey(ourAPIKeyForThem, req.Name) + if err := s.createClusterAPIKey(ourAPIKeyForThem, req.Name); err != nil { + _ = mgr.RemovePeer(req.Name) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create peer credential"}) + return + } } c.JSON(http.StatusOK, exchangeResponse{ @@ -391,28 +414,114 @@ func (s *Server) clusterExchange(c *gin.Context) { }) } -func (s *Server) createClusterAPIKey(rawKey, peerName string) { +func (s *Server) createClusterAPIKey(rawKey, peerName string) error { if s.authManager == nil { - return + return fmt.Errorf("Authentication manager is not available") } - _, _ = s.authManager.CreateAPIKeyFromRaw( + userID, err := s.clusterServiceUserID() + if err != nil { + return err + } + permissions, deployments := clusterPolicyAccess(cluster.PeerPolicy{Peer: peerName, Grants: cluster.DefaultPeerGrants()}) + _, err = s.authManager.CreateAPIKeyFromRaw( rawKey, - 1, + userID, fmt.Sprintf("cluster-peer-%s", peerName), fmt.Sprintf("Auto-generated API key for cluster peer %s", peerName), auth.Role(""), - []string{ - auth.PermClusterRead.String(), - auth.PermDeploymentsRead.String(), - auth.PermDeploymentsWrite.String(), - auth.PermContainersRead.String(), - auth.PermContainersWrite.String(), - auth.PermSystemRead.String(), - auth.PermTrafficRead.String(), - }, - nil, + permissions, + deployments, time.Time{}, ) + return err +} + +func (s *Server) clusterServiceUserID() (int64, error) { + const username = "__flatrun_cluster" + user, err := s.authManager.GetUserByUsername(username) + if err == nil { + if user.Role != auth.RoleService { + return 0, fmt.Errorf("Reserved Fleet identity has an invalid role") + } + return user.ID, nil + } + passwordBytes := make([]byte, 32) + if _, err := rand.Read(passwordBytes); err != nil { + return 0, err + } + user, err = s.authManager.CreateUser(username, "", base64.RawURLEncoding.EncodeToString(passwordBytes), auth.RoleService, nil) + if err != nil { + return 0, err + } + return user.ID, nil +} + +func clusterPolicyAccess(policy cluster.PeerPolicy) ([]string, auth.DeploymentAccess) { + permissions := make(map[string]bool) + deployments := make(auth.DeploymentAccess) + unrestrictedDeployments := false + for _, grant := range policy.Grants { + switch grant.Capability { + case cluster.CapabilityFleetRead: + permissions[auth.PermClusterRead.String()] = true + case cluster.CapabilityDeploymentsRead: + permissions[auth.PermDeploymentsRead.String()] = true + permissions[auth.PermContainersRead.String()] = true + unrestrictedDeployments = mergeClusterDeploymentAccess(deployments, grant.Deployments, auth.AccessLevelRead, unrestrictedDeployments) + case cluster.CapabilityDeploymentsRun: + permissions[auth.PermDeploymentsRead.String()] = true + permissions[auth.PermDeploymentsWrite.String()] = true + permissions[auth.PermContainersRead.String()] = true + permissions[auth.PermContainersWrite.String()] = true + unrestrictedDeployments = mergeClusterDeploymentAccess(deployments, grant.Deployments, auth.AccessLevelWrite, unrestrictedDeployments) + case cluster.CapabilityCapacityRead: + permissions[auth.PermSystemRead.String()] = true + case cluster.CapabilityRoutingManage: + permissions[auth.PermInfrastructureRead.String()] = true + permissions[auth.PermInfrastructureWrite.String()] = true + } + } + result := make([]string, 0, len(permissions)) + for permission := range permissions { + result = append(result, permission) + } + slices.Sort(result) + if unrestrictedDeployments { + deployments = nil + } + return result, deployments +} + +func mergeClusterDeploymentAccess(access auth.DeploymentAccess, names []string, level string, unrestricted bool) bool { + if unrestricted || len(names) == 0 { + return true + } + for _, name := range names { + if current, ok := access[name]; !ok || current == auth.AccessLevelRead && level == auth.AccessLevelWrite { + access[name] = level + } + } + return false +} + +func (s *Server) applyClusterPeerPolicy(policy cluster.PeerPolicy) error { + if s.authManager == nil { + return fmt.Errorf("Authentication manager is not available") + } + keys, err := s.authManager.GetAllAPIKeys() + if err != nil { + return err + } + name := fmt.Sprintf("cluster-peer-%s", policy.Peer) + for _, key := range keys { + if key.Name != name || !key.IsActive { + continue + } + permissions, deployments := clusterPolicyAccess(policy) + _, err := s.authManager.UpdateAPIKey(key.ID, key.Name, key.Description, key.Role, permissions, deployments, key.ExpiresAt) + return err + } + return fmt.Errorf("Active peer credential not found") } func (s *Server) clusterRemovePeer(c *gin.Context) { diff --git a/internal/api/cluster_handlers_test.go b/internal/api/cluster_handlers_test.go index 659c685..9e8f85f 100644 --- a/internal/api/cluster_handlers_test.go +++ b/internal/api/cluster_handlers_test.go @@ -95,6 +95,12 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool protected.Use(authMiddleware.RequireAuth()) { protected.GET("/capacity", authMiddleware.RequirePermission(auth.PermSystemRead), server.getCapacityStatus) + protected.GET("/test/deployments", authMiddleware.RequirePermission(auth.PermDeploymentsRead), func(c *gin.Context) { + c.Status(http.StatusNoContent) + }) + protected.GET("/test/users", authMiddleware.RequirePermission(auth.PermUsersWrite), func(c *gin.Context) { + c.Status(http.StatusNoContent) + }) clusterGroup := protected.Group("/cluster") clusterGroup.Use(authMiddleware.RequirePermission(auth.PermClusterRead)) { @@ -168,6 +174,9 @@ func TestUpdateClusterPeerPolicyThroughHTTP(t *testing.T) { if err := env.server.clusterManager.AddPeer("server-b", "https://server-b.example.com", "peer-key"); err != nil { t.Fatalf("AddPeer failed: %v", err) } + if err := env.server.createClusterAPIKey("server-b-inbound-key", "server-b"); err != nil { + t.Fatalf("createClusterAPIKey failed: %v", err) + } token := clusterLogin(t, env.router) body := []byte(`{"grants":[{"capability":"capacity.offer","max_cpu":2,"max_memory":2147483648,"max_replicas":2}]}`) @@ -195,6 +204,40 @@ func TestUpdateClusterPeerPolicyThroughHTTP(t *testing.T) { if len(policy.Grants) != 1 || policy.Grants[0].MaxCPU != 2 || policy.Grants[0].MaxReplicas != 2 { t.Fatalf("policy = %#v", policy) } + keys, err := env.server.authManager.GetAllAPIKeys() + if err != nil { + t.Fatalf("list API keys: %v", err) + } + for _, key := range keys { + if key.Name == "cluster-peer-server-b" && len(key.Permissions) != 0 { + t.Fatalf("capacity offer unexpectedly granted general API permissions: %#v", key.Permissions) + } + } +} + +func TestClusterPolicyAccessScopesDeployments(t *testing.T) { + permissions, deployments := clusterPolicyAccess(cluster.PeerPolicy{Grants: []cluster.Grant{ + {Capability: cluster.CapabilityDeploymentsRead, Deployments: []string{"public-site", "docs"}}, + {Capability: cluster.CapabilityDeploymentsRun, Deployments: []string{"public-site"}}, + {Capability: cluster.CapabilityCapacityRead}, + }}) + + if deployments["public-site"] != auth.AccessLevelWrite || deployments["docs"] != auth.AccessLevelRead { + t.Fatalf("deployment access = %#v", deployments) + } + wanted := map[string]bool{ + auth.PermDeploymentsRead.String(): true, + auth.PermDeploymentsWrite.String(): true, + auth.PermContainersRead.String(): true, + auth.PermContainersWrite.String(): true, + auth.PermSystemRead.String(): true, + } + for _, permission := range permissions { + delete(wanted, permission) + } + if len(wanted) != 0 { + t.Fatalf("missing permissions = %#v", wanted) + } } func TestClusterSetupEnablesClusterWithoutRestart(t *testing.T) { @@ -260,7 +303,9 @@ func TestClusterAPIKeyUsesExplicitPermissions(t *testing.T) { env := setupClusterTestServer(t, "server-a", true) defer env.cleanup() - env.server.createClusterAPIKey("peer-key-for-test", "server-b") + if err := env.server.createClusterAPIKey("peer-key-for-test", "server-b"); err != nil { + t.Fatalf("createClusterAPIKey failed: %v", err) + } keys, err := env.server.authManager.GetAllAPIKeys() if err != nil { t.Fatal(err) @@ -278,6 +323,20 @@ func TestClusterAPIKeyUsesExplicitPermissions(t *testing.T) { if peerKey.Role == auth.RoleAdmin { t.Fatal("Cluster API key has administrator role") } + owner, err := env.server.authManager.GetUser(peerKey.UserID) + if err != nil { + t.Fatalf("get key owner: %v", err) + } + if owner.Role != auth.RoleService { + t.Fatalf("key owner role = %q", owner.Role) + } + actor, err := env.server.authManager.BuildActorContext(owner, peerKey) + if err != nil { + t.Fatalf("build actor: %v", err) + } + if actor.HasPermission(auth.PermUsersWrite) || !actor.HasPermission(auth.PermDeploymentsWrite) { + t.Fatalf("effective actor = %#v", actor) + } permissions := make(map[string]bool, len(peerKey.Permissions)) for _, permission := range peerKey.Permissions { permissions[permission] = true @@ -290,6 +349,29 @@ func TestClusterAPIKeyUsesExplicitPermissions(t *testing.T) { } } +func TestClusterAPIKeyEnforcesPermissionsThroughHTTP(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + const rawKey = "peer-http-key-for-test" + if err := env.server.createClusterAPIKey(rawKey, "server-b"); err != nil { + t.Fatalf("createClusterAPIKey failed: %v", err) + } + + request := func(path string) int { + req := httptest.NewRequest(http.MethodGet, path, nil) + req.Header.Set("Authorization", "Bearer "+rawKey) + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + return w.Code + } + if status := request("/api/test/deployments"); status != http.StatusNoContent { + t.Fatalf("deployment read status = %d", status) + } + if status := request("/api/test/users"); status != http.StatusForbidden { + t.Fatalf("user write status = %d", status) + } +} + func clusterLogin(t *testing.T, router *gin.Engine) string { t.Helper() body, _ := json.Marshal(map[string]string{ diff --git a/internal/auth/models.go b/internal/auth/models.go index 7ae4be4..ee83119 100644 --- a/internal/auth/models.go +++ b/internal/auth/models.go @@ -13,6 +13,7 @@ const ( RoleAdmin Role = "admin" RoleOperator Role = "operator" RoleViewer Role = "viewer" + RoleService Role = "service" ) const ( @@ -27,7 +28,7 @@ func ValidAccessLevel(level string) bool { func (r Role) IsValid() bool { switch r { - case RoleAdmin, RoleOperator, RoleViewer: + case RoleAdmin, RoleOperator, RoleViewer, RoleService: return true } return false From 1bdd1890457dccac6e41ff5a7be7f67a10f4beed Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 14:23:14 +0100 Subject: [PATCH 10/46] refactor(notifications): Version incident storage --- internal/events/migrations/00001_initial.sql | 20 +++++++++ internal/events/store.go | 36 +++++++++------- internal/events/store_test.go | 44 ++++++++++++++++++++ 3 files changed, 85 insertions(+), 15 deletions(-) create mode 100644 internal/events/migrations/00001_initial.sql create mode 100644 internal/events/store_test.go diff --git a/internal/events/migrations/00001_initial.sql b/internal/events/migrations/00001_initial.sql new file mode 100644 index 0000000..2704123 --- /dev/null +++ b/internal/events/migrations/00001_initial.sql @@ -0,0 +1,20 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS incidents ( + correlation_key TEXT PRIMARY KEY, + payload BLOB NOT NULL, + last_event_at DATETIME NOT NULL +); + +CREATE TABLE IF NOT EXISTS events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + incident_id TEXT NOT NULL, + payload BLOB NOT NULL, + occurred_at DATETIME NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_events_incident_id ON events(incident_id); +CREATE INDEX IF NOT EXISTS idx_events_occurred_at ON events(occurred_at); + +-- +goose Down +DROP TABLE IF EXISTS events; +DROP TABLE IF EXISTS incidents; diff --git a/internal/events/store.go b/internal/events/store.go index 4b314bf..df6f9f8 100644 --- a/internal/events/store.go +++ b/internal/events/store.go @@ -1,15 +1,22 @@ package events import ( + "context" "database/sql" + "embed" "encoding/json" "fmt" + "io/fs" "os" "path/filepath" + "github.com/pressly/goose/v3" _ "modernc.org/sqlite" ) +//go:embed migrations/*.sql +var migrationFiles embed.FS + type Store struct { db *sql.DB } @@ -32,21 +39,20 @@ func NewStore(basePath string) (*Store, error) { } func (s *Store) migrate() error { - _, err := s.db.Exec(` - CREATE TABLE IF NOT EXISTS incidents ( - correlation_key TEXT PRIMARY KEY, - payload BLOB NOT NULL, - last_event_at DATETIME NOT NULL - ); - CREATE TABLE IF NOT EXISTS events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - incident_id TEXT NOT NULL, - payload BLOB NOT NULL, - occurred_at DATETIME NOT NULL - ); - CREATE INDEX IF NOT EXISTS idx_events_incident_id ON events(incident_id); - CREATE INDEX IF NOT EXISTS idx_events_occurred_at ON events(occurred_at); - `) + migrations, err := fs.Sub(migrationFiles, "migrations") + if err != nil { + return err + } + provider, err := goose.NewProvider( + goose.DialectSQLite3, + s.db, + migrations, + goose.WithTableName("events_schema_version"), + ) + if err != nil { + return err + } + _, err = provider.Up(context.Background()) return err } diff --git a/internal/events/store_test.go b/internal/events/store_test.go new file mode 100644 index 0000000..26280f5 --- /dev/null +++ b/internal/events/store_test.go @@ -0,0 +1,44 @@ +package events + +import ( + "testing" + "time" +) + +func TestStoreRecordsMigrationVersionAndPreservesIncidents(t *testing.T) { + dir := t.TempDir() + store, err := NewStore(dir) + if err != nil { + t.Fatalf("NewStore failed: %v", err) + } + var version int64 + if err := store.db.QueryRow(`SELECT MAX(version_id) FROM events_schema_version WHERE is_applied = 1`).Scan(&version); err != nil { + t.Fatalf("read schema version: %v", err) + } + if version != 1 { + t.Fatalf("schema version = %d, want 1", version) + } + + now := time.Now().UTC() + event := Event{ID: "event-1", Source: "capacity", Type: "host.pressure", Severity: SeverityWarning, OccurredAt: now} + incident := Incident{ID: "incident-1", CorrelationKey: "node:prod-1", Status: IncidentOpen, Severity: SeverityWarning, LastEventAt: now} + if err := store.Record(event, incident); err != nil { + t.Fatalf("Record failed: %v", err) + } + if err := store.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } + + reopened, err := NewStore(dir) + if err != nil { + t.Fatalf("reopen store: %v", err) + } + defer reopened.Close() + incidents, err := reopened.ListIncidents() + if err != nil { + t.Fatalf("ListIncidents failed: %v", err) + } + if len(incidents) != 1 || incidents[0].ID != "incident-1" { + t.Fatalf("incidents = %#v", incidents) + } +} From b01a9b611c56254b9588899bf8f2f03e5d6e35fb Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 14:26:29 +0100 Subject: [PATCH 11/46] feat(cluster): Add Swarm orchestration adapter --- internal/orchestrator/swarm.go | 185 ++++++++++++++++++++++++++++ internal/orchestrator/swarm_test.go | 99 +++++++++++++++ 2 files changed, 284 insertions(+) create mode 100644 internal/orchestrator/swarm.go create mode 100644 internal/orchestrator/swarm_test.go diff --git a/internal/orchestrator/swarm.go b/internal/orchestrator/swarm.go new file mode 100644 index 0000000..7f3a297 --- /dev/null +++ b/internal/orchestrator/swarm.go @@ -0,0 +1,185 @@ +package orchestrator + +import ( + "context" + "fmt" + "strings" + + "github.com/containerd/errdefs" + "github.com/moby/moby/api/types/swarm" + "github.com/moby/moby/client" +) + +type swarmClient interface { + SwarmInspect(context.Context, client.SwarmInspectOptions) (client.SwarmInspectResult, error) + ServiceCreate(context.Context, client.ServiceCreateOptions) (client.ServiceCreateResult, error) + ServiceInspect(context.Context, string, client.ServiceInspectOptions) (client.ServiceInspectResult, error) + ServiceUpdate(context.Context, string, client.ServiceUpdateOptions) (client.ServiceUpdateResult, error) + ServiceRemove(context.Context, string, client.ServiceRemoveOptions) (client.ServiceRemoveResult, error) + TaskList(context.Context, client.TaskListOptions) (client.TaskListResult, error) +} + +type SwarmProvider struct { + client swarmClient +} + +func NewSwarmProvider(client swarmClient) *SwarmProvider { + return &SwarmProvider{client: client} +} + +func NewSwarmProviderFromEnv() (*SwarmProvider, error) { + cli, err := client.New(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return nil, err + } + return NewSwarmProvider(cli), nil +} + +func (p *SwarmProvider) Ready(ctx context.Context) error { + if _, err := p.client.SwarmInspect(ctx, client.SwarmInspectOptions{}); err != nil { + return fmt.Errorf("Docker Swarm is not available: %w", err) + } + return nil +} + +func (p *SwarmProvider) ID() ProviderID { + return ProviderSwarm +} + +func (p *SwarmProvider) Validate(_ context.Context, workload Workload) error { + if strings.TrimSpace(workload.ID) == "" { + return fmt.Errorf("Workload ID is required") + } + if strings.TrimSpace(workload.Image) == "" { + return fmt.Errorf("Workload image is required") + } + if workload.Replicas < 1 { + return fmt.Errorf("Replicas must be at least one") + } + if workload.Stateful && workload.Replicas > 1 { + return fmt.Errorf("Stateful workloads cannot use multiple replicas without a storage policy") + } + return nil +} + +func (p *SwarmProvider) Apply(ctx context.Context, workload Workload) (Status, error) { + if err := p.Validate(ctx, workload); err != nil { + return Status{}, err + } + inspected, err := p.client.ServiceInspect(ctx, workload.ID, client.ServiceInspectOptions{}) + if err != nil { + if !errdefs.IsNotFound(err) { + return Status{}, fmt.Errorf("inspect Swarm service: %w", err) + } + if _, err := p.client.ServiceCreate(ctx, client.ServiceCreateOptions{Spec: swarmSpec(workload)}); err != nil { + return Status{}, fmt.Errorf("create Swarm service: %w", err) + } + return p.Status(ctx, workload.ID) + } + if _, err := p.client.ServiceUpdate(ctx, inspected.Service.ID, client.ServiceUpdateOptions{ + Version: inspected.Service.Version, + Spec: swarmSpec(workload), + }); err != nil { + return Status{}, fmt.Errorf("update Swarm service: %w", err) + } + return p.Status(ctx, workload.ID) +} + +func (p *SwarmProvider) Resize(ctx context.Context, id string, resources Resources) (Status, error) { + service, err := p.client.ServiceInspect(ctx, id, client.ServiceInspectOptions{}) + if err != nil { + return Status{}, err + } + service.Service.Spec.TaskTemplate.Resources = swarmResources(resources) + if _, err := p.client.ServiceUpdate(ctx, service.Service.ID, client.ServiceUpdateOptions{ + Version: service.Service.Version, + Spec: service.Service.Spec, + }); err != nil { + return Status{}, fmt.Errorf("resize Swarm service: %w", err) + } + return p.Status(ctx, id) +} + +func (p *SwarmProvider) Scale(ctx context.Context, id string, replicas int) (Status, error) { + if replicas < 0 { + return Status{}, fmt.Errorf("Replicas cannot be negative") + } + service, err := p.client.ServiceInspect(ctx, id, client.ServiceInspectOptions{}) + if err != nil { + return Status{}, err + } + count := uint64(replicas) + service.Service.Spec.Mode = swarm.ServiceMode{Replicated: &swarm.ReplicatedService{Replicas: &count}} + if _, err := p.client.ServiceUpdate(ctx, service.Service.ID, client.ServiceUpdateOptions{ + Version: service.Service.Version, + Spec: service.Service.Spec, + }); err != nil { + return Status{}, fmt.Errorf("scale Swarm service: %w", err) + } + return p.Status(ctx, id) +} + +func (p *SwarmProvider) Status(ctx context.Context, id string) (Status, error) { + service, err := p.client.ServiceInspect(ctx, id, client.ServiceInspectOptions{}) + if err != nil { + return Status{}, err + } + tasks, err := p.client.TaskList(ctx, client.TaskListOptions{Filters: make(client.Filters).Add("service", service.Service.ID)}) + if err != nil { + return Status{}, fmt.Errorf("list Swarm tasks: %w", err) + } + status := Status{Workload: id} + if service.Service.Spec.Mode.Replicated != nil && service.Service.Spec.Mode.Replicated.Replicas != nil { + status.Desired = int(*service.Service.Spec.Mode.Replicated.Replicas) + } + for _, task := range tasks.Items { + if task.DesiredState == swarm.TaskStateShutdown || task.DesiredState == swarm.TaskStateRemove { + continue + } + running := task.Status.State == swarm.TaskStateRunning + if running { + status.Available++ + } + status.Instances = append(status.Instances, Instance{ + ID: task.ID, Node: task.NodeID, Healthy: running, Ready: running, + }) + } + return status, nil +} + +func (p *SwarmProvider) Remove(ctx context.Context, id string) error { + if _, err := p.client.ServiceRemove(ctx, id, client.ServiceRemoveOptions{}); err != nil && !errdefs.IsNotFound(err) { + return fmt.Errorf("remove Swarm service: %w", err) + } + return nil +} + +func swarmSpec(workload Workload) swarm.ServiceSpec { + replicas := uint64(workload.Replicas) + labels := make(map[string]string, len(workload.Labels)+1) + for key, value := range workload.Labels { + labels[key] = value + } + labels["flatrun.workload"] = workload.ID + return swarm.ServiceSpec{ + Annotations: swarm.Annotations{Name: workload.ID, Labels: labels}, + TaskTemplate: swarm.TaskSpec{ + ContainerSpec: &swarm.ContainerSpec{Image: workload.Image, Labels: labels}, + Resources: swarmResources(workload.Resources), + }, + Mode: swarm.ServiceMode{Replicated: &swarm.ReplicatedService{Replicas: &replicas}}, + } +} + +func swarmResources(resources Resources) *swarm.ResourceRequirements { + return &swarm.ResourceRequirements{ + Limits: &swarm.Limit{ + NanoCPUs: int64(resources.CPULimit * 1_000_000_000), + MemoryBytes: int64(resources.MemoryLimit), + }, + Reservations: &swarm.Resources{ + NanoCPUs: int64(resources.CPURequest * 1_000_000_000), + MemoryBytes: int64(resources.MemoryRequest), + }, + } +} diff --git a/internal/orchestrator/swarm_test.go b/internal/orchestrator/swarm_test.go new file mode 100644 index 0000000..7364110 --- /dev/null +++ b/internal/orchestrator/swarm_test.go @@ -0,0 +1,99 @@ +package orchestrator + +import ( + "context" + "testing" + + "github.com/containerd/errdefs" + "github.com/moby/moby/api/types/swarm" + "github.com/moby/moby/client" +) + +type fakeSwarmClient struct { + service swarm.Service + tasks []swarm.Task + created *swarm.ServiceSpec + updated *swarm.ServiceSpec +} + +func (f *fakeSwarmClient) SwarmInspect(_ context.Context, _ client.SwarmInspectOptions) (client.SwarmInspectResult, error) { + return client.SwarmInspectResult{Swarm: swarm.Swarm{ClusterInfo: swarm.ClusterInfo{ID: "swarm-1"}}}, nil +} + +func (f *fakeSwarmClient) ServiceCreate(_ context.Context, options client.ServiceCreateOptions) (client.ServiceCreateResult, error) { + f.created = &options.Spec + f.service = swarm.Service{ID: "service-1", Spec: options.Spec} + return client.ServiceCreateResult{ID: f.service.ID}, nil +} + +func (f *fakeSwarmClient) ServiceInspect(_ context.Context, _ string, _ client.ServiceInspectOptions) (client.ServiceInspectResult, error) { + if f.service.ID == "" { + return client.ServiceInspectResult{}, errdefs.ErrNotFound + } + return client.ServiceInspectResult{Service: f.service}, nil +} + +func (f *fakeSwarmClient) ServiceUpdate(_ context.Context, _ string, options client.ServiceUpdateOptions) (client.ServiceUpdateResult, error) { + f.updated = &options.Spec + f.service.Spec = options.Spec + return client.ServiceUpdateResult{}, nil +} + +func (f *fakeSwarmClient) ServiceRemove(_ context.Context, _ string, _ client.ServiceRemoveOptions) (client.ServiceRemoveResult, error) { + f.service = swarm.Service{} + return client.ServiceRemoveResult{}, nil +} + +func (f *fakeSwarmClient) TaskList(_ context.Context, _ client.TaskListOptions) (client.TaskListResult, error) { + return client.TaskListResult{Items: f.tasks}, nil +} + +func TestSwarmProviderCreatesReplicatedService(t *testing.T) { + client := &fakeSwarmClient{tasks: []swarm.Task{{ID: "task-1", NodeID: "node-1", DesiredState: swarm.TaskStateRunning, Status: swarm.TaskStatus{State: swarm.TaskStateRunning}}}} + provider := NewSwarmProvider(client) + + status, err := provider.Apply(context.Background(), Workload{ + ID: "shop", Image: "example/shop:1", Replicas: 1, + Resources: Resources{CPURequest: 0.5, CPULimit: 1, MemoryRequest: 256 << 20, MemoryLimit: 512 << 20}, + }) + if err != nil { + t.Fatalf("Apply failed: %v", err) + } + if client.created == nil || client.created.Annotations.Name != "shop" { + t.Fatalf("created spec = %#v", client.created) + } + if client.created.TaskTemplate.Resources.Limits.NanoCPUs != 1_000_000_000 { + t.Fatalf("CPU limit = %d", client.created.TaskTemplate.Resources.Limits.NanoCPUs) + } + if status.Desired != 1 || status.Available != 1 || len(status.Instances) != 1 { + t.Fatalf("status = %#v", status) + } +} + +func TestSwarmProviderScalesExistingService(t *testing.T) { + replicas := uint64(1) + client := &fakeSwarmClient{service: swarm.Service{ + ID: "service-1", + Spec: swarm.ServiceSpec{Mode: swarm.ServiceMode{Replicated: &swarm.ReplicatedService{Replicas: &replicas}}}, + }} + provider := NewSwarmProvider(client) + + status, err := provider.Scale(context.Background(), "shop", 3) + if err != nil { + t.Fatalf("Scale failed: %v", err) + } + if client.updated == nil || *client.updated.Mode.Replicated.Replicas != 3 { + t.Fatalf("updated spec = %#v", client.updated) + } + if status.Desired != 3 { + t.Fatalf("status = %#v", status) + } +} + +func TestSwarmProviderRejectsUnsafeStatefulReplication(t *testing.T) { + provider := NewSwarmProvider(&fakeSwarmClient{}) + err := provider.Validate(context.Background(), Workload{ID: "database", Image: "postgres:17", Replicas: 2, Stateful: true}) + if err == nil { + t.Fatal("stateful replication should require a storage policy") + } +} From 9f5de7f69a423357887015ff79c23eca17ace072 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 14:28:23 +0100 Subject: [PATCH 12/46] feat(cluster): Add routing provider adapters --- internal/routing/adapters.go | 178 ++++++++++++++++++++++++++++++ internal/routing/adapters_test.go | 83 ++++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 internal/routing/adapters.go create mode 100644 internal/routing/adapters_test.go diff --git a/internal/routing/adapters.go b/internal/routing/adapters.go new file mode 100644 index 0000000..e522dd2 --- /dev/null +++ b/internal/routing/adapters.go @@ -0,0 +1,178 @@ +package routing + +import ( + "context" + "fmt" + "net" + "regexp" + "sort" + "strconv" + "strings" + "sync" + + "gopkg.in/yaml.v3" +) + +type ConfigWriter interface { + Apply(context.Context, string, string, []byte) error + Remove(context.Context, string) error +} + +type routeProvider struct { + id ProviderID + writer ConfigWriter + render func(Route) ([]byte, error) + mu sync.RWMutex + routes map[string]Route +} + +func NewNginxProvider(writer ConfigWriter) Provider { + return &routeProvider{id: ProviderNginx, writer: writer, render: renderNginx, routes: make(map[string]Route)} +} + +func NewTraefikProvider(writer ConfigWriter) Provider { + return &routeProvider{id: ProviderTraefik, writer: writer, render: renderTraefik, routes: make(map[string]Route)} +} + +func (p *routeProvider) ID() ProviderID { + return p.id +} + +func (p *routeProvider) Validate(_ context.Context, route Route) error { + if strings.TrimSpace(route.ID) == "" { + return fmt.Errorf("Route ID is required") + } + if strings.TrimSpace(route.Domain) == "" || strings.ContainsAny(route.Domain, " /\\") { + return fmt.Errorf("Route domain is invalid") + } + if route.Protocol != "http" && route.Protocol != "https" { + return fmt.Errorf("Route protocol must be http or https") + } + if route.Path != "" && !strings.HasPrefix(route.Path, "/") { + return fmt.Errorf("Route path must start with a slash") + } + if len(route.Backends) == 0 { + return fmt.Errorf("Route needs at least one backend") + } + for _, backend := range route.Backends { + if strings.TrimSpace(backend.ID) == "" { + return fmt.Errorf("Backend ID is required") + } + host, portValue, err := net.SplitHostPort(backend.Address) + if err != nil { + return fmt.Errorf("Backend %q address is invalid: %w", backend.ID, err) + } + if net.ParseIP(host) == nil && !safeHost.MatchString(host) { + return fmt.Errorf("Backend %q host is invalid", backend.ID) + } + port, err := strconv.Atoi(portValue) + if err != nil || port < 1 || port > 65535 { + return fmt.Errorf("Backend %q port is invalid", backend.ID) + } + if backend.Weight < 0 { + return fmt.Errorf("Backend %q weight cannot be negative", backend.ID) + } + } + return nil +} + +func (p *routeProvider) Reconcile(ctx context.Context, route Route) error { + if err := p.Validate(ctx, route); err != nil { + return err + } + content, err := p.render(route) + if err != nil { + return err + } + if err := p.writer.Apply(ctx, route.ID, string(p.id), content); err != nil { + return err + } + p.mu.Lock() + p.routes[route.ID] = route + p.mu.Unlock() + return nil +} + +func (p *routeProvider) Drain(ctx context.Context, routeID, backendID string) error { + p.mu.RLock() + route, ok := p.routes[routeID] + p.mu.RUnlock() + if !ok { + return fmt.Errorf("Route %q is not managed", routeID) + } + found := false + for index := range route.Backends { + if route.Backends[index].ID == backendID { + route.Backends[index].Healthy = false + found = true + } + } + if !found { + return fmt.Errorf("Backend %q is not part of route %q", backendID, routeID) + } + return p.Reconcile(ctx, route) +} + +func (p *routeProvider) Remove(ctx context.Context, routeID string) error { + if err := p.writer.Remove(ctx, routeID); err != nil { + return err + } + p.mu.Lock() + delete(p.routes, routeID) + p.mu.Unlock() + return nil +} + +var safeID = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`) +var safeHost = regexp.MustCompile(`^[a-zA-Z0-9.-]+$`) + +func renderNginx(route Route) ([]byte, error) { + name := "flatrun_" + safeID.ReplaceAllString(route.ID, "_") + path := route.Path + if path == "" { + path = "/" + } + backends := append([]Backend(nil), route.Backends...) + sort.Slice(backends, func(i, j int) bool { return backends[i].ID < backends[j].ID }) + var output strings.Builder + fmt.Fprintf(&output, "upstream %s {\n", name) + for _, backend := range backends { + fmt.Fprintf(&output, " server %s", backend.Address) + if backend.Weight > 0 { + fmt.Fprintf(&output, " weight=%d", backend.Weight) + } + if !backend.Healthy { + output.WriteString(" down") + } + output.WriteString(";\n") + } + output.WriteString("}\n\nserver {\n listen 80;\n") + fmt.Fprintf(&output, " server_name %s;\n\n", route.Domain) + fmt.Fprintf(&output, " location %s {\n", path) + fmt.Fprintf(&output, " proxy_pass %s://%s;\n", route.Protocol, name) + output.WriteString(" proxy_set_header Host $host;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n }\n}\n") + return []byte(output.String()), nil +} + +func renderTraefik(route Route) ([]byte, error) { + path := route.Path + if path == "" { + path = "/" + } + servers := make([]map[string]string, 0, len(route.Backends)) + for _, backend := range route.Backends { + if backend.Healthy { + servers = append(servers, map[string]string{"url": route.Protocol + "://" + backend.Address}) + } + } + config := map[string]any{"http": map[string]any{ + "routers": map[string]any{route.ID: map[string]any{ + "rule": fmt.Sprintf("Host(`%s`) && PathPrefix(`%s`)", route.Domain, path), + "service": route.ID, + }}, + "services": map[string]any{route.ID: map[string]any{ + "loadBalancer": map[string]any{"servers": servers}, + }}, + }} + return yaml.Marshal(config) +} diff --git a/internal/routing/adapters_test.go b/internal/routing/adapters_test.go new file mode 100644 index 0000000..88e1fdf --- /dev/null +++ b/internal/routing/adapters_test.go @@ -0,0 +1,83 @@ +package routing + +import ( + "context" + "strings" + "testing" +) + +type memoryConfigWriter struct { + format string + content string + removed string +} + +func (w *memoryConfigWriter) Apply(_ context.Context, _ string, format string, content []byte) error { + w.format = format + w.content = string(content) + return nil +} + +func (w *memoryConfigWriter) Remove(_ context.Context, id string) error { + w.removed = id + return nil +} + +func testRoute() Route { + return Route{ + ID: "shop", Domain: "shop.example.com", Path: "/", Protocol: "http", + Backends: []Backend{ + {ID: "replica-b", Address: "10.0.0.12:8080", Healthy: true, Weight: 1}, + {ID: "replica-a", Address: "10.0.0.11:8080", Healthy: true, Weight: 2}, + }, + } +} + +func TestNginxProviderRendersWeightedUpstreamAndDrainsBackend(t *testing.T) { + writer := &memoryConfigWriter{} + provider := NewNginxProvider(writer) + if err := provider.Reconcile(context.Background(), testRoute()); err != nil { + t.Fatalf("Reconcile failed: %v", err) + } + for _, expected := range []string{ + "upstream flatrun_shop", + "server 10.0.0.11:8080 weight=2;", + "server 10.0.0.12:8080 weight=1;", + "proxy_pass http://flatrun_shop;", + } { + if !strings.Contains(writer.content, expected) { + t.Fatalf("missing %q in:\n%s", expected, writer.content) + } + } + if err := provider.Drain(context.Background(), "shop", "replica-b"); err != nil { + t.Fatalf("Drain failed: %v", err) + } + if !strings.Contains(writer.content, "server 10.0.0.12:8080 weight=1 down;") { + t.Fatalf("drained backend remains active:\n%s", writer.content) + } +} + +func TestTraefikProviderExcludesDrainedBackend(t *testing.T) { + writer := &memoryConfigWriter{} + provider := NewTraefikProvider(writer) + if err := provider.Reconcile(context.Background(), testRoute()); err != nil { + t.Fatalf("Reconcile failed: %v", err) + } + if writer.format != "traefik" || !strings.Contains(writer.content, "Host(`shop.example.com`) && PathPrefix(`/`)") { + t.Fatalf("Traefik config:\n%s", writer.content) + } + if err := provider.Drain(context.Background(), "shop", "replica-a"); err != nil { + t.Fatalf("Drain failed: %v", err) + } + if strings.Contains(writer.content, "10.0.0.11:8080") || !strings.Contains(writer.content, "10.0.0.12:8080") { + t.Fatalf("Traefik drain config:\n%s", writer.content) + } +} + +func TestRoutingProvidersRejectUnsafeAddresses(t *testing.T) { + route := testRoute() + route.Backends[0].Address = "10.0.0.12:8080; include /etc/passwd" + if err := NewNginxProvider(&memoryConfigWriter{}).Validate(context.Background(), route); err == nil { + t.Fatal("invalid backend address should be rejected") + } +} From f36028c43dbb3b927b76d9d1694e1f33312dc3c2 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 14:29:45 +0100 Subject: [PATCH 13/46] feat(capacity): Add autoscaling policy engine --- internal/autoscale/controller.go | 138 ++++++++++++++++++++++++++ internal/autoscale/controller_test.go | 74 ++++++++++++++ 2 files changed, 212 insertions(+) create mode 100644 internal/autoscale/controller.go create mode 100644 internal/autoscale/controller_test.go diff --git a/internal/autoscale/controller.go b/internal/autoscale/controller.go new file mode 100644 index 0000000..8a18652 --- /dev/null +++ b/internal/autoscale/controller.go @@ -0,0 +1,138 @@ +package autoscale + +import ( + "fmt" + "math" + "time" + + "github.com/flatrun/agent/internal/capacity" + "github.com/flatrun/agent/internal/orchestrator" +) + +type Action string + +const ( + ActionNone Action = "none" + ActionIncreaseCPU Action = "increase_cpu" + ActionIncreaseMemory Action = "increase_memory" + ActionAddReplica Action = "add_replica" + ActionRemoveReplica Action = "remove_replica" + ActionNotify Action = "notify" +) + +type Policy struct { + Enabled bool `json:"enabled"` + MinReplicas int `json:"min_replicas"` + MaxReplicas int `json:"max_replicas"` + ScaleUpPercent float64 `json:"scale_up_percent"` + ScaleDownPercent float64 `json:"scale_down_percent"` + ScaleUpWindows int `json:"scale_up_windows"` + ScaleDownWindows int `json:"scale_down_windows"` + Cooldown time.Duration `json:"cooldown"` + AllowFleetCapacity bool `json:"allow_fleet_capacity"` +} + +type State struct { + HighWindows int `json:"high_windows"` + LowWindows int `json:"low_windows"` + LastAction time.Time `json:"last_action,omitempty"` +} + +type Input struct { + Now time.Time + Replicas int + CPUPercent float64 + MemoryPercent float64 + Diagnosis capacity.Diagnosis + FleetOffer capacity.Offer + RequiresFleet bool + CurrentResources orchestrator.Resources + SuggestedResource orchestrator.Resources +} + +type Decision struct { + Action Action `json:"action"` + Replicas int `json:"replicas,omitempty"` + Resources orchestrator.Resources `json:"resources,omitempty"` + Reason string `json:"reason"` +} + +func DefaultPolicy() Policy { + return Policy{ + Enabled: true, MinReplicas: 1, MaxReplicas: 3, + ScaleUpPercent: 80, ScaleDownPercent: 30, + ScaleUpWindows: 3, ScaleDownWindows: 10, + Cooldown: 5 * time.Minute, + } +} + +func Reconcile(policy Policy, state State, input Input) (State, Decision) { + if err := validatePolicy(policy); err != nil { + return state, Decision{Action: ActionNotify, Reason: err.Error()} + } + if !policy.Enabled { + return State{}, Decision{Action: ActionNone, Reason: "Autoscaling is disabled"} + } + if input.Now.IsZero() { + input.Now = time.Now() + } + if input.Replicas < policy.MinReplicas { + return acted(state, input.Now), Decision{Action: ActionAddReplica, Replicas: policy.MinReplicas, Reason: "Replica count is below policy minimum"} + } + if !state.LastAction.IsZero() && input.Now.Sub(state.LastAction) < policy.Cooldown { + return state, Decision{Action: ActionNone, Reason: "Autoscaling is in cooldown"} + } + + switch input.Diagnosis.Action { + case capacity.ActionIncreaseCPU: + return acted(state, input.Now), Decision{Action: ActionIncreaseCPU, Resources: input.SuggestedResource, Reason: input.Diagnosis.Reason} + case capacity.ActionIncreaseMemory: + return acted(state, input.Now), Decision{Action: ActionIncreaseMemory, Resources: input.SuggestedResource, Reason: input.Diagnosis.Reason} + } + + utilization := math.Max(input.CPUPercent, input.MemoryPercent) + if utilization >= policy.ScaleUpPercent { + state.HighWindows++ + state.LowWindows = 0 + } else if utilization <= policy.ScaleDownPercent { + state.LowWindows++ + state.HighWindows = 0 + } else { + state.HighWindows = 0 + state.LowWindows = 0 + } + + if state.HighWindows >= policy.ScaleUpWindows { + if input.Replicas >= policy.MaxReplicas { + return state, Decision{Action: ActionNotify, Reason: "Workload has reached its replica limit"} + } + if input.RequiresFleet && (!policy.AllowFleetCapacity || !input.FleetOffer.Enabled) { + return state, Decision{Action: ActionNotify, Reason: "No permitted fleet capacity is available"} + } + return acted(state, input.Now), Decision{Action: ActionAddReplica, Replicas: input.Replicas + 1, Reason: "Resource pressure remained above the scale-up threshold"} + } + if state.LowWindows >= policy.ScaleDownWindows && input.Replicas > policy.MinReplicas { + return acted(state, input.Now), Decision{Action: ActionRemoveReplica, Replicas: input.Replicas - 1, Reason: "Resource use remained below the scale-down threshold"} + } + return state, Decision{Action: ActionNone, Reason: "No scaling threshold has been sustained"} +} + +func validatePolicy(policy Policy) error { + if policy.MinReplicas < 1 || policy.MaxReplicas < policy.MinReplicas { + return fmt.Errorf("Replica limits are invalid") + } + if policy.ScaleUpPercent <= policy.ScaleDownPercent || policy.ScaleUpPercent > 100 || policy.ScaleDownPercent < 0 { + return fmt.Errorf("Scaling thresholds are invalid") + } + if policy.ScaleUpWindows < 1 || policy.ScaleDownWindows < 1 || policy.Cooldown < 0 { + return fmt.Errorf("Scaling timing is invalid") + } + return nil +} + +func acted(state State, now time.Time) State { + state.HighWindows = 0 + state.LowWindows = 0 + state.LastAction = now + return state +} diff --git a/internal/autoscale/controller_test.go b/internal/autoscale/controller_test.go new file mode 100644 index 0000000..7bb202e --- /dev/null +++ b/internal/autoscale/controller_test.go @@ -0,0 +1,74 @@ +package autoscale + +import ( + "testing" + "time" + + "github.com/flatrun/agent/internal/capacity" + "github.com/flatrun/agent/internal/orchestrator" +) + +func TestReconcileIncreasesLocalAllocationBeforeReplicas(t *testing.T) { + policy := DefaultPolicy() + now := time.Now() + state, decision := Reconcile(policy, State{HighWindows: policy.ScaleUpWindows - 1}, Input{ + Now: now, Replicas: 1, CPUPercent: 95, + Diagnosis: capacity.Diagnosis{Action: capacity.ActionIncreaseCPU, Reason: "Host has headroom"}, + SuggestedResource: orchestrator.Resources{CPULimit: 2}, + }) + + if decision.Action != ActionIncreaseCPU || decision.Resources.CPULimit != 2 { + t.Fatalf("decision = %#v", decision) + } + if !state.LastAction.Equal(now) || state.HighWindows != 0 { + t.Fatalf("state = %#v", state) + } +} + +func TestReconcileRequiresSustainedPressureToAddReplica(t *testing.T) { + policy := DefaultPolicy() + now := time.Now() + state := State{} + for window := 1; window <= policy.ScaleUpWindows; window++ { + var decision Decision + state, decision = Reconcile(policy, state, Input{Now: now, Replicas: 1, CPUPercent: 90}) + if window < policy.ScaleUpWindows && decision.Action != ActionNone { + t.Fatalf("window %d decision = %#v", window, decision) + } + if window == policy.ScaleUpWindows && (decision.Action != ActionAddReplica || decision.Replicas != 2) { + t.Fatalf("window %d decision = %#v", window, decision) + } + } +} + +func TestReconcileRequiresPermittedFleetCapacity(t *testing.T) { + policy := DefaultPolicy() + policy.AllowFleetCapacity = true + state := State{HighWindows: policy.ScaleUpWindows - 1} + _, decision := Reconcile(policy, state, Input{ + Now: time.Now(), Replicas: 1, CPUPercent: 95, RequiresFleet: true, + FleetOffer: capacity.Offer{Enabled: false}, + }) + if decision.Action != ActionNotify || decision.Reason != "No permitted fleet capacity is available" { + t.Fatalf("decision = %#v", decision) + } +} + +func TestReconcileRemovesReplicaAfterSustainedRecovery(t *testing.T) { + policy := DefaultPolicy() + state := State{LowWindows: policy.ScaleDownWindows - 1} + _, decision := Reconcile(policy, state, Input{Now: time.Now(), Replicas: 3, CPUPercent: 10, MemoryPercent: 20}) + if decision.Action != ActionRemoveReplica || decision.Replicas != 2 { + t.Fatalf("decision = %#v", decision) + } +} + +func TestReconcileHonorsCooldown(t *testing.T) { + policy := DefaultPolicy() + now := time.Now() + state := State{HighWindows: policy.ScaleUpWindows - 1, LastAction: now.Add(-time.Minute)} + unchanged, decision := Reconcile(policy, state, Input{Now: now, Replicas: 1, CPUPercent: 99}) + if decision.Action != ActionNone || unchanged.HighWindows != state.HighWindows { + t.Fatalf("state = %#v, decision = %#v", unchanged, decision) + } +} From 64eddef6f082e21140702b1afb86f918b415d81e Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 14:31:03 +0100 Subject: [PATCH 14/46] feat(capacity): Execute safe scaling actions --- internal/autoscale/executor.go | 104 ++++++++++++++++++++++++++ internal/autoscale/executor_test.go | 111 ++++++++++++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 internal/autoscale/executor.go create mode 100644 internal/autoscale/executor_test.go diff --git a/internal/autoscale/executor.go b/internal/autoscale/executor.go new file mode 100644 index 0000000..ae5f5bd --- /dev/null +++ b/internal/autoscale/executor.go @@ -0,0 +1,104 @@ +package autoscale + +import ( + "context" + "fmt" + + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" +) + +type Executor struct { + orchestrator orchestrator.Provider + routing routing.Provider +} + +type Execution struct { + Decision Decision `json:"decision"` + Status orchestrator.Status `json:"status"` + Pending bool `json:"pending"` +} + +func NewExecutor(orchestrator orchestrator.Provider, routing routing.Provider) *Executor { + return &Executor{orchestrator: orchestrator, routing: routing} +} + +func (e *Executor) Execute(ctx context.Context, workloadID string, route routing.Route, decision Decision) (Execution, error) { + result := Execution{Decision: decision} + switch decision.Action { + case ActionNone, ActionNotify: + status, err := e.orchestrator.Status(ctx, workloadID) + result.Status = status + return result, err + case ActionIncreaseCPU, ActionIncreaseMemory: + status, err := e.orchestrator.Resize(ctx, workloadID, decision.Resources) + result.Status = status + return result, err + case ActionAddReplica: + status, err := e.orchestrator.Scale(ctx, workloadID, decision.Replicas) + if err != nil { + return result, err + } + result.Status = status + if status.Available < status.Desired { + result.Pending = true + return result, nil + } + updated, err := routeWithReadyInstances(route, status) + if err != nil { + return result, err + } + if err := e.routing.Reconcile(ctx, updated); err != nil { + return result, fmt.Errorf("publish scaled route: %w", err) + } + return result, nil + case ActionRemoveReplica: + status, err := e.orchestrator.Status(ctx, workloadID) + if err != nil { + return result, err + } + backendID := retiringBackend(route, status) + if backendID == "" { + return result, fmt.Errorf("No routable replica is available to drain") + } + if err := e.routing.Drain(ctx, route.ID, backendID); err != nil { + return result, fmt.Errorf("drain replica: %w", err) + } + status, err = e.orchestrator.Scale(ctx, workloadID, decision.Replicas) + result.Status = status + return result, err + default: + return result, fmt.Errorf("Unknown autoscaling action %q", decision.Action) + } +} + +func routeWithReadyInstances(route routing.Route, status orchestrator.Status) (routing.Route, error) { + backends := make([]routing.Backend, 0, len(status.Instances)) + for _, instance := range status.Instances { + if !instance.Ready || !instance.Healthy { + continue + } + if instance.Address == "" { + return route, fmt.Errorf("Ready instance %q has no routable address", instance.ID) + } + backends = append(backends, routing.Backend{ID: instance.ID, Address: instance.Address, Healthy: true, Weight: 1}) + } + if len(backends) != status.Desired { + return route, fmt.Errorf("Only %d of %d replicas are routable", len(backends), status.Desired) + } + route.Backends = backends + return route, nil +} + +func retiringBackend(route routing.Route, status orchestrator.Status) string { + ready := make(map[string]bool, len(status.Instances)) + for _, instance := range status.Instances { + ready[instance.ID] = instance.Ready + } + for index := len(route.Backends) - 1; index >= 0; index-- { + if ready[route.Backends[index].ID] { + return route.Backends[index].ID + } + } + return "" +} diff --git a/internal/autoscale/executor_test.go b/internal/autoscale/executor_test.go new file mode 100644 index 0000000..725a0cd --- /dev/null +++ b/internal/autoscale/executor_test.go @@ -0,0 +1,111 @@ +package autoscale + +import ( + "context" + "testing" + + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" +) + +type fakeOrchestrator struct { + status orchestrator.Status + scaledTo int + resizedWith orchestrator.Resources +} + +func (f *fakeOrchestrator) ID() orchestrator.ProviderID { return orchestrator.ProviderSwarm } +func (f *fakeOrchestrator) Validate(context.Context, orchestrator.Workload) error { return nil } +func (f *fakeOrchestrator) Apply(context.Context, orchestrator.Workload) (orchestrator.Status, error) { + return f.status, nil +} +func (f *fakeOrchestrator) Resize(_ context.Context, _ string, resources orchestrator.Resources) (orchestrator.Status, error) { + f.resizedWith = resources + return f.status, nil +} +func (f *fakeOrchestrator) Scale(_ context.Context, _ string, replicas int) (orchestrator.Status, error) { + f.scaledTo = replicas + f.status.Desired = replicas + return f.status, nil +} +func (f *fakeOrchestrator) Status(context.Context, string) (orchestrator.Status, error) { + return f.status, nil +} +func (f *fakeOrchestrator) Remove(context.Context, string) error { return nil } + +type fakeRouter struct { + reconciled routing.Route + drained string +} + +func (f *fakeRouter) ID() routing.ProviderID { return routing.ProviderNginx } +func (f *fakeRouter) Validate(context.Context, routing.Route) error { return nil } +func (f *fakeRouter) Reconcile(_ context.Context, route routing.Route) error { + f.reconciled = route + return nil +} +func (f *fakeRouter) Drain(_ context.Context, _, backendID string) error { + f.drained = backendID + return nil +} +func (f *fakeRouter) Remove(context.Context, string) error { return nil } + +func TestExecutorPublishesOnlyReadyScaledReplicas(t *testing.T) { + orchestratorProvider := &fakeOrchestrator{status: orchestrator.Status{ + Workload: "shop", Desired: 1, Available: 2, + Instances: []orchestrator.Instance{ + {ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}, + {ID: "two", Address: "10.0.0.2:8080", Healthy: true, Ready: true}, + }, + }} + router := &fakeRouter{} + execution, err := NewExecutor(orchestratorProvider, router).Execute( + context.Background(), + "shop", + routing.Route{ID: "shop", Domain: "shop.example.com", Protocol: "http"}, + Decision{Action: ActionAddReplica, Replicas: 2}, + ) + if err != nil { + t.Fatalf("Execute failed: %v", err) + } + if execution.Pending || orchestratorProvider.scaledTo != 2 || len(router.reconciled.Backends) != 2 { + t.Fatalf("execution = %#v, route = %#v", execution, router.reconciled) + } +} + +func TestExecutorWaitsForNewReplicaBeforeRouting(t *testing.T) { + orchestratorProvider := &fakeOrchestrator{status: orchestrator.Status{ + Workload: "shop", Desired: 1, Available: 1, + Instances: []orchestrator.Instance{{ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}}, + }} + router := &fakeRouter{} + execution, err := NewExecutor(orchestratorProvider, router).Execute( + context.Background(), "shop", routing.Route{ID: "shop"}, Decision{Action: ActionAddReplica, Replicas: 2}, + ) + if err != nil { + t.Fatalf("Execute failed: %v", err) + } + if !execution.Pending || len(router.reconciled.Backends) != 0 { + t.Fatalf("execution = %#v, route = %#v", execution, router.reconciled) + } +} + +func TestExecutorDrainsBeforeRemovingReplica(t *testing.T) { + orchestratorProvider := &fakeOrchestrator{status: orchestrator.Status{ + Workload: "shop", Desired: 2, Available: 2, + Instances: []orchestrator.Instance{{ID: "one", Ready: true}, {ID: "two", Ready: true}}, + }} + router := &fakeRouter{} + _, err := NewExecutor(orchestratorProvider, router).Execute( + context.Background(), + "shop", + routing.Route{ID: "shop", Backends: []routing.Backend{{ID: "one"}, {ID: "two"}}}, + Decision{Action: ActionRemoveReplica, Replicas: 1}, + ) + if err != nil { + t.Fatalf("Execute failed: %v", err) + } + if router.drained != "two" || orchestratorProvider.scaledTo != 1 { + t.Fatalf("drained = %q, scaled = %d", router.drained, orchestratorProvider.scaledTo) + } +} From bf2d1a703ac42ec3172cd5693bef1bfaeb7b75da Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 14:32:15 +0100 Subject: [PATCH 15/46] feat(cluster): Report routable Swarm replicas --- internal/orchestrator/provider.go | 1 + internal/orchestrator/swarm.go | 29 ++++++++++++++++++++++++++++- internal/orchestrator/swarm_test.go | 12 ++++++++++-- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/internal/orchestrator/provider.go b/internal/orchestrator/provider.go index a3ad228..4d3384a 100644 --- a/internal/orchestrator/provider.go +++ b/internal/orchestrator/provider.go @@ -27,6 +27,7 @@ type Health struct { type Workload struct { ID string `json:"id"` Image string `json:"image"` + Port int `json:"port,omitempty"` Replicas int `json:"replicas"` Resources Resources `json:"resources"` Health Health `json:"health"` diff --git a/internal/orchestrator/swarm.go b/internal/orchestrator/swarm.go index 7f3a297..592119f 100644 --- a/internal/orchestrator/swarm.go +++ b/internal/orchestrator/swarm.go @@ -3,6 +3,8 @@ package orchestrator import ( "context" "fmt" + "net" + "strconv" "strings" "github.com/containerd/errdefs" @@ -59,6 +61,9 @@ func (p *SwarmProvider) Validate(_ context.Context, workload Workload) error { if workload.Stateful && workload.Replicas > 1 { return fmt.Errorf("Stateful workloads cannot use multiple replicas without a storage policy") } + if workload.Port < 0 || workload.Port > 65535 { + return fmt.Errorf("Workload port is invalid") + } return nil } @@ -129,6 +134,7 @@ func (p *SwarmProvider) Status(ctx context.Context, id string) (Status, error) { return Status{}, fmt.Errorf("list Swarm tasks: %w", err) } status := Status{Workload: id} + port := servicePort(service.Service.Spec) if service.Service.Spec.Mode.Replicated != nil && service.Service.Spec.Mode.Replicated.Replicas != nil { status.Desired = int(*service.Service.Spec.Mode.Replicated.Replicas) } @@ -141,7 +147,7 @@ func (p *SwarmProvider) Status(ctx context.Context, id string) (Status, error) { status.Available++ } status.Instances = append(status.Instances, Instance{ - ID: task.ID, Node: task.NodeID, Healthy: running, Ready: running, + ID: task.ID, Node: task.NodeID, Address: taskAddress(task, port), Healthy: running, Ready: running, }) } return status, nil @@ -161,6 +167,9 @@ func swarmSpec(workload Workload) swarm.ServiceSpec { labels[key] = value } labels["flatrun.workload"] = workload.ID + if workload.Port > 0 { + labels["flatrun.port"] = strconv.Itoa(workload.Port) + } return swarm.ServiceSpec{ Annotations: swarm.Annotations{Name: workload.ID, Labels: labels}, TaskTemplate: swarm.TaskSpec{ @@ -171,6 +180,24 @@ func swarmSpec(workload Workload) swarm.ServiceSpec { } } +func servicePort(spec swarm.ServiceSpec) int { + value := spec.Annotations.Labels["flatrun.port"] + port, _ := strconv.Atoi(value) + return port +} + +func taskAddress(task swarm.Task, port int) string { + if port == 0 { + return "" + } + for _, attachment := range task.NetworksAttachments { + for _, address := range attachment.Addresses { + return net.JoinHostPort(address.Addr().String(), strconv.Itoa(port)) + } + } + return "" +} + func swarmResources(resources Resources) *swarm.ResourceRequirements { return &swarm.ResourceRequirements{ Limits: &swarm.Limit{ diff --git a/internal/orchestrator/swarm_test.go b/internal/orchestrator/swarm_test.go index 7364110..9889dd5 100644 --- a/internal/orchestrator/swarm_test.go +++ b/internal/orchestrator/swarm_test.go @@ -2,6 +2,7 @@ package orchestrator import ( "context" + "net/netip" "testing" "github.com/containerd/errdefs" @@ -49,11 +50,15 @@ func (f *fakeSwarmClient) TaskList(_ context.Context, _ client.TaskListOptions) } func TestSwarmProviderCreatesReplicatedService(t *testing.T) { - client := &fakeSwarmClient{tasks: []swarm.Task{{ID: "task-1", NodeID: "node-1", DesiredState: swarm.TaskStateRunning, Status: swarm.TaskStatus{State: swarm.TaskStateRunning}}}} + client := &fakeSwarmClient{tasks: []swarm.Task{{ + ID: "task-1", NodeID: "node-1", DesiredState: swarm.TaskStateRunning, + Status: swarm.TaskStatus{State: swarm.TaskStateRunning}, + NetworksAttachments: []swarm.NetworkAttachment{{Addresses: []netip.Prefix{netip.MustParsePrefix("10.0.0.12/24")}}}, + }}} provider := NewSwarmProvider(client) status, err := provider.Apply(context.Background(), Workload{ - ID: "shop", Image: "example/shop:1", Replicas: 1, + ID: "shop", Image: "example/shop:1", Port: 8080, Replicas: 1, Resources: Resources{CPURequest: 0.5, CPULimit: 1, MemoryRequest: 256 << 20, MemoryLimit: 512 << 20}, }) if err != nil { @@ -68,6 +73,9 @@ func TestSwarmProviderCreatesReplicatedService(t *testing.T) { if status.Desired != 1 || status.Available != 1 || len(status.Instances) != 1 { t.Fatalf("status = %#v", status) } + if status.Instances[0].Address != "10.0.0.12:8080" { + t.Fatalf("instance address = %q", status.Instances[0].Address) + } } func TestSwarmProviderScalesExistingService(t *testing.T) { From 0a32b8b20d05cd3f03c1649a404a0b8b5fd7b221 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 14:38:42 +0100 Subject: [PATCH 16/46] feat(cluster): Configure Fleet providers Administrators can select available orchestration and routing providers without being offered adapters that are not ready on the server. --- internal/api/cluster_handlers.go | 133 ++++++++++++++++++++++++++ internal/api/cluster_handlers_test.go | 93 ++++++++++++++++++ internal/api/openapi.json | 115 ++++++++++++++++++++++ internal/api/server.go | 6 ++ internal/orchestrator/swarm.go | 7 ++ pkg/config/config.go | 8 ++ 6 files changed, 362 insertions(+) diff --git a/internal/api/cluster_handlers.go b/internal/api/cluster_handlers.go index 8e45800..f173104 100644 --- a/internal/api/cluster_handlers.go +++ b/internal/api/cluster_handlers.go @@ -17,12 +17,145 @@ import ( "github.com/flatrun/agent/internal/auth" "github.com/flatrun/agent/internal/capacity" "github.com/flatrun/agent/internal/cluster" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" "github.com/flatrun/agent/internal/system" "github.com/flatrun/agent/pkg/config" "github.com/flatrun/agent/pkg/version" "github.com/gin-gonic/gin" ) +type clusterProviderOption struct { + ID string `json:"id"` + Active bool `json:"active"` + Available bool `json:"available"` + Reason string `json:"reason,omitempty"` +} + +type clusterProvidersResponse struct { + Orchestrators []clusterProviderOption `json:"orchestrators"` + Routing []clusterProviderOption `json:"routing"` +} + +func (s *Server) clusterProviders(c *gin.Context) { + orchestratorID := s.config.Cluster.Orchestrator + if orchestratorID == "" { + orchestratorID = string(orchestrator.ProviderStandalone) + } + routingID := s.config.Cluster.Routing + if routingID == "" { + routingID = string(routing.ProviderNginx) + } + + c.JSON(http.StatusOK, clusterProvidersResponse{ + Orchestrators: []clusterProviderOption{ + s.orchestratorOption(c, orchestrator.ProviderStandalone, orchestratorID), + s.orchestratorOption(c, orchestrator.ProviderSwarm, orchestratorID), + s.orchestratorOption(c, orchestrator.ProviderK3s, orchestratorID), + }, + Routing: []clusterProviderOption{ + s.routingOption(c, routing.ProviderNginx, routingID), + s.routingOption(c, routing.ProviderTraefik, routingID), + }, + }) +} + +func (s *Server) orchestratorOption(c *gin.Context, id orchestrator.ProviderID, active string) clusterProviderOption { + option := clusterProviderOption{ID: string(id), Active: active == string(id)} + if err := s.checkOrchestrator(c, id); err != nil { + option.Reason = err.Error() + return option + } + option.Available = true + return option +} + +func (s *Server) routingOption(c *gin.Context, id routing.ProviderID, active string) clusterProviderOption { + option := clusterProviderOption{ID: string(id), Active: active == string(id)} + if err := s.checkRouting(c, id); err != nil { + option.Reason = err.Error() + return option + } + option.Available = true + return option +} + +func (s *Server) checkOrchestrator(ctx context.Context, id orchestrator.ProviderID) error { + if s.probeOrchestrator != nil { + return s.probeOrchestrator(ctx, id) + } + switch id { + case orchestrator.ProviderStandalone: + return nil + case orchestrator.ProviderSwarm: + provider, err := orchestrator.NewSwarmProviderFromEnv() + if err != nil { + return err + } + defer provider.Close() + return provider.Ready(ctx) + case orchestrator.ProviderK3s: + return fmt.Errorf("k3s adapter is not configured") + default: + return fmt.Errorf("orchestrator %q is not supported", id) + } +} + +func (s *Server) checkRouting(ctx context.Context, id routing.ProviderID) error { + if s.probeRouting != nil { + return s.probeRouting(ctx, id) + } + switch id { + case routing.ProviderNginx: + if !s.config.Nginx.Enabled { + return fmt.Errorf("Nginx is not enabled") + } + return nil + case routing.ProviderTraefik: + return fmt.Errorf("Traefik adapter is not configured") + default: + return fmt.Errorf("routing provider %q is not supported", id) + } +} + +type updateClusterProvidersRequest struct { + Orchestrator string `json:"orchestrator" binding:"required"` + Routing string `json:"routing" binding:"required"` +} + +func (s *Server) updateClusterProviders(c *gin.Context) { + var req updateClusterProvidersRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + orchestratorID := orchestrator.ProviderID(strings.TrimSpace(req.Orchestrator)) + routingID := routing.ProviderID(strings.TrimSpace(req.Routing)) + if err := s.checkOrchestrator(c, orchestratorID); err != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()}) + return + } + if err := s.checkRouting(c, routingID); err != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()}) + return + } + + previous := s.config.Cluster + s.config.Cluster.Orchestrator = string(orchestratorID) + s.config.Cluster.Routing = string(routingID) + if s.configPath != "" { + if err := config.Save(s.config, s.configPath); err != nil { + s.config.Cluster = previous + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to save provider configuration: %v", err)}) + return + } + } + c.JSON(http.StatusOK, gin.H{ + "orchestrator": s.config.Cluster.Orchestrator, + "routing": s.config.Cluster.Routing, + }) +} + func (s *Server) clusterStatus(c *gin.Context) { mgr := s.getClusterManager() if mgr == nil { diff --git a/internal/api/cluster_handlers_test.go b/internal/api/cluster_handlers_test.go index 9e8f85f..69e3dbb 100644 --- a/internal/api/cluster_handlers_test.go +++ b/internal/api/cluster_handlers_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "os" @@ -13,6 +14,8 @@ import ( "github.com/flatrun/agent/internal/auth" "github.com/flatrun/agent/internal/capacity" "github.com/flatrun/agent/internal/cluster" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" "github.com/flatrun/agent/pkg/config" "github.com/gin-gonic/gin" ) @@ -105,6 +108,8 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool clusterGroup.Use(authMiddleware.RequirePermission(auth.PermClusterRead)) { clusterGroup.GET("/status", server.clusterStatus) + clusterGroup.GET("/providers", server.clusterProviders) + clusterGroup.PUT("/providers", authMiddleware.RequirePermission(auth.PermClusterWrite), server.updateClusterProviders) clusterGroup.POST("/setup", authMiddleware.RequirePermission(auth.PermClusterWrite), server.clusterSetup) clusterGroup.GET("/peers", server.clusterListPeers) clusterGroup.GET("/peers/:name/policy", server.clusterPeerPolicy) @@ -749,6 +754,94 @@ func TestClusterUnauthorizedAccess(t *testing.T) { } } +func TestClusterProvidersReportActiveAndAvailableAdapters(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + env.server.probeOrchestrator = func(_ context.Context, id orchestrator.ProviderID) error { + if id == orchestrator.ProviderK3s { + return fmt.Errorf("k3s adapter is not configured") + } + return nil + } + env.server.probeRouting = func(_ context.Context, id routing.ProviderID) error { + if id == routing.ProviderTraefik { + return fmt.Errorf("Traefik adapter is not configured") + } + return nil + } + + req := httptest.NewRequest(http.MethodGet, "/api/cluster/providers", nil) + req.Header.Set("Authorization", "Bearer "+clusterLogin(t, env.router)) + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + var response clusterProvidersResponse + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if !response.Orchestrators[0].Active || !response.Orchestrators[0].Available { + t.Fatalf("unexpected standalone provider: %+v", response.Orchestrators[0]) + } + if response.Orchestrators[2].Available || response.Orchestrators[2].Reason == "" { + t.Fatalf("unexpected k3s provider: %+v", response.Orchestrators[2]) + } + if !response.Routing[0].Active || !response.Routing[0].Available { + t.Fatalf("unexpected Nginx provider: %+v", response.Routing[0]) + } +} + +func TestUpdateClusterProvidersPersistsAvailableSelection(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + env.server.probeOrchestrator = func(_ context.Context, _ orchestrator.ProviderID) error { return nil } + env.server.probeRouting = func(_ context.Context, _ routing.ProviderID) error { return nil } + + body := bytes.NewBufferString(`{"orchestrator":"swarm","routing":"nginx"}`) + req := httptest.NewRequest(http.MethodPut, "/api/cluster/providers", body) + req.Header.Set("Authorization", "Bearer "+clusterLogin(t, env.router)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + + saved, err := config.Load(env.server.configPath) + if err != nil { + t.Fatal(err) + } + if saved.Cluster.Orchestrator != "swarm" || saved.Cluster.Routing != "nginx" { + t.Fatalf("unexpected saved providers: %+v", saved.Cluster) + } +} + +func TestUpdateClusterProvidersRejectsUnavailableSelection(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + env.server.probeOrchestrator = func(_ context.Context, id orchestrator.ProviderID) error { + if id == orchestrator.ProviderK3s { + return fmt.Errorf("k3s adapter is not configured") + } + return nil + } + env.server.probeRouting = func(_ context.Context, _ routing.ProviderID) error { return nil } + + body := bytes.NewBufferString(`{"orchestrator":"k3s","routing":"nginx"}`) + req := httptest.NewRequest(http.MethodPut, "/api/cluster/providers", body) + req.Header.Set("Authorization", "Bearer "+clusterLogin(t, env.router)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + if w.Code != http.StatusUnprocessableEntity { + t.Fatalf("expected 422, got %d: %s", w.Code, w.Body.String()) + } + if env.server.config.Cluster.Orchestrator != "" { + t.Fatalf("unexpected orchestrator change: %q", env.server.config.Cluster.Orchestrator) + } +} + func TestClusterFullPeeringE2E(t *testing.T) { // This test simulates the full peering flow between two servers // using httptest servers as live HTTP endpoints. diff --git a/internal/api/openapi.json b/internal/api/openapi.json index 151fd02..e7c0d45 100644 --- a/internal/api/openapi.json +++ b/internal/api/openapi.json @@ -1715,6 +1715,48 @@ "x-permission": "cluster:write" } }, + "/api/cluster/providers": { + "get": { + "operationId": "get-cluster-providers", + "tags": [ + "cluster" + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/api.clusterProvidersResponse" + } + } + } + } + } + }, + "put": { + "operationId": "put-cluster-providers", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/api.updateClusterProvidersRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "cluster" + ], + "x-permission": "cluster:write" + } + }, "/api/cluster/setup": { "post": { "operationId": "post-cluster-setup", @@ -11218,6 +11260,56 @@ "tail" ] }, + "api.clusterProviderOption": { + "type": "object", + "properties": { + "active": { + "type": "boolean" + }, + "available": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "x-property-order": [ + "id", + "active", + "available", + "reason" + ], + "x-columns": [ + "id", + "active", + "available", + "reason" + ] + }, + "api.clusterProvidersResponse": { + "type": "object", + "properties": { + "orchestrators": { + "type": "array", + "items": { + "$ref": "#/components/schemas/api.clusterProviderOption" + } + }, + "routing": { + "type": "array", + "items": { + "$ref": "#/components/schemas/api.clusterProviderOption" + } + } + }, + "x-property-order": [ + "orchestrators", + "routing" + ] + }, "api.clusterSetupRequest": { "type": "object", "properties": { @@ -11476,6 +11568,29 @@ "deployment" ] }, + "api.updateClusterProvidersRequest": { + "type": "object", + "properties": { + "orchestrator": { + "type": "string" + }, + "routing": { + "type": "string" + } + }, + "x-property-order": [ + "orchestrator", + "routing" + ], + "x-columns": [ + "orchestrator", + "routing" + ], + "required": [ + "orchestrator", + "routing" + ] + }, "audit.ActorStats": { "type": "object", "properties": { diff --git a/internal/api/server.go b/internal/api/server.go index b3dc4ca..a2357d8 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -40,9 +40,11 @@ import ( "github.com/flatrun/agent/internal/infra" "github.com/flatrun/agent/internal/networks" "github.com/flatrun/agent/internal/notify" + "github.com/flatrun/agent/internal/orchestrator" "github.com/flatrun/agent/internal/plan" "github.com/flatrun/agent/internal/pluginhost" "github.com/flatrun/agent/internal/proxy" + "github.com/flatrun/agent/internal/routing" "github.com/flatrun/agent/internal/scheduler" "github.com/flatrun/agent/internal/security" "github.com/flatrun/agent/internal/setup" @@ -100,6 +102,8 @@ type Server struct { powerDNSManager *dns.PowerDNSManager clusterMu sync.RWMutex clusterManager *cluster.Manager + probeOrchestrator func(context.Context, orchestrator.ProviderID) error + probeRouting func(context.Context, routing.ProviderID) error setupManager *setup.Manager setupHandlers *setup.Handlers certRenewer *ssl.Renewer @@ -889,6 +893,8 @@ func (s *Server) setupRoutes() { clusterGroup.GET("/deployments", s.clusterAggregateDeployments) clusterGroup.GET("/stats", s.clusterAggregateStats) clusterGroup.GET("/capacity", s.clusterAggregateCapacity) + clusterGroup.GET("/providers", s.clusterProviders) + clusterGroup.PUT("/providers", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.updateClusterProviders) } } diff --git a/internal/orchestrator/swarm.go b/internal/orchestrator/swarm.go index 592119f..05b57e3 100644 --- a/internal/orchestrator/swarm.go +++ b/internal/orchestrator/swarm.go @@ -44,6 +44,13 @@ func (p *SwarmProvider) Ready(ctx context.Context) error { return nil } +func (p *SwarmProvider) Close() error { + if closer, ok := p.client.(interface{ Close() error }); ok { + return closer.Close() + } + return nil +} + func (p *SwarmProvider) ID() ProviderID { return ProviderSwarm } diff --git a/pkg/config/config.go b/pkg/config/config.go index 8fbe965..211d1c2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -19,6 +19,8 @@ type ClusterConfig struct { AdvertiseURL string `yaml:"advertise_url"` HealthInterval string `yaml:"health_interval"` RequestTimeout string `yaml:"request_timeout"` + Orchestrator string `yaml:"orchestrator" json:"orchestrator"` + Routing string `yaml:"routing" json:"routing"` } type CapacityConfig struct { @@ -401,6 +403,12 @@ func setDefaults(cfg *Config) { if cfg.DefaultTimeout == 0 { cfg.DefaultTimeout = 2 * time.Minute } + if cfg.Cluster.Orchestrator == "" { + cfg.Cluster.Orchestrator = "standalone" + } + if cfg.Cluster.Routing == "" { + cfg.Cluster.Routing = "nginx" + } if cfg.Capacity.AllocationThresholdPercent == 0 { cfg.Capacity.AllocationThresholdPercent = 90 } From eb2bf517ac5ae8f8539621d2dad2cfe005c2bad0 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 14:43:20 +0100 Subject: [PATCH 17/46] refactor: Mark Moby API as a direct dependency The Swarm runtime now declares the Moby API contract it imports directly, keeping module validation reproducible in CI. --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index fdac7ee..c44bbc6 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 github.com/lib/pq v1.10.9 + github.com/moby/moby/api v1.53.0 github.com/moby/moby/client v0.2.2 github.com/modelcontextprotocol/go-sdk v1.6.1 github.com/nicholas-fedor/shoutrrr v0.16.1 @@ -144,7 +145,6 @@ require ( github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect github.com/moby/locker v1.0.1 // indirect - github.com/moby/moby/api v1.53.0 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/moby/sys/capability v0.4.0 // indirect From 5ccf5f4d36531c2b68cfdfad7fa2aeb0aa505f05 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 14:46:47 +0100 Subject: [PATCH 18/46] feat(capacity): Persist deployment scaling policies Each deployment can retain validated scaling limits and controller state across agent restarts, including on a single server without Fleet enabled. --- internal/api/autoscale_handlers.go | 91 ++++++++ internal/api/autoscale_handlers_test.go | 80 +++++++ internal/api/openapi.json | 207 ++++++++++++++++++ internal/api/server.go | 14 +- internal/autoscale/controller.go | 4 +- .../autoscale/migrations/00001_initial.sql | 16 ++ internal/autoscale/store.go | 122 +++++++++++ internal/autoscale/store_test.go | 60 +++++ 8 files changed, 591 insertions(+), 3 deletions(-) create mode 100644 internal/api/autoscale_handlers.go create mode 100644 internal/api/autoscale_handlers_test.go create mode 100644 internal/autoscale/migrations/00001_initial.sql create mode 100644 internal/autoscale/store.go create mode 100644 internal/autoscale/store_test.go diff --git a/internal/api/autoscale_handlers.go b/internal/api/autoscale_handlers.go new file mode 100644 index 0000000..8d75079 --- /dev/null +++ b/internal/api/autoscale_handlers.go @@ -0,0 +1,91 @@ +package api + +import ( + "net/http" + "time" + + "github.com/flatrun/agent/internal/autoscale" + "github.com/gin-gonic/gin" +) + +type autoscalePolicyRequest struct { + Enabled bool `json:"enabled"` + MinReplicas int `json:"min_replicas"` + MaxReplicas int `json:"max_replicas"` + ScaleUpPercent float64 `json:"scale_up_percent"` + ScaleDownPercent float64 `json:"scale_down_percent"` + ScaleUpWindows int `json:"scale_up_windows"` + ScaleDownWindows int `json:"scale_down_windows"` + CooldownSeconds int64 `json:"cooldown_seconds"` + AllowFleetCapacity bool `json:"allow_fleet_capacity"` +} + +type autoscalePolicyResponse struct { + autoscalePolicyRequest + State autoscale.State `json:"state"` +} + +func (s *Server) getDeploymentAutoscalePolicy(c *gin.Context) { + if s.autoscaleStore == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Autoscaling storage is unavailable"}) + return + } + name := c.Param("name") + if _, err := s.manager.GetDeployment(name); err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "Deployment not found"}) + return + } + policy, err := s.autoscaleStore.Policy(name) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + state, err := s.autoscaleStore.State(name) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, autoscalePolicyResponse{autoscalePolicyRequest: policyRequest(policy), State: state}) +} + +func (s *Server) updateDeploymentAutoscalePolicy(c *gin.Context) { + if s.autoscaleStore == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Autoscaling storage is unavailable"}) + return + } + name := c.Param("name") + if _, err := s.manager.GetDeployment(name); err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "Deployment not found"}) + return + } + var req autoscalePolicyRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + policy := autoscale.Policy{ + Enabled: req.Enabled, MinReplicas: req.MinReplicas, MaxReplicas: req.MaxReplicas, + ScaleUpPercent: req.ScaleUpPercent, ScaleDownPercent: req.ScaleDownPercent, + ScaleUpWindows: req.ScaleUpWindows, ScaleDownWindows: req.ScaleDownWindows, + Cooldown: time.Duration(req.CooldownSeconds) * time.Second, AllowFleetCapacity: req.AllowFleetCapacity, + } + if err := s.autoscaleStore.SetPolicy(name, policy); err != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()}) + return + } + state, err := s.autoscaleStore.State(name) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, autoscalePolicyResponse{autoscalePolicyRequest: req, State: state}) +} + +func policyRequest(policy autoscale.Policy) autoscalePolicyRequest { + return autoscalePolicyRequest{ + Enabled: policy.Enabled, MinReplicas: policy.MinReplicas, MaxReplicas: policy.MaxReplicas, + ScaleUpPercent: policy.ScaleUpPercent, ScaleDownPercent: policy.ScaleDownPercent, + ScaleUpWindows: policy.ScaleUpWindows, ScaleDownWindows: policy.ScaleDownWindows, + CooldownSeconds: int64(policy.Cooldown / time.Second), AllowFleetCapacity: policy.AllowFleetCapacity, + } +} diff --git a/internal/api/autoscale_handlers_test.go b/internal/api/autoscale_handlers_test.go new file mode 100644 index 0000000..ba88cd6 --- /dev/null +++ b/internal/api/autoscale_handlers_test.go @@ -0,0 +1,80 @@ +package api + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/flatrun/agent/internal/auth" + "github.com/flatrun/agent/internal/autoscale" + "github.com/flatrun/agent/internal/docker" + "github.com/flatrun/agent/pkg/config" + "github.com/gin-gonic/gin" +) + +func TestDeploymentAutoscalePolicyThroughHTTP(t *testing.T) { + dir := t.TempDir() + deploymentDir := filepath.Join(dir, "shop") + if err := os.MkdirAll(deploymentDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(deploymentDir, "compose.yml"), []byte("services:\n app:\n image: nginx:alpine\n"), 0644); err != nil { + t.Fatal(err) + } + cfg := &config.Config{DeploymentsPath: dir, Auth: config.AuthConfig{Enabled: true, JWTSecret: "autoscale-test-secret"}} + t.Setenv("FLATRUN_ADMIN_PASSWORD", "testadminpass") + authManager, err := auth.NewManager(dir, &cfg.Auth, true) + if err != nil { + t.Fatal(err) + } + defer authManager.Close() + store, err := autoscale.NewStore(dir) + if err != nil { + t.Fatal(err) + } + defer store.Close() + server := &Server{config: cfg, manager: docker.NewManager(dir), authManager: authManager, autoscaleStore: store} + middleware := auth.NewMiddlewareWithManager(&cfg.Auth, authManager) + router := gin.New() + router.POST("/api/auth/login", middleware.Login) + protected := router.Group("/api", middleware.RequireAuth()) + protected.GET("/deployments/:name/autoscale", middleware.RequirePermission(auth.PermDeploymentsRead), middleware.RequireDeploymentAccess(auth.AccessLevelRead), server.getDeploymentAutoscalePolicy) + protected.PUT("/deployments/:name/autoscale", middleware.RequirePermission(auth.PermDeploymentsWrite), middleware.RequireDeploymentAccess(auth.AccessLevelWrite), server.updateDeploymentAutoscalePolicy) + token := loginAndGetToken(t, router, "admin", "testadminpass") + + payload := autoscalePolicyRequest{ + Enabled: true, MinReplicas: 2, MaxReplicas: 6, ScaleUpPercent: 75, ScaleDownPercent: 25, + ScaleUpWindows: 3, ScaleDownWindows: 8, CooldownSeconds: 120, AllowFleetCapacity: true, + } + raw, err := json.Marshal(payload) + if err != nil { + t.Fatal(err) + } + req := httptest.NewRequest(http.MethodPut, "/api/deployments/shop/autoscale", bytes.NewReader(raw)) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + + req = httptest.NewRequest(http.MethodGet, "/api/deployments/shop/autoscale", nil) + req.Header.Set("Authorization", "Bearer "+token) + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + var response autoscalePolicyResponse + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.MaxReplicas != 6 || response.CooldownSeconds != 120 || !response.AllowFleetCapacity { + t.Fatalf("unexpected response: %+v", response) + } +} diff --git a/internal/api/openapi.json b/internal/api/openapi.json index e7c0d45..44f249d 100644 --- a/internal/api/openapi.json +++ b/internal/api/openapi.json @@ -3835,6 +3835,76 @@ "x-permission": "deployments:read" } }, + "/api/deployments/{name}/autoscale": { + "get": { + "operationId": "get-deployments-by-name-autoscale", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/api.autoscalePolicyResponse" + } + } + }, + "description": "Success" + } + }, + "tags": [ + "deployments" + ], + "x-permission": "deployments:read" + }, + "put": { + "operationId": "put-deployments-by-name-autoscale", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/api.autoscalePolicyRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/api.autoscalePolicyResponse" + } + } + }, + "description": "Success" + } + }, + "tags": [ + "deployments" + ], + "x-permission": "deployments:write" + } + }, "/api/deployments/{name}/backup-config": { "get": { "operationId": "get-deployments-by-name-backup-config", @@ -11260,6 +11330,118 @@ "tail" ] }, + "api.autoscalePolicyRequest": { + "type": "object", + "properties": { + "allow_fleet_capacity": { + "type": "boolean" + }, + "cooldown_seconds": { + "type": "integer" + }, + "enabled": { + "type": "boolean" + }, + "max_replicas": { + "type": "integer" + }, + "min_replicas": { + "type": "integer" + }, + "scale_down_percent": { + "type": "number" + }, + "scale_down_windows": { + "type": "integer" + }, + "scale_up_percent": { + "type": "number" + }, + "scale_up_windows": { + "type": "integer" + } + }, + "x-property-order": [ + "enabled", + "min_replicas", + "max_replicas", + "scale_up_percent", + "scale_down_percent", + "scale_up_windows", + "scale_down_windows", + "cooldown_seconds", + "allow_fleet_capacity" + ], + "x-columns": [ + "enabled", + "min_replicas", + "max_replicas", + "scale_up_percent", + "scale_down_percent", + "scale_up_windows", + "scale_down_windows", + "cooldown_seconds", + "allow_fleet_capacity" + ] + }, + "api.autoscalePolicyResponse": { + "type": "object", + "properties": { + "allow_fleet_capacity": { + "type": "boolean" + }, + "cooldown_seconds": { + "type": "integer" + }, + "enabled": { + "type": "boolean" + }, + "max_replicas": { + "type": "integer" + }, + "min_replicas": { + "type": "integer" + }, + "scale_down_percent": { + "type": "number" + }, + "scale_down_windows": { + "type": "integer" + }, + "scale_up_percent": { + "type": "number" + }, + "scale_up_windows": { + "type": "integer" + }, + "state": { + "$ref": "#/components/schemas/autoscale.State" + } + }, + "x-property-order": [ + "enabled", + "min_replicas", + "max_replicas", + "scale_up_percent", + "scale_down_percent", + "scale_up_windows", + "scale_down_windows", + "cooldown_seconds", + "allow_fleet_capacity", + "state" + ], + "x-columns": [ + "enabled", + "min_replicas", + "max_replicas", + "scale_up_percent", + "scale_down_percent", + "scale_up_windows", + "scale_down_windows", + "cooldown_seconds", + "allow_fleet_capacity" + ] + }, "api.clusterProviderOption": { "type": "object", "properties": { @@ -11818,6 +12000,31 @@ "count" ] }, + "autoscale.State": { + "type": "object", + "properties": { + "high_windows": { + "type": "integer" + }, + "last_action": { + "type": "string", + "format": "date-time" + }, + "low_windows": { + "type": "integer" + } + }, + "x-property-order": [ + "high_windows", + "low_windows", + "last_action" + ], + "x-columns": [ + "high_windows", + "low_windows", + "last_action" + ] + }, "backup.Backup": { "type": "object", "properties": { diff --git a/internal/api/server.go b/internal/api/server.go index a2357d8..10c85f9 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -28,6 +28,7 @@ import ( "github.com/flatrun/agent/internal/ai" "github.com/flatrun/agent/internal/audit" "github.com/flatrun/agent/internal/auth" + "github.com/flatrun/agent/internal/autoscale" "github.com/flatrun/agent/internal/backup" "github.com/flatrun/agent/internal/certs" "github.com/flatrun/agent/internal/cluster" @@ -99,6 +100,7 @@ type Server struct { schedulerManager *scheduler.Manager auditManager *audit.Manager auditMiddleware *audit.Middleware + autoscaleStore *autoscale.Store powerDNSManager *dns.PowerDNSManager clusterMu sync.RWMutex clusterManager *cluster.Manager @@ -182,6 +184,10 @@ func New(cfg *config.Config, configPath string) *Server { manager := docker.NewManager(cfg.DeploymentsPath) manager.SetCleanupTimeout(cfg.Cleanup.Timeout) + autoscaleStore, err := autoscale.NewStore(cfg.DeploymentsPath) + if err != nil { + log.Printf("Warning: Failed to initialize autoscaling store: %v", err) + } // Deploys read template copies from disk. Seed the embedded infra and // welcome content, then pull the app catalog from its external source into @@ -283,7 +289,7 @@ func New(cfg *config.Config, configPath string) *Server { } var trafficManager *traffic.Manager - trafficManager, err := traffic.NewManager(cfg.DeploymentsPath, 7) + trafficManager, err = traffic.NewManager(cfg.DeploymentsPath, 7) if err != nil { log.Printf("Warning: Failed to initialize traffic manager: %v", err) } @@ -367,6 +373,7 @@ func New(cfg *config.Config, configPath string) *Server { backupManager: backupManager, auditManager: auditManager, auditMiddleware: auditMiddleware, + autoscaleStore: autoscaleStore, powerDNSManager: powerDNSManager, clusterManager: clusterManager, setupManager: setupManager, @@ -618,6 +625,8 @@ func (s *Server) setupRoutes() { protected.PUT("/containers/:id/resources", s.authMiddleware.RequirePermission(auth.PermContainersWrite), s.updateContainerResources) protected.GET("/deployments/:name/stats", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentContainerStats) protected.GET("/deployments/:name/resources", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentResources) + protected.GET("/deployments/:name/autoscale", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentAutoscalePolicy) + protected.PUT("/deployments/:name/autoscale", s.authMiddleware.RequirePermission(auth.PermDeploymentsWrite), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelWrite), s.updateDeploymentAutoscalePolicy) // Image endpoints protected.GET("/images", s.authMiddleware.RequirePermission(auth.PermImagesRead), s.listImages) @@ -970,6 +979,9 @@ func (s *Server) Stop() error { if s.clusterManager != nil { s.clusterManager.Stop() } + if s.autoscaleStore != nil { + _ = s.autoscaleStore.Close() + } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() return s.server.Shutdown(ctx) diff --git a/internal/autoscale/controller.go b/internal/autoscale/controller.go index 8a18652..a653316 100644 --- a/internal/autoscale/controller.go +++ b/internal/autoscale/controller.go @@ -67,7 +67,7 @@ func DefaultPolicy() Policy { } func Reconcile(policy Policy, state State, input Input) (State, Decision) { - if err := validatePolicy(policy); err != nil { + if err := ValidatePolicy(policy); err != nil { return state, Decision{Action: ActionNotify, Reason: err.Error()} } if !policy.Enabled { @@ -117,7 +117,7 @@ func Reconcile(policy Policy, state State, input Input) (State, Decision) { return state, Decision{Action: ActionNone, Reason: "No scaling threshold has been sustained"} } -func validatePolicy(policy Policy) error { +func ValidatePolicy(policy Policy) error { if policy.MinReplicas < 1 || policy.MaxReplicas < policy.MinReplicas { return fmt.Errorf("Replica limits are invalid") } diff --git a/internal/autoscale/migrations/00001_initial.sql b/internal/autoscale/migrations/00001_initial.sql new file mode 100644 index 0000000..fc72550 --- /dev/null +++ b/internal/autoscale/migrations/00001_initial.sql @@ -0,0 +1,16 @@ +-- +goose Up +CREATE TABLE autoscale_policies ( + deployment TEXT PRIMARY KEY, + policy_json TEXT NOT NULL, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE autoscale_states ( + deployment TEXT PRIMARY KEY, + state_json TEXT NOT NULL, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +-- +goose Down +DROP TABLE autoscale_states; +DROP TABLE autoscale_policies; diff --git a/internal/autoscale/store.go b/internal/autoscale/store.go new file mode 100644 index 0000000..bbff29a --- /dev/null +++ b/internal/autoscale/store.go @@ -0,0 +1,122 @@ +package autoscale + +import ( + "context" + "database/sql" + "embed" + "encoding/json" + "io/fs" + "os" + "path/filepath" + "sync" + + "github.com/pressly/goose/v3" + _ "modernc.org/sqlite" +) + +//go:embed migrations/*.sql +var migrationFiles embed.FS + +type Store struct { + conn *sql.DB + mu sync.RWMutex +} + +func NewStore(deploymentsPath string) (*Store, error) { + dir := filepath.Join(deploymentsPath, ".flatrun") + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, err + } + conn, err := sql.Open("sqlite", "file:"+filepath.Join(dir, "autoscale.db")+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + if err != nil { + return nil, err + } + store := &Store{conn: conn} + if err := store.migrate(); err != nil { + conn.Close() + return nil, err + } + return store, nil +} + +func (s *Store) migrate() error { + migrations, err := fs.Sub(migrationFiles, "migrations") + if err != nil { + return err + } + provider, err := goose.NewProvider(goose.DialectSQLite3, s.conn, migrations, goose.WithTableName("autoscale_schema_version")) + if err != nil { + return err + } + _, err = provider.Up(context.Background()) + return err +} + +func (s *Store) Close() error { + return s.conn.Close() +} + +func (s *Store) Policy(deployment string) (Policy, error) { + s.mu.RLock() + defer s.mu.RUnlock() + var raw string + err := s.conn.QueryRow(`SELECT policy_json FROM autoscale_policies WHERE deployment = ?`, deployment).Scan(&raw) + if err == sql.ErrNoRows { + return DefaultPolicy(), nil + } + if err != nil { + return Policy{}, err + } + var policy Policy + if err := json.Unmarshal([]byte(raw), &policy); err != nil { + return Policy{}, err + } + return policy, nil +} + +func (s *Store) SetPolicy(deployment string, policy Policy) error { + if err := ValidatePolicy(policy); err != nil { + return err + } + raw, err := json.Marshal(policy) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + _, err = s.conn.Exec(` + INSERT INTO autoscale_policies (deployment, policy_json, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(deployment) DO UPDATE SET policy_json = excluded.policy_json, updated_at = CURRENT_TIMESTAMP`, deployment, raw) + return err +} + +func (s *Store) State(deployment string) (State, error) { + s.mu.RLock() + defer s.mu.RUnlock() + var raw string + err := s.conn.QueryRow(`SELECT state_json FROM autoscale_states WHERE deployment = ?`, deployment).Scan(&raw) + if err == sql.ErrNoRows { + return State{}, nil + } + if err != nil { + return State{}, err + } + var state State + if err := json.Unmarshal([]byte(raw), &state); err != nil { + return State{}, err + } + return state, nil +} + +func (s *Store) SetState(deployment string, state State) error { + raw, err := json.Marshal(state) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + _, err = s.conn.Exec(` + INSERT INTO autoscale_states (deployment, state_json, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(deployment) DO UPDATE SET state_json = excluded.state_json, updated_at = CURRENT_TIMESTAMP`, deployment, raw) + return err +} diff --git a/internal/autoscale/store_test.go b/internal/autoscale/store_test.go new file mode 100644 index 0000000..4d59383 --- /dev/null +++ b/internal/autoscale/store_test.go @@ -0,0 +1,60 @@ +package autoscale + +import ( + "testing" + "time" +) + +func TestStorePersistsPolicyAndState(t *testing.T) { + dir := t.TempDir() + store, err := NewStore(dir) + if err != nil { + t.Fatal(err) + } + policy := DefaultPolicy() + policy.MaxReplicas = 7 + policy.AllowFleetCapacity = true + state := State{HighWindows: 2, LastAction: time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC)} + if err := store.SetPolicy("shop", policy); err != nil { + t.Fatal(err) + } + if err := store.SetState("shop", state); err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + + store, err = NewStore(dir) + if err != nil { + t.Fatal(err) + } + defer store.Close() + savedPolicy, err := store.Policy("shop") + if err != nil { + t.Fatal(err) + } + savedState, err := store.State("shop") + if err != nil { + t.Fatal(err) + } + if savedPolicy.MaxReplicas != 7 || !savedPolicy.AllowFleetCapacity { + t.Fatalf("unexpected policy: %+v", savedPolicy) + } + if savedState.HighWindows != 2 || !savedState.LastAction.Equal(state.LastAction) { + t.Fatalf("unexpected state: %+v", savedState) + } +} + +func TestStoreRejectsInvalidPolicy(t *testing.T) { + store, err := NewStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer store.Close() + policy := DefaultPolicy() + policy.MaxReplicas = 0 + if err := store.SetPolicy("shop", policy); err == nil { + t.Fatal("expected invalid policy error") + } +} From 758bab0a828f0a382ae645c0903d369c7894d03a Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 14:51:03 +0100 Subject: [PATCH 19/46] feat(cluster): Apply autoscaled Nginx routes Fleet can publish and remove validated replica routes through the existing Nginx manager, including configuration checks and reloads before traffic changes take effect. --- internal/routing/adapters.go | 5 ++- internal/routing/nginx_writer.go | 62 +++++++++++++++++++++++++++ internal/routing/nginx_writer_test.go | 55 ++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 internal/routing/nginx_writer.go create mode 100644 internal/routing/nginx_writer_test.go diff --git a/internal/routing/adapters.go b/internal/routing/adapters.go index e522dd2..0f3ab11 100644 --- a/internal/routing/adapters.go +++ b/internal/routing/adapters.go @@ -39,8 +39,8 @@ func (p *routeProvider) ID() ProviderID { } func (p *routeProvider) Validate(_ context.Context, route Route) error { - if strings.TrimSpace(route.ID) == "" { - return fmt.Errorf("Route ID is required") + if !safeRouteID.MatchString(route.ID) { + return fmt.Errorf("Route ID is invalid") } if strings.TrimSpace(route.Domain) == "" || strings.ContainsAny(route.Domain, " /\\") { return fmt.Errorf("Route domain is invalid") @@ -125,6 +125,7 @@ func (p *routeProvider) Remove(ctx context.Context, routeID string) error { var safeID = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`) var safeHost = regexp.MustCompile(`^[a-zA-Z0-9.-]+$`) +var safeRouteID = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_.-]*$`) func renderNginx(route Route) ([]byte, error) { name := "flatrun_" + safeID.ReplaceAllString(route.ID, "_") diff --git a/internal/routing/nginx_writer.go b/internal/routing/nginx_writer.go new file mode 100644 index 0000000..6e1881d --- /dev/null +++ b/internal/routing/nginx_writer.go @@ -0,0 +1,62 @@ +package routing + +import ( + "context" + "fmt" +) + +type NginxManager interface { + WriteVirtualHost(string, string) error + DeleteVirtualHost(string) error + TestConfig() error + Reload() error +} + +type NginxWriter struct { + manager NginxManager +} + +func NewNginxWriter(manager NginxManager) *NginxWriter { + return &NginxWriter{manager: manager} +} + +func (w *NginxWriter) Apply(ctx context.Context, id, provider string, content []byte) error { + if err := ctx.Err(); err != nil { + return err + } + if provider != string(ProviderNginx) { + return fmt.Errorf("Nginx writer cannot apply %q configuration", provider) + } + if !safeRouteID.MatchString(id) { + return fmt.Errorf("Route ID is invalid") + } + if err := w.manager.WriteVirtualHost(id, string(content)); err != nil { + return fmt.Errorf("write Nginx route: %w", err) + } + if err := w.manager.TestConfig(); err != nil { + return fmt.Errorf("test Nginx configuration: %w", err) + } + if err := w.manager.Reload(); err != nil { + return fmt.Errorf("reload Nginx: %w", err) + } + return nil +} + +func (w *NginxWriter) Remove(ctx context.Context, id string) error { + if err := ctx.Err(); err != nil { + return err + } + if !safeRouteID.MatchString(id) { + return fmt.Errorf("Route ID is invalid") + } + if err := w.manager.DeleteVirtualHost(id); err != nil { + return fmt.Errorf("delete Nginx route: %w", err) + } + if err := w.manager.TestConfig(); err != nil { + return fmt.Errorf("test Nginx configuration: %w", err) + } + if err := w.manager.Reload(); err != nil { + return fmt.Errorf("reload Nginx: %w", err) + } + return nil +} diff --git a/internal/routing/nginx_writer_test.go b/internal/routing/nginx_writer_test.go new file mode 100644 index 0000000..d55da75 --- /dev/null +++ b/internal/routing/nginx_writer_test.go @@ -0,0 +1,55 @@ +package routing + +import ( + "context" + "testing" +) + +type recordingNginx struct { + written string + deleted string + tested int + reloaded int +} + +func (m *recordingNginx) WriteVirtualHost(id, content string) error { + m.written = id + ":" + content + return nil +} + +func (m *recordingNginx) DeleteVirtualHost(id string) error { + m.deleted = id + return nil +} + +func (m *recordingNginx) TestConfig() error { + m.tested++ + return nil +} + +func (m *recordingNginx) Reload() error { + m.reloaded++ + return nil +} + +func TestNginxWriterAppliesAndReloadsRoute(t *testing.T) { + manager := &recordingNginx{} + writer := NewNginxWriter(manager) + if err := writer.Apply(context.Background(), "shop", "nginx", []byte("upstream shop {}")); err != nil { + t.Fatal(err) + } + if manager.written != "shop:upstream shop {}" || manager.tested != 1 || manager.reloaded != 1 { + t.Fatalf("unexpected manager calls: %+v", manager) + } +} + +func TestNginxWriterRejectsUnsafeRouteID(t *testing.T) { + manager := &recordingNginx{} + writer := NewNginxWriter(manager) + if err := writer.Apply(context.Background(), "../shop", "nginx", []byte("route")); err == nil { + t.Fatal("expected invalid route error") + } + if manager.written != "" { + t.Fatal("unsafe route was written") + } +} From 516e713bcf813b1febd89a7b047a3364bc2cec78 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 14:52:02 +0100 Subject: [PATCH 20/46] feat(capacity): Reconcile scaling observations Autoscaling evaluations now persist their state, execute safe decisions, and group blocked or failed actions into one deployment incident. --- internal/autoscale/runner.go | 83 +++++++++++++++++++++++++++++++ internal/autoscale/runner_test.go | 73 +++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 internal/autoscale/runner.go create mode 100644 internal/autoscale/runner_test.go diff --git a/internal/autoscale/runner.go b/internal/autoscale/runner.go new file mode 100644 index 0000000..d6cdb0f --- /dev/null +++ b/internal/autoscale/runner.go @@ -0,0 +1,83 @@ +package autoscale + +import ( + "context" + "fmt" + "time" + + "github.com/flatrun/agent/internal/events" + "github.com/flatrun/agent/internal/routing" +) + +type PolicyStore interface { + Policy(string) (Policy, error) + State(string) (State, error) + SetState(string, State) error +} + +type ActionExecutor interface { + Execute(context.Context, string, routing.Route, Decision) (Execution, error) +} + +type EventPublisher interface { + Publish(events.Event) (events.IngestResult, error) +} + +type Runner struct { + store PolicyStore + executor ActionExecutor + publisher EventPublisher + node string +} + +type ReconcileResult struct { + State State `json:"state"` + Decision Decision `json:"decision"` + Execution *Execution `json:"execution,omitempty"` +} + +func NewRunner(store PolicyStore, executor ActionExecutor, publisher EventPublisher, node string) *Runner { + return &Runner{store: store, executor: executor, publisher: publisher, node: node} +} + +func (r *Runner) Reconcile(ctx context.Context, deployment string, input Input, route routing.Route) (ReconcileResult, error) { + policy, err := r.store.Policy(deployment) + if err != nil { + return ReconcileResult{}, fmt.Errorf("load autoscaling policy: %w", err) + } + state, err := r.store.State(deployment) + if err != nil { + return ReconcileResult{}, fmt.Errorf("load autoscaling state: %w", err) + } + nextState, decision := Reconcile(policy, state, input) + if err := r.store.SetState(deployment, nextState); err != nil { + return ReconcileResult{}, fmt.Errorf("save autoscaling state: %w", err) + } + result := ReconcileResult{State: nextState, Decision: decision} + if decision.Action == ActionNone { + return result, nil + } + if decision.Action == ActionNotify { + r.publishFailure(deployment, decision.Reason, events.SeverityWarning) + return result, nil + } + execution, err := r.executor.Execute(ctx, deployment, route, decision) + result.Execution = &execution + if err != nil { + r.publishFailure(deployment, err.Error(), events.SeverityCritical) + return result, fmt.Errorf("execute autoscaling decision: %w", err) + } + return result, nil +} + +func (r *Runner) publishFailure(deployment, message string, severity events.Severity) { + if r.publisher == nil { + return + } + _, _ = r.publisher.Publish(events.Event{ + Source: "capacity", Type: "autoscale.blocked", Severity: severity, + Title: "Autoscaling needs attention", Message: message, + Scope: events.Scope{Node: r.node, Deployment: deployment}, + CorrelationKey: "autoscale:" + r.node + ":" + deployment, OccurredAt: time.Now(), + }) +} diff --git a/internal/autoscale/runner_test.go b/internal/autoscale/runner_test.go new file mode 100644 index 0000000..87f9c91 --- /dev/null +++ b/internal/autoscale/runner_test.go @@ -0,0 +1,73 @@ +package autoscale + +import ( + "context" + "testing" + "time" + + "github.com/flatrun/agent/internal/capacity" + "github.com/flatrun/agent/internal/events" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" +) + +type runnerStore struct { + policy Policy + state State +} + +func (s *runnerStore) Policy(string) (Policy, error) { return s.policy, nil } +func (s *runnerStore) State(string) (State, error) { return s.state, nil } +func (s *runnerStore) SetState(_ string, state State) error { s.state = state; return nil } + +type runnerExecutor struct { + decision Decision +} + +func (e *runnerExecutor) Execute(_ context.Context, _ string, _ routing.Route, decision Decision) (Execution, error) { + e.decision = decision + return Execution{Decision: decision, Status: orchestrator.Status{Desired: decision.Replicas}}, nil +} + +type runnerPublisher struct { + events []events.Event +} + +func (p *runnerPublisher) Publish(event events.Event) (events.IngestResult, error) { + p.events = append(p.events, event) + return events.IngestResult{}, nil +} + +func TestRunnerPersistsObservationAndExecutesDecision(t *testing.T) { + policy := DefaultPolicy() + policy.ScaleUpWindows = 1 + store := &runnerStore{policy: policy} + executor := &runnerExecutor{} + runner := NewRunner(store, executor, nil, "prod-1") + result, err := runner.Reconcile(context.Background(), "shop", Input{ + Now: time.Now(), Replicas: 1, CPUPercent: 90, + Diagnosis: capacity.Diagnosis{Action: capacity.ActionAddReplica}, + }, routing.Route{ID: "shop"}) + if err != nil { + t.Fatal(err) + } + if result.Decision.Action != ActionAddReplica || executor.decision.Replicas != 2 || store.state.HighWindows != 0 { + t.Fatalf("unexpected reconciliation: %+v", result) + } +} + +func TestRunnerPublishesCorrelatedBlockedEvent(t *testing.T) { + policy := DefaultPolicy() + policy.MaxReplicas = 1 + policy.ScaleUpWindows = 1 + store := &runnerStore{policy: policy} + publisher := &runnerPublisher{} + runner := NewRunner(store, &runnerExecutor{}, publisher, "prod-1") + _, err := runner.Reconcile(context.Background(), "shop", Input{Now: time.Now(), Replicas: 1, CPUPercent: 95}, routing.Route{ID: "shop"}) + if err != nil { + t.Fatal(err) + } + if len(publisher.events) != 1 || publisher.events[0].CorrelationKey != "autoscale:prod-1:shop" { + t.Fatalf("unexpected events: %+v", publisher.events) + } +} From e67e358e7098c2ff7d8448277a049365d962fda2 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 15:13:20 +0100 Subject: [PATCH 21/46] feat(system): Report agent URL Server Info now reports the stable agent address used by Fleet. Before setup, it derives the address from the authenticated request so setup can offer an editable default. --- internal/api/server_info_handlers.go | 30 +++++++++++++++++++++++ internal/api/server_info_handlers_test.go | 26 ++++++++++++++++++++ internal/system/network.go | 1 + 3 files changed, 57 insertions(+) diff --git a/internal/api/server_info_handlers.go b/internal/api/server_info_handlers.go index 9663b05..b863b91 100644 --- a/internal/api/server_info_handlers.go +++ b/internal/api/server_info_handlers.go @@ -2,6 +2,7 @@ package api import ( "net/http" + "strings" "github.com/flatrun/agent/internal/system" "github.com/gin-gonic/gin" @@ -15,12 +16,41 @@ func (s *Server) getServerInfo(c *gin.Context) { }) return } + info.AgentURL = s.agentURL(c) c.JSON(http.StatusOK, gin.H{ "server": info, }) } +func (s *Server) agentURL(c *gin.Context) string { + if configured := strings.TrimRight(strings.TrimSpace(s.config.Cluster.AdvertiseURL), "/"); configured != "" { + return configured + } + + scheme := "http" + if c.Request.TLS != nil { + scheme = "https" + } + if forwarded := firstForwardedValue(c.GetHeader("X-Forwarded-Proto")); forwarded != "" { + scheme = forwarded + } + + host := c.Request.Host + if forwarded := firstForwardedValue(c.GetHeader("X-Forwarded-Host")); forwarded != "" { + host = forwarded + } + if host == "" { + return "" + } + return scheme + "://" + host +} + +func firstForwardedValue(value string) string { + value, _, _ = strings.Cut(value, ",") + return strings.TrimSpace(value) +} + func (s *Server) getNetworkHealth(c *gin.Context) { health, err := system.CheckNetworkHealth(c.Request.Context()) if err != nil { diff --git a/internal/api/server_info_handlers_test.go b/internal/api/server_info_handlers_test.go index 41860b2..2dd9a53 100644 --- a/internal/api/server_info_handlers_test.go +++ b/internal/api/server_info_handlers_test.go @@ -101,6 +101,32 @@ func TestGetServerInfo(t *testing.T) { if _, ok := serverMap["interfaces"]; !ok { t.Error("server should have interfaces") } + if got := serverMap["agent_url"]; got != "http://example.com" { + t.Errorf("agent_url = %q, want %q", got, "http://example.com") + } +} + +func TestGetServerInfoUsesForwardedAgentURL(t *testing.T) { + router, token, cleanup := setupServerInfoTestServer(t) + defer cleanup() + + req := httptest.NewRequest(http.MethodGet, "/api/server/info", nil) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("X-Forwarded-Proto", "https") + req.Header.Set("X-Forwarded-Host", "agent.example.com") + + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + var resp struct { + Server system.ServerInfo `json:"server"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("Failed to parse response: %v", err) + } + if resp.Server.AgentURL != "https://agent.example.com" { + t.Errorf("agent_url = %q, want %q", resp.Server.AgentURL, "https://agent.example.com") + } } func TestGetServerInfoRequiresAuth(t *testing.T) { diff --git a/internal/system/network.go b/internal/system/network.go index 29f819a..7af65c1 100644 --- a/internal/system/network.go +++ b/internal/system/network.go @@ -38,6 +38,7 @@ type NetworkInterface struct { type ServerInfo struct { Hostname string `json:"hostname"` + AgentURL string `json:"agent_url"` PublicIPv4 string `json:"public_ipv4"` PublicIPv6 string `json:"public_ipv6"` Interfaces []NetworkInterface `json:"interfaces"` From 118157f93bb97323027a5ff6ea19a1cf3a3eecab Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 15:27:07 +0100 Subject: [PATCH 22/46] fix(notifications): Test saved delivery targets Saved targets can be tested without exposing their connection URL. Responses identify the delivery type so clients can preserve credentials during edits. --- internal/api/notifications_test.go | 24 ++++++++++++++++++++++++ internal/api/server.go | 11 +++++++++-- internal/notify/notify.go | 28 +++++++++++++++++++++++++++- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/internal/api/notifications_test.go b/internal/api/notifications_test.go index c39932e..9f45026 100644 --- a/internal/api/notifications_test.go +++ b/internal/api/notifications_test.go @@ -27,6 +27,7 @@ func setupNotifyTest(t *testing.T) (*Server, *gin.Engine) { r.GET("/notifications/rules", s.listNotificationRules) r.PUT("/notifications/rules", s.updateNotificationRules) r.PUT("/notifications/targets", s.updateNotificationTargets) + r.POST("/notifications/test", s.testNotification) r.POST("/internal/notify/emit", s.emitNotification) r.POST("/internal/events", s.emitEvent) return s, r @@ -98,6 +99,29 @@ func TestGetNotificationTargetsMasksSecret(t *testing.T) { if !strings.Contains(body, notify.MaskedURL) { t.Errorf("response should mask the target URL: %s", body) } + if !strings.Contains(body, `"kind":"email"`) { + t.Errorf("response should identify the safe target kind: %s", body) + } +} + +func TestNotificationTargetByIDThroughHTTP(t *testing.T) { + s, r := setupNotifyTest(t) + if err := s.notify.Save(notify.Config{Targets: []notify.Target{ + {ID: "ops", Name: "Operations", URL: "generic+https://example.com/hook", Enabled: false}, + }}); err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/notifications/test", strings.NewReader(`{"target_id":"ops"}`)) + req.Header.Set("Content-Type", "application/json") + r.ServeHTTP(w, req) + + if w.Code != http.StatusBadGateway { + t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "target is disabled") { + t.Fatalf("saved target was not selected: %s", w.Body.String()) + } } func TestUpdateNotificationTargetsPreservesMaskedSecret(t *testing.T) { diff --git a/internal/api/server.go b/internal/api/server.go index 10c85f9..2e506e9 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -3439,13 +3439,20 @@ func (s *Server) updateNotificationTargets(c *gin.Context) { func (s *Server) testNotification(c *gin.Context) { var req struct { - URL string `json:"url"` + URL string `json:"url"` + TargetID string `json:"target_id"` } if err := c.ShouldBindJSON(&req); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request"}) return } - if err := s.notify.Test(req.URL); err != nil { + var err error + if req.TargetID != "" { + err = s.notify.TestTarget(req.TargetID) + } else { + err = s.notify.Test(req.URL) + } + if err != nil { c.JSON(http.StatusBadGateway, gin.H{"error": "delivery failed: " + err.Error()}) return } diff --git a/internal/notify/notify.go b/internal/notify/notify.go index e5a9be2..304ca8a 100644 --- a/internal/notify/notify.go +++ b/internal/notify/notify.go @@ -46,7 +46,21 @@ func (t Target) MarshalJSON() ([]byte, error) { if masked.URL != "" { masked.URL = MaskedURL } - return json.Marshal(masked) + return json.Marshal(struct { + alias + Kind string `json:"kind"` + }{alias: masked, Kind: targetKind(t.URL)}) +} + +func targetKind(rawURL string) string { + switch { + case strings.HasPrefix(rawURL, "smtp://"): + return "email" + case strings.HasPrefix(rawURL, "generic+"): + return "webhook" + default: + return "custom" + } } // Config is the persisted notification settings. @@ -290,6 +304,18 @@ func (s *Service) NotifyTargets(title, message string, ids []string) error { return s.NotifyNotificationTargets(Notification{Title: title, Message: message}, ids) } +func (s *Service) TestTarget(id string) error { + for _, target := range s.Load().Targets { + if target.ID == id { + if !target.Enabled { + return fmt.Errorf("target is disabled") + } + return s.deliver(target.URL, Notification{Title: "FlatRun test", Message: "Notification delivery is working."}) + } + } + return fmt.Errorf("target not found") +} + func (s *Service) NotifyNotificationTargets(notification Notification, ids []string) error { cfg := s.Load() var only map[string]bool From 061d3982a5cf4be997e1ddeba23f779f4e9ada7a Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 16:17:56 +0100 Subject: [PATCH 23/46] feat(cluster): Add K3s orchestration adapter Fleet operators can select K3s for stateless workload placement and scaling. Configured cluster credentials and namespaces are used for readiness checks and workload operations. --- internal/api/cluster_handlers.go | 3 +- internal/orchestrator/k3s.go | 228 ++++++++++++++++++++++++++++++ internal/orchestrator/k3s_test.go | 74 ++++++++++ pkg/config/config.go | 20 ++- 4 files changed, 317 insertions(+), 8 deletions(-) create mode 100644 internal/orchestrator/k3s.go create mode 100644 internal/orchestrator/k3s_test.go diff --git a/internal/api/cluster_handlers.go b/internal/api/cluster_handlers.go index f173104..3e2d19d 100644 --- a/internal/api/cluster_handlers.go +++ b/internal/api/cluster_handlers.go @@ -95,7 +95,8 @@ func (s *Server) checkOrchestrator(ctx context.Context, id orchestrator.Provider defer provider.Close() return provider.Ready(ctx) case orchestrator.ProviderK3s: - return fmt.Errorf("k3s adapter is not configured") + provider := orchestrator.NewK3sProvider(s.config.Cluster.K3s.Kubeconfig, s.config.Cluster.K3s.Namespace) + return provider.Ready(ctx) default: return fmt.Errorf("orchestrator %q is not supported", id) } diff --git a/internal/orchestrator/k3s.go b/internal/orchestrator/k3s.go new file mode 100644 index 0000000..ba9a96c --- /dev/null +++ b/internal/orchestrator/k3s.go @@ -0,0 +1,228 @@ +package orchestrator + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net" + "os/exec" + "strconv" + "strings" +) + +type kubectlRunner interface { + Run(context.Context, []byte, ...string) ([]byte, error) +} + +type commandKubectl struct { + binary string +} + +func (r commandKubectl) Run(ctx context.Context, input []byte, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, r.binary, args...) + cmd.Stdin = bytes.NewReader(input) + return cmd.CombinedOutput() +} + +type K3sProvider struct { + runner kubectlRunner + kubeconfig string + namespace string +} + +func NewK3sProvider(kubeconfig, namespace string) *K3sProvider { + if namespace == "" { + namespace = "default" + } + return &K3sProvider{ + runner: commandKubectl{binary: "kubectl"}, + kubeconfig: strings.TrimSpace(kubeconfig), + namespace: strings.TrimSpace(namespace), + } +} + +func (p *K3sProvider) ID() ProviderID { return ProviderK3s } + +func (p *K3sProvider) Ready(ctx context.Context) error { + if _, err := p.run(ctx, nil, "cluster-info"); err != nil { + return fmt.Errorf("K3s is not available: %w", err) + } + return nil +} + +func (p *K3sProvider) Validate(_ context.Context, workload Workload) error { + if strings.TrimSpace(workload.ID) == "" { + return fmt.Errorf("Workload ID is required") + } + if strings.TrimSpace(workload.Image) == "" { + return fmt.Errorf("Workload image is required") + } + if workload.Replicas < 1 { + return fmt.Errorf("Replicas must be at least one") + } + if workload.Stateful && workload.Replicas > 1 { + return fmt.Errorf("Stateful workloads cannot use multiple replicas without a storage policy") + } + if workload.Port < 0 || workload.Port > 65535 { + return fmt.Errorf("Workload port is invalid") + } + return nil +} + +func (p *K3sProvider) Apply(ctx context.Context, workload Workload) (Status, error) { + if err := p.Validate(ctx, workload); err != nil { + return Status{}, err + } + manifest, err := json.Marshal(k3sManifest(workload)) + if err != nil { + return Status{}, err + } + if _, err := p.run(ctx, manifest, "apply", "-f", "-"); err != nil { + return Status{}, fmt.Errorf("apply K3s workload: %w", err) + } + return p.Status(ctx, workload.ID) +} + +func (p *K3sProvider) Resize(ctx context.Context, id string, resources Resources) (Status, error) { + patch := map[string]any{"spec": map[string]any{"template": map[string]any{"spec": map[string]any{"containers": []any{map[string]any{ + "name": id, "resources": k3sResources(resources), + }}}}}} + encoded, _ := json.Marshal(patch) + if _, err := p.run(ctx, nil, "patch", "deployment", id, "--type", "merge", "-p", string(encoded)); err != nil { + return Status{}, fmt.Errorf("resize K3s workload: %w", err) + } + return p.Status(ctx, id) +} + +func (p *K3sProvider) Scale(ctx context.Context, id string, replicas int) (Status, error) { + if replicas < 0 { + return Status{}, fmt.Errorf("Replicas cannot be negative") + } + if _, err := p.run(ctx, nil, "scale", "deployment", id, "--replicas", strconv.Itoa(replicas)); err != nil { + return Status{}, fmt.Errorf("scale K3s workload: %w", err) + } + return p.Status(ctx, id) +} + +func (p *K3sProvider) Status(ctx context.Context, id string) (Status, error) { + deploymentRaw, err := p.run(ctx, nil, "get", "deployment", id, "-o", "json") + if err != nil { + return Status{}, fmt.Errorf("inspect K3s workload: %w", err) + } + var deployment struct { + Metadata struct { + Labels map[string]string `json:"labels"` + } `json:"metadata"` + Spec struct { + Replicas int `json:"replicas"` + } `json:"spec"` + Status struct { + Available int `json:"availableReplicas"` + } `json:"status"` + } + if err := json.Unmarshal(deploymentRaw, &deployment); err != nil { + return Status{}, fmt.Errorf("decode K3s workload: %w", err) + } + podsRaw, err := p.run(ctx, nil, "get", "pods", "-l", "flatrun.workload="+id, "-o", "json") + if err != nil { + return Status{}, fmt.Errorf("list K3s workload pods: %w", err) + } + var pods struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + Node string `json:"nodeName"` + } `json:"spec"` + Status struct { + IP string `json:"podIP"` + Phase string `json:"phase"` + Conditions []struct{ Type, Status string } `json:"conditions"` + } `json:"status"` + } `json:"items"` + } + if err := json.Unmarshal(podsRaw, &pods); err != nil { + return Status{}, fmt.Errorf("decode K3s workload pods: %w", err) + } + status := Status{Workload: id, Desired: deployment.Spec.Replicas, Available: deployment.Status.Available} + port := deployment.Metadata.Labels["flatrun.port"] + for _, pod := range pods.Items { + ready := false + for _, condition := range pod.Status.Conditions { + if condition.Type == "Ready" && condition.Status == "True" { + ready = true + } + } + address := "" + if pod.Status.IP != "" && port != "" { + address = net.JoinHostPort(pod.Status.IP, port) + } + status.Instances = append(status.Instances, Instance{ID: pod.Metadata.Name, Node: pod.Spec.Node, Address: address, Healthy: pod.Status.Phase == "Running", Ready: ready}) + } + return status, nil +} + +func (p *K3sProvider) Remove(ctx context.Context, id string) error { + if _, err := p.run(ctx, nil, "delete", "deployment", id, "--ignore-not-found=true"); err != nil { + return fmt.Errorf("remove K3s workload: %w", err) + } + return nil +} + +func (p *K3sProvider) run(ctx context.Context, input []byte, args ...string) ([]byte, error) { + base := make([]string, 0, len(args)+4) + if p.kubeconfig != "" { + base = append(base, "--kubeconfig", p.kubeconfig) + } + base = append(base, "--namespace", p.namespace) + output, err := p.runner.Run(ctx, input, append(base, args...)...) + if err != nil { + message := strings.TrimSpace(string(output)) + if message != "" { + return nil, fmt.Errorf("%s", message) + } + return nil, err + } + return output, nil +} + +func k3sManifest(workload Workload) map[string]any { + labels := map[string]string{"app.kubernetes.io/name": workload.ID, "flatrun.workload": workload.ID} + for key, value := range workload.Labels { + labels[key] = value + } + if workload.Port > 0 { + labels["flatrun.port"] = strconv.Itoa(workload.Port) + } + container := map[string]any{"name": workload.ID, "image": workload.Image, "resources": k3sResources(workload.Resources)} + if workload.Port > 0 { + container["ports"] = []any{map[string]any{"containerPort": workload.Port}} + } + return map[string]any{ + "apiVersion": "apps/v1", "kind": "Deployment", + "metadata": map[string]any{"name": workload.ID, "labels": labels}, + "spec": map[string]any{"replicas": workload.Replicas, "selector": map[string]any{"matchLabels": map[string]string{"flatrun.workload": workload.ID}}, "template": map[string]any{ + "metadata": map[string]any{"labels": labels}, "spec": map[string]any{"containers": []any{container}}, + }}, + } +} + +func k3sResources(resources Resources) map[string]any { + requests := map[string]string{} + limits := map[string]string{} + if resources.CPURequest > 0 { + requests["cpu"] = fmt.Sprintf("%gm", resources.CPURequest*1000) + } + if resources.MemoryRequest > 0 { + requests["memory"] = strconv.FormatUint(resources.MemoryRequest, 10) + } + if resources.CPULimit > 0 { + limits["cpu"] = fmt.Sprintf("%gm", resources.CPULimit*1000) + } + if resources.MemoryLimit > 0 { + limits["memory"] = strconv.FormatUint(resources.MemoryLimit, 10) + } + return map[string]any{"requests": requests, "limits": limits} +} diff --git a/internal/orchestrator/k3s_test.go b/internal/orchestrator/k3s_test.go new file mode 100644 index 0000000..9f628b2 --- /dev/null +++ b/internal/orchestrator/k3s_test.go @@ -0,0 +1,74 @@ +package orchestrator + +import ( + "context" + "encoding/json" + "reflect" + "testing" +) + +type kubectlCall struct { + input []byte + args []string +} + +type fakeKubectl struct { + calls []kubectlCall + responses [][]byte +} + +func (f *fakeKubectl) Run(_ context.Context, input []byte, args ...string) ([]byte, error) { + f.calls = append(f.calls, kubectlCall{input: input, args: append([]string(nil), args...)}) + if len(f.responses) == 0 { + return nil, nil + } + response := f.responses[0] + f.responses = f.responses[1:] + return response, nil +} + +func TestK3sApplyUsesConfiguredClusterAndNamespace(t *testing.T) { + runner := &fakeKubectl{responses: [][]byte{ + nil, + []byte(`{"metadata":{"labels":{"flatrun.port":"8080"}},"spec":{"replicas":2},"status":{"availableReplicas":2}}`), + []byte(`{"items":[]}`), + }} + provider := NewK3sProvider("/etc/rancher/k3s.yaml", "apps") + provider.runner = runner + + status, err := provider.Apply(context.Background(), Workload{ID: "shop", Image: "shop:1", Port: 8080, Replicas: 2}) + if err != nil { + t.Fatal(err) + } + if status.Desired != 2 || status.Available != 2 { + t.Fatalf("status = %#v", status) + } + want := []string{"--kubeconfig", "/etc/rancher/k3s.yaml", "--namespace", "apps", "apply", "-f", "-"} + if !reflect.DeepEqual(runner.calls[0].args, want) { + t.Fatalf("apply args = %#v", runner.calls[0].args) + } + var manifest map[string]any + if err := json.Unmarshal(runner.calls[0].input, &manifest); err != nil { + t.Fatal(err) + } + if manifest["kind"] != "Deployment" { + t.Fatalf("manifest = %#v", manifest) + } +} + +func TestK3sStatusReturnsRoutableReadyPods(t *testing.T) { + runner := &fakeKubectl{responses: [][]byte{ + []byte(`{"metadata":{"labels":{"flatrun.port":"8080"}},"spec":{"replicas":1},"status":{"availableReplicas":1}}`), + []byte(`{"items":[{"metadata":{"name":"shop-a"},"spec":{"nodeName":"prod2"},"status":{"podIP":"10.42.0.8","phase":"Running","conditions":[{"type":"Ready","status":"True"}]}}]}`), + }} + provider := NewK3sProvider("", "apps") + provider.runner = runner + + status, err := provider.Status(context.Background(), "shop") + if err != nil { + t.Fatal(err) + } + if len(status.Instances) != 1 || status.Instances[0].Address != "10.42.0.8:8080" || !status.Instances[0].Ready { + t.Fatalf("status = %#v", status) + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 211d1c2..d2a36f2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -14,13 +14,19 @@ import ( ) type ClusterConfig struct { - Enabled bool `yaml:"enabled"` - ServerName string `yaml:"server_name"` - AdvertiseURL string `yaml:"advertise_url"` - HealthInterval string `yaml:"health_interval"` - RequestTimeout string `yaml:"request_timeout"` - Orchestrator string `yaml:"orchestrator" json:"orchestrator"` - Routing string `yaml:"routing" json:"routing"` + Enabled bool `yaml:"enabled"` + ServerName string `yaml:"server_name"` + AdvertiseURL string `yaml:"advertise_url"` + HealthInterval string `yaml:"health_interval"` + RequestTimeout string `yaml:"request_timeout"` + Orchestrator string `yaml:"orchestrator" json:"orchestrator"` + Routing string `yaml:"routing" json:"routing"` + K3s K3sConfig `yaml:"k3s" json:"k3s"` +} + +type K3sConfig struct { + Kubeconfig string `yaml:"kubeconfig" json:"kubeconfig"` + Namespace string `yaml:"namespace" json:"namespace"` } type CapacityConfig struct { From 27518cbab2185e2bc90132820a9407a28789a1ef Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 16:19:53 +0100 Subject: [PATCH 24/46] feat(notifications): Group Fleet node outages Fleet health transitions now open one node incident, suppress repeated outage alerts, and resolve the same incident when the node recovers. Related deployment failures remain grouped under that incident. --- internal/api/server.go | 3 ++ internal/cluster/manager.go | 47 +++++++++++++++++++++++++ internal/cluster/manager_events_test.go | 26 ++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 internal/cluster/manager_events_test.go diff --git a/internal/api/server.go b/internal/api/server.go index 2e506e9..be77fad 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -342,6 +342,9 @@ func New(cfg *config.Config, configPath string) *Server { } } } + if clusterManager != nil { + clusterManager.SetEventPublisher(notifyService) + } s := &Server{ config: cfg, diff --git a/internal/cluster/manager.go b/internal/cluster/manager.go index 1d8cb6f..2867517 100644 --- a/internal/cluster/manager.go +++ b/internal/cluster/manager.go @@ -12,8 +12,14 @@ import ( "log" "sync" "time" + + "github.com/flatrun/agent/internal/events" ) +type EventPublisher interface { + Publish(events.Event) (events.IngestResult, error) +} + type PeerStatus struct { Name string `json:"name"` URL string `json:"url"` @@ -36,6 +42,7 @@ type Manager struct { healthInterval time.Duration requestTimeout time.Duration encryptionKey []byte + publisher EventPublisher cancel context.CancelFunc } @@ -52,6 +59,12 @@ func NewManager(db *DB, serverName string, healthInterval, requestTimeout time.D } } +func (m *Manager) SetEventPublisher(publisher EventPublisher) { + m.mu.Lock() + m.publisher = publisher + m.mu.Unlock() +} + func (m *Manager) Start(ctx context.Context) error { ctx, m.cancel = context.WithCancel(ctx) @@ -128,6 +141,9 @@ func (m *Manager) checkAllPeers(ctx context.Context) { m.mu.Lock() st, exists := m.status[p.name] + wasOnline := exists && st.Online + wasKnown := exists && (st.Online || st.Error != "" || !st.LastSeen.IsZero()) + publisher := m.publisher if exists { if err != nil { st.Online = false @@ -140,6 +156,14 @@ func (m *Manager) checkAllPeers(ctx context.Context) { } } m.mu.Unlock() + + event := fleetHealthEvent(p.name, wasKnown, wasOnline, err) + if publisher == nil || event == nil { + continue + } + if _, publishErr := publisher.Publish(*event); publishErr != nil { + log.Printf("Warning: Failed to publish Fleet health event for %s: %v", p.name, publishErr) + } } for _, name := range seenNames { @@ -147,6 +171,29 @@ func (m *Manager) checkAllPeers(ctx context.Context) { } } +func fleetHealthEvent(name string, wasKnown, wasOnline bool, healthErr error) *events.Event { + isOnline := healthErr == nil + if (!wasKnown && isOnline) || (wasKnown && wasOnline == isOnline) { + return nil + } + event := &events.Event{ + Source: "fleet", Scope: events.Scope{Node: name}, CorrelationKey: "node:" + name, OccurredAt: time.Now(), + } + if healthErr != nil { + event.Type = "node.unavailable" + event.Severity = events.SeverityCritical + event.Title = name + " is unavailable" + event.Message = "The Fleet node stopped responding. Related deployment failures will be grouped into this incident." + return event + } + event.Type = "node.recovered" + event.Severity = events.SeverityInfo + event.Title = name + " recovered" + event.Message = "The Fleet node is responding again." + event.Resolved = true + return event +} + func (m *Manager) GetPeer(name string) (*Client, error) { m.mu.RLock() defer m.mu.RUnlock() diff --git a/internal/cluster/manager_events_test.go b/internal/cluster/manager_events_test.go new file mode 100644 index 0000000..335854f --- /dev/null +++ b/internal/cluster/manager_events_test.go @@ -0,0 +1,26 @@ +package cluster + +import ( + "errors" + "testing" +) + +func TestFleetHealthEventOpensAndResolvesOneNodeIncident(t *testing.T) { + failed := fleetHealthEvent("prod2", false, false, errors.New("connection refused")) + if failed == nil || failed.Type != "node.unavailable" || failed.CorrelationKey != "node:prod2" || failed.Resolved { + t.Fatalf("failed event = %#v", failed) + } + if repeated := fleetHealthEvent("prod2", true, false, errors.New("connection refused")); repeated != nil { + t.Fatalf("repeated event = %#v", repeated) + } + recovered := fleetHealthEvent("prod2", true, false, nil) + if recovered == nil || recovered.Type != "node.recovered" || recovered.CorrelationKey != failed.CorrelationKey || !recovered.Resolved { + t.Fatalf("recovered event = %#v", recovered) + } +} + +func TestFleetHealthEventDoesNotAnnounceInitialSuccess(t *testing.T) { + if event := fleetHealthEvent("prod2", false, false, nil); event != nil { + t.Fatalf("event = %#v", event) + } +} From 73459431a7fc3950d8bf324d11ec207962f61987 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 16:21:06 +0100 Subject: [PATCH 25/46] feat(cluster): Configure K3s connections Fleet provider setup can now save and report the K3s cluster context required for readiness checks and workload operations. --- internal/api/cluster_handlers.go | 19 +++++++++++++++---- internal/api/cluster_handlers_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/internal/api/cluster_handlers.go b/internal/api/cluster_handlers.go index 3e2d19d..fe7e74e 100644 --- a/internal/api/cluster_handlers.go +++ b/internal/api/cluster_handlers.go @@ -35,6 +35,7 @@ type clusterProviderOption struct { type clusterProvidersResponse struct { Orchestrators []clusterProviderOption `json:"orchestrators"` Routing []clusterProviderOption `json:"routing"` + K3s config.K3sConfig `json:"k3s"` } func (s *Server) clusterProviders(c *gin.Context) { @@ -57,6 +58,7 @@ func (s *Server) clusterProviders(c *gin.Context) { s.routingOption(c, routing.ProviderNginx, routingID), s.routingOption(c, routing.ProviderTraefik, routingID), }, + K3s: s.config.Cluster.K3s, }) } @@ -120,8 +122,9 @@ func (s *Server) checkRouting(ctx context.Context, id routing.ProviderID) error } type updateClusterProvidersRequest struct { - Orchestrator string `json:"orchestrator" binding:"required"` - Routing string `json:"routing" binding:"required"` + Orchestrator string `json:"orchestrator" binding:"required"` + Routing string `json:"routing" binding:"required"` + K3s config.K3sConfig `json:"k3s"` } func (s *Server) updateClusterProviders(c *gin.Context) { @@ -132,8 +135,14 @@ func (s *Server) updateClusterProviders(c *gin.Context) { } orchestratorID := orchestrator.ProviderID(strings.TrimSpace(req.Orchestrator)) routingID := routing.ProviderID(strings.TrimSpace(req.Routing)) - if err := s.checkOrchestrator(c, orchestratorID); err != nil { - c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()}) + var orchestratorErr error + if orchestratorID == orchestrator.ProviderK3s && s.probeOrchestrator == nil { + orchestratorErr = orchestrator.NewK3sProvider(req.K3s.Kubeconfig, req.K3s.Namespace).Ready(c) + } else { + orchestratorErr = s.checkOrchestrator(c, orchestratorID) + } + if orchestratorErr != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": orchestratorErr.Error()}) return } if err := s.checkRouting(c, routingID); err != nil { @@ -144,6 +153,7 @@ func (s *Server) updateClusterProviders(c *gin.Context) { previous := s.config.Cluster s.config.Cluster.Orchestrator = string(orchestratorID) s.config.Cluster.Routing = string(routingID) + s.config.Cluster.K3s = req.K3s if s.configPath != "" { if err := config.Save(s.config, s.configPath); err != nil { s.config.Cluster = previous @@ -154,6 +164,7 @@ func (s *Server) updateClusterProviders(c *gin.Context) { c.JSON(http.StatusOK, gin.H{ "orchestrator": s.config.Cluster.Orchestrator, "routing": s.config.Cluster.Routing, + "k3s": s.config.Cluster.K3s, }) } diff --git a/internal/api/cluster_handlers_test.go b/internal/api/cluster_handlers_test.go index 69e3dbb..7a6644b 100644 --- a/internal/api/cluster_handlers_test.go +++ b/internal/api/cluster_handlers_test.go @@ -817,6 +817,31 @@ func TestUpdateClusterProvidersPersistsAvailableSelection(t *testing.T) { } } +func TestUpdateClusterProvidersPersistsK3sConnection(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + env.server.probeOrchestrator = func(_ context.Context, _ orchestrator.ProviderID) error { return nil } + env.server.probeRouting = func(_ context.Context, _ routing.ProviderID) error { return nil } + + body := bytes.NewBufferString(`{"orchestrator":"k3s","routing":"nginx","k3s":{"kubeconfig":"/etc/rancher/k3s/k3s.yaml","namespace":"flatrun"}}`) + req := httptest.NewRequest(http.MethodPut, "/api/cluster/providers", body) + req.Header.Set("Authorization", "Bearer "+clusterLogin(t, env.router)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + + saved, err := config.Load(env.server.configPath) + if err != nil { + t.Fatal(err) + } + if saved.Cluster.K3s.Kubeconfig != "/etc/rancher/k3s/k3s.yaml" || saved.Cluster.K3s.Namespace != "flatrun" { + t.Fatalf("unexpected K3s connection: %+v", saved.Cluster.K3s) + } +} + func TestUpdateClusterProvidersRejectsUnavailableSelection(t *testing.T) { env := setupClusterTestServer(t, "server-a", true) defer env.cleanup() From bea26c893d1b85bbbfde57b957beb7df4b8a6802 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 16:30:07 +0100 Subject: [PATCH 26/46] feat(routing): Preserve deployment proxy during scaling Scaled replicas now use the existing deployment proxy configuration, retaining SSL, aliases, security controls, and request behavior. Removing managed routing restores the original Compose target. --- internal/nginx/manager.go | 94 +++++++++++++++-- internal/nginx/manager_test.go | 46 ++++++++- internal/routing/adapters.go | 4 + internal/routing/managed_nginx.go | 133 +++++++++++++++++++++++++ internal/routing/managed_nginx_test.go | 78 +++++++++++++++ internal/routing/provider.go | 1 + 6 files changed, 346 insertions(+), 10 deletions(-) create mode 100644 internal/routing/managed_nginx.go create mode 100644 internal/routing/managed_nginx_test.go diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index 5a6f6f8..c8dd6b8 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -4,10 +4,12 @@ import ( "bytes" "fmt" "log" + "net" "os" "os/exec" "path/filepath" "sort" + "strconv" "strings" "sync" "text/template" @@ -221,6 +223,61 @@ func (m *Manager) RenderVirtualHost(deployment *models.Deployment) (string, erro return m.generateMultiDomainConfig(deployment) } +type UpstreamBackend struct { + Address string + Healthy bool + Weight int +} + +func (m *Manager) RenderVirtualHostWithBackends(deployment *models.Deployment, backends map[string][]UpstreamBackend) (string, error) { + if deployment.Metadata == nil { + return "", fmt.Errorf("deployment has no metadata") + } + if len(deployment.Metadata.GetDomains()) == 0 { + return "", nil + } + if err := validateUpstreamBackends(backends); err != nil { + return "", err + } + m.mu.RLock() + defer m.mu.RUnlock() + return m.renderMultiDomainConfigWithBackends(deployment, m.keepaliveSupported(), backends) +} + +func validateUpstreamBackends(backends map[string][]UpstreamBackend) error { + for service, entries := range backends { + if strings.TrimSpace(service) == "" || len(entries) == 0 { + return fmt.Errorf("backend service and targets are required") + } + for _, backend := range entries { + host, rawPort, err := net.SplitHostPort(backend.Address) + if err != nil || !safeBackendHost(host) { + return fmt.Errorf("backend address %q is invalid", backend.Address) + } + port, portErr := strconv.Atoi(rawPort) + if portErr != nil || port < 1 || port > 65535 || backend.Weight < 0 { + return fmt.Errorf("backend address %q is invalid", backend.Address) + } + } + } + return nil +} + +func safeBackendHost(host string) bool { + if net.ParseIP(host) != nil { + return true + } + if host == "" { + return false + } + for _, value := range host { + if (value < 'a' || value > 'z') && (value < 'A' || value > 'Z') && (value < '0' || value > '9') && value != '.' && value != '-' && value != '_' { + return false + } + } + return true +} + func (m *Manager) VirtualHostExists(deploymentName string) bool { configFile := filepath.Join(m.configPath, deploymentName+".conf") _, err := os.Stat(configFile) @@ -589,6 +646,10 @@ func (m *Manager) generateMultiDomainConfig(deployment *models.Deployment) (stri } func (m *Manager) renderMultiDomainConfig(deployment *models.Deployment, keepalive bool) (string, error) { + return m.renderMultiDomainConfigWithBackends(deployment, keepalive, nil) +} + +func (m *Manager) renderMultiDomainConfigWithBackends(deployment *models.Deployment, keepalive bool, backendOverrides map[string][]UpstreamBackend) (string, error) { domains := deployment.Metadata.GetDomains() if len(domains) == 0 { return "", fmt.Errorf("no domains configured") @@ -624,7 +685,7 @@ func (m *Manager) renderMultiDomainConfig(deployment *models.Deployment, keepali servers := m.groupDomainsByHost(domains, deployment.Name, m.deploymentComposeContent(deployment)) - upstreams := assignUpstreams(servers, keepalive) + upstreams := assignUpstreams(servers, keepalive, backendOverrides) data := multiRouteTemplateData{ DeploymentName: deployment.Name, @@ -673,8 +734,8 @@ func (m *Manager) renderMultiDomainConfig(deployment *models.Deployment, keepali // service:port target and returns no blocks, so the generated config is // unchanged. With keepalive on it points each location at a shared, deduplicated // upstream block so requests to the same service:port reuse one connection pool. -func assignUpstreams(servers []serverData, keepalive bool) []upstreamData { - if !keepalive { +func assignUpstreams(servers []serverData, keepalive bool, backendOverrides map[string][]UpstreamBackend) []upstreamData { + if !keepalive && len(backendOverrides) == 0 { for si := range servers { for li := range servers[si].Locations { loc := &servers[si].Locations[li] @@ -691,11 +752,22 @@ func assignUpstreams(servers []serverData, keepalive bool) []upstreamData { for li := range servers[si].Locations { loc := &servers[si].Locations[li] target := fmt.Sprintf("%s:%d", loc.Service, loc.ContainerPort) + overrides := backendOverrides[loc.RouteService] + if len(overrides) == 0 { + overrides = backendOverrides[loc.Service] + } + if len(overrides) > 0 { + target = "override:" + loc.Service + } name, ok := byTarget[target] if !ok { name = upstreamNameFor(loc.Service, loc.ContainerPort, used) byTarget[target] = name - upstreams = append(upstreams, upstreamData{Name: name, Target: target}) + targets := []UpstreamBackend{{Address: target, Healthy: true}} + if len(overrides) > 0 { + targets = append([]UpstreamBackend(nil), overrides...) + } + upstreams = append(upstreams, upstreamData{Name: name, Targets: targets}) } loc.Upstream = name } @@ -755,10 +827,12 @@ func (m *Manager) groupDomainsByHost(domains []models.DomainConfig, deploymentNa // Route to the service's unique container name, not the bare Compose service // name (which is not unique across deployments sharing the proxy network and // resolves via embedded DNS to an arbitrary deployment's container). - service := d.Service + routeService := d.Service + service := routeService if service == "" { log.Printf("[proxy] warning: domain %q has no service set for deployment %q, falling back to deployment name", d.Domain, deploymentName) service = deploymentName + routeService = deploymentName } else { service = docker.ContainerNameForService(composeContent, deploymentName, service) } @@ -776,6 +850,7 @@ func (m *Manager) groupDomainsByHost(domains []models.DomainConfig, deploymentNa locations = append(locations, locationData{ Path: path, Service: service, + RouteService: routeService, ContainerPort: port, Protocol: "http", StripPrefix: d.StripPrefix, @@ -897,8 +972,8 @@ type multiRouteTemplateData struct { } type upstreamData struct { - Name string - Target string + Name string + Targets []UpstreamBackend } type serverData struct { @@ -914,6 +989,7 @@ type serverData struct { type locationData struct { Path string Service string + RouteService string ContainerPort int Protocol string StripPrefix bool @@ -1085,7 +1161,9 @@ const upstreamBlocks = `{{- range .Upstreams}} upstream {{.Name}} { zone {{.Name}} 64k; resolver 127.0.0.11 valid=30s ipv6=off; - server {{.Target}} resolve; +{{- range .Targets}} + server {{.Address}}{{if .Weight}} weight={{.Weight}}{{end}}{{if not .Healthy}} down{{end}} resolve; +{{- end}} keepalive 16; keepalive_timeout 60s; keepalive_requests 1000; diff --git a/internal/nginx/manager_test.go b/internal/nginx/manager_test.go index aa71e1c..99ecf89 100644 --- a/internal/nginx/manager_test.go +++ b/internal/nginx/manager_test.go @@ -2398,7 +2398,7 @@ func TestAssignUpstreams(t *testing.T) { {Service: "app", ContainerPort: 80}, }}, } - ups := assignUpstreams(servers, true) + ups := assignUpstreams(servers, true, nil) if len(ups) != 2 { t.Fatalf("expected 2 deduped upstreams, got %d: %+v", len(ups), ups) } @@ -2411,7 +2411,7 @@ func TestAssignUpstreams(t *testing.T) { } off := []serverData{{Locations: []locationData{{Service: "app", ContainerPort: 80}}}} - if ups := assignUpstreams(off, false); ups != nil { + if ups := assignUpstreams(off, false, nil); ups != nil { t.Errorf("expected no upstream blocks when keepalive is off, got %+v", ups) } if off[0].Locations[0].Upstream != "app:80" { @@ -2419,6 +2419,48 @@ func TestAssignUpstreams(t *testing.T) { } } +func TestRenderMultiDomain_BackendOverridesPreserveDeploymentConfig(t *testing.T) { + compose := "name: tenant-a\nservices:\n web:\n container_name: tenant-a-web\n" + m, deployment := newManagerWithDeployment(t, []models.DomainConfig{ + {ID: "d1", Service: "web", ContainerPort: 8080, Domain: "app.example.com", SSL: models.SSLConfig{Enabled: true}}, + }, compose) + deployment.Metadata.Security = &models.DeploymentSecurityConfig{Enabled: true, BlockedIPs: []string{"192.0.2.10"}} + + config, err := m.renderMultiDomainConfigWithBackends(deployment, false, map[string][]UpstreamBackend{ + "web": { + {Address: "10.42.0.8:8080", Healthy: true, Weight: 2}, + {Address: "10.42.1.9:8080", Healthy: false, Weight: 1}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, expected := range []string{ + "listen 443 ssl", "deny 192.0.2.10", "server 10.42.0.8:8080 weight=2 resolve;", + "server 10.42.1.9:8080 weight=1 down resolve;", "set $upstream flatrun_tenant-a-web_8080;", + } { + if !strings.Contains(config, expected) { + t.Fatalf("missing %q in:\n%s", expected, config) + } + } + if strings.Contains(config, "server tenant-a-web:8080 resolve;") { + t.Fatalf("Compose backend remains in overridden route:\n%s", config) + } +} + +func TestRenderVirtualHostWithBackendsRejectsDirectiveInjection(t *testing.T) { + m, deployment := newManagerWithDeployment(t, []models.DomainConfig{ + {ID: "d1", Service: "web", ContainerPort: 8080, Domain: "app.example.com"}, + }, "services:\n web:\n image: app\n") + + _, err := m.RenderVirtualHostWithBackends(deployment, map[string][]UpstreamBackend{ + "web": {{Address: "10.42.0.8:8080; include /etc/nginx/nginx.conf", Healthy: true}}, + }) + if err == nil { + t.Fatal("unsafe backend address was accepted") + } +} + // Dot, hyphen and underscore are valid in both container and upstream names, so // they survive and keep distinct container names distinct. func TestUpstreamNameForPreservesContainerChars(t *testing.T) { diff --git a/internal/routing/adapters.go b/internal/routing/adapters.go index 0f3ab11..3396fab 100644 --- a/internal/routing/adapters.go +++ b/internal/routing/adapters.go @@ -39,6 +39,10 @@ func (p *routeProvider) ID() ProviderID { } func (p *routeProvider) Validate(_ context.Context, route Route) error { + return validateRoute(route) +} + +func validateRoute(route Route) error { if !safeRouteID.MatchString(route.ID) { return fmt.Errorf("Route ID is invalid") } diff --git a/internal/routing/managed_nginx.go b/internal/routing/managed_nginx.go new file mode 100644 index 0000000..e14599b --- /dev/null +++ b/internal/routing/managed_nginx.go @@ -0,0 +1,133 @@ +package routing + +import ( + "context" + "fmt" + "sync" + + "github.com/flatrun/agent/internal/nginx" + "github.com/flatrun/agent/pkg/models" +) + +type DeploymentSource interface { + GetDeployment(string) (*models.Deployment, error) +} + +type ManagedNginx interface { + RenderVirtualHost(*models.Deployment) (string, error) + RenderVirtualHostWithBackends(*models.Deployment, map[string][]nginx.UpstreamBackend) (string, error) + GetVirtualHost(string) (string, error) + WriteVirtualHost(string, string) error + TestConfig() error + Reload() error +} + +type managedNginxProvider struct { + manager ManagedNginx + deployments DeploymentSource + mu sync.RWMutex + routes map[string]Route +} + +func NewManagedNginxProvider(manager ManagedNginx, deployments DeploymentSource) Provider { + return &managedNginxProvider{manager: manager, deployments: deployments, routes: make(map[string]Route)} +} + +func (p *managedNginxProvider) ID() ProviderID { return ProviderNginx } + +func (p *managedNginxProvider) Validate(_ context.Context, route Route) error { + if route.Service == "" { + return fmt.Errorf("Route service is required") + } + return validateRoute(route) +} + +func (p *managedNginxProvider) Reconcile(ctx context.Context, route Route) error { + if err := ctx.Err(); err != nil { + return err + } + if err := p.Validate(ctx, route); err != nil { + return err + } + deployment, err := p.deployments.GetDeployment(route.ID) + if err != nil { + return fmt.Errorf("load deployment for route: %w", err) + } + backends := make([]nginx.UpstreamBackend, 0, len(route.Backends)) + for _, backend := range route.Backends { + backends = append(backends, nginx.UpstreamBackend{Address: backend.Address, Healthy: backend.Healthy, Weight: backend.Weight}) + } + content, err := p.manager.RenderVirtualHostWithBackends(deployment, map[string][]nginx.UpstreamBackend{route.Service: backends}) + if err != nil { + return fmt.Errorf("render deployment route: %w", err) + } + previousContent, previousErr := p.manager.GetVirtualHost(route.ID) + if err := p.manager.WriteVirtualHost(route.ID, content); err != nil { + return fmt.Errorf("write deployment route: %w", err) + } + if err := p.manager.TestConfig(); err != nil { + if previousErr == nil { + _ = p.manager.WriteVirtualHost(route.ID, previousContent) + } + return fmt.Errorf("test Nginx configuration: %w", err) + } + if err := p.manager.Reload(); err != nil { + return fmt.Errorf("reload Nginx: %w", err) + } + p.mu.Lock() + p.routes[route.ID] = route + p.mu.Unlock() + return nil +} + +func (p *managedNginxProvider) Drain(ctx context.Context, routeID, backendID string) error { + route, ok := p.route(routeID) + if !ok { + return fmt.Errorf("Route %q is not managed", routeID) + } + found := false + for index := range route.Backends { + if route.Backends[index].ID == backendID { + route.Backends[index].Healthy = false + found = true + } + } + if !found { + return fmt.Errorf("Backend %q is not part of route %q", backendID, routeID) + } + return p.Reconcile(ctx, route) +} + +func (p *managedNginxProvider) Remove(ctx context.Context, routeID string) error { + if err := ctx.Err(); err != nil { + return err + } + deployment, err := p.deployments.GetDeployment(routeID) + if err != nil { + return fmt.Errorf("load deployment for route: %w", err) + } + content, err := p.manager.RenderVirtualHost(deployment) + if err != nil { + return fmt.Errorf("render deployment route: %w", err) + } + if err := p.manager.WriteVirtualHost(routeID, content); err != nil { + return fmt.Errorf("write deployment route: %w", err) + } + if err := p.manager.TestConfig(); err != nil { + return fmt.Errorf("test Nginx configuration: %w", err) + } + if err := p.manager.Reload(); err != nil { + return fmt.Errorf("reload Nginx: %w", err) + } + p.mu.Lock() + delete(p.routes, routeID) + p.mu.Unlock() + return nil +} + +func (p *managedNginxProvider) route(id string) (Route, bool) { + p.mu.RLock() + defer p.mu.RUnlock() + route, ok := p.routes[id] + return route, ok +} diff --git a/internal/routing/managed_nginx_test.go b/internal/routing/managed_nginx_test.go new file mode 100644 index 0000000..77effc0 --- /dev/null +++ b/internal/routing/managed_nginx_test.go @@ -0,0 +1,78 @@ +package routing + +import ( + "context" + "strings" + "testing" + + "github.com/flatrun/agent/internal/nginx" + "github.com/flatrun/agent/pkg/models" +) + +type managedNginxRecorder struct { + backends map[string][]nginx.UpstreamBackend + content string + reloads int +} + +func (m *managedNginxRecorder) RenderVirtualHost(_ *models.Deployment) (string, error) { + return "compose deployment config", nil +} + +func (m *managedNginxRecorder) RenderVirtualHostWithBackends(_ *models.Deployment, backends map[string][]nginx.UpstreamBackend) (string, error) { + m.backends = backends + return "preserved deployment config", nil +} +func (m *managedNginxRecorder) GetVirtualHost(string) (string, error) { return "previous config", nil } +func (m *managedNginxRecorder) WriteVirtualHost(_, content string) error { + m.content = content + return nil +} +func (m *managedNginxRecorder) TestConfig() error { return nil } +func (m *managedNginxRecorder) Reload() error { m.reloads++; return nil } + +type deploymentSourceStub struct{ deployment *models.Deployment } + +func (s deploymentSourceStub) GetDeployment(string) (*models.Deployment, error) { + return s.deployment, nil +} + +func TestManagedNginxProviderReconcilesAndDrainsDeploymentBackends(t *testing.T) { + manager := &managedNginxRecorder{} + provider := NewManagedNginxProvider(manager, deploymentSourceStub{deployment: &models.Deployment{Name: "shop"}}) + route := Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http", Backends: []Backend{ + {ID: "one", Address: "10.42.0.8:8080", Healthy: true, Weight: 1}, + {ID: "two", Address: "10.42.1.9:8080", Healthy: true, Weight: 1}, + }} + if err := provider.Reconcile(context.Background(), route); err != nil { + t.Fatal(err) + } + if manager.content != "preserved deployment config" || manager.reloads != 1 || len(manager.backends["web"]) != 2 { + t.Fatalf("manager = %#v", manager) + } + if err := provider.Drain(context.Background(), "shop", "two"); err != nil { + t.Fatal(err) + } + if manager.backends["web"][1].Healthy || manager.reloads != 2 { + t.Fatalf("drained backends = %#v", manager.backends["web"]) + } +} + +func TestManagedNginxProviderRequiresDeploymentService(t *testing.T) { + provider := NewManagedNginxProvider(&managedNginxRecorder{}, deploymentSourceStub{deployment: &models.Deployment{Name: "shop"}}) + err := provider.Validate(context.Background(), Route{ID: "shop", Domain: "shop.example.com", Protocol: "http", Backends: []Backend{{ID: "one", Address: "10.0.0.1:80", Healthy: true}}}) + if err == nil || !strings.Contains(err.Error(), "service") { + t.Fatalf("error = %v", err) + } +} + +func TestManagedNginxProviderRestoresComposeRouteOnRemove(t *testing.T) { + manager := &managedNginxRecorder{} + provider := NewManagedNginxProvider(manager, deploymentSourceStub{deployment: &models.Deployment{Name: "shop"}}) + if err := provider.Remove(context.Background(), "shop"); err != nil { + t.Fatal(err) + } + if manager.content != "compose deployment config" || manager.reloads != 1 { + t.Fatalf("manager = %#v", manager) + } +} diff --git a/internal/routing/provider.go b/internal/routing/provider.go index 31d0c59..55a83dc 100644 --- a/internal/routing/provider.go +++ b/internal/routing/provider.go @@ -18,6 +18,7 @@ type Backend struct { type Route struct { ID string `json:"id"` + Service string `json:"service,omitempty"` Domain string `json:"domain"` Path string `json:"path,omitempty"` Protocol string `json:"protocol"` From 9eb26a1ed09a5d15131eae3c5d097605a606f347 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 16:33:52 +0100 Subject: [PATCH 27/46] feat(autoscale): Validate scale-ready workloads Deployments can declare a portable scaling workload and storage policy. Operators receive specific compatibility blockers before Fleet activation can move or replicate the service. --- internal/api/autoscale_handlers.go | 15 ++++ internal/api/autoscale_handlers_test.go | 19 +++++ internal/api/server.go | 4 ++ internal/autoscale/compatibility.go | 91 ++++++++++++++++++++++++ internal/autoscale/compatibility_test.go | 23 ++++++ pkg/models/deployment.go | 12 ++++ 6 files changed, 164 insertions(+) create mode 100644 internal/autoscale/compatibility.go create mode 100644 internal/autoscale/compatibility_test.go diff --git a/internal/api/autoscale_handlers.go b/internal/api/autoscale_handlers.go index 8d75079..ec8b73f 100644 --- a/internal/api/autoscale_handlers.go +++ b/internal/api/autoscale_handlers.go @@ -48,6 +48,21 @@ func (s *Server) getDeploymentAutoscalePolicy(c *gin.Context) { c.JSON(http.StatusOK, autoscalePolicyResponse{autoscalePolicyRequest: policyRequest(policy), State: state}) } +func (s *Server) getDeploymentAutoscaleCompatibility(c *gin.Context) { + name := c.Param("name") + deployment, err := s.manager.GetDeployment(name) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "Deployment not found"}) + return + } + composeContent, _, err := s.manager.GetComposeFile(name) + if err != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": "Compose configuration is unavailable"}) + return + } + c.JSON(http.StatusOK, autoscale.AssessCompatibility(deployment, composeContent)) +} + func (s *Server) updateDeploymentAutoscalePolicy(c *gin.Context) { if s.autoscaleStore == nil { c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Autoscaling storage is unavailable"}) diff --git a/internal/api/autoscale_handlers_test.go b/internal/api/autoscale_handlers_test.go index ba88cd6..60221ac 100644 --- a/internal/api/autoscale_handlers_test.go +++ b/internal/api/autoscale_handlers_test.go @@ -25,6 +25,9 @@ func TestDeploymentAutoscalePolicyThroughHTTP(t *testing.T) { if err := os.WriteFile(filepath.Join(deploymentDir, "compose.yml"), []byte("services:\n app:\n image: nginx:alpine\n"), 0644); err != nil { t.Fatal(err) } + if err := os.WriteFile(filepath.Join(deploymentDir, "service.yml"), []byte("name: shop\nscaling:\n service: app\n stateless: true\n"), 0644); err != nil { + t.Fatal(err) + } cfg := &config.Config{DeploymentsPath: dir, Auth: config.AuthConfig{Enabled: true, JWTSecret: "autoscale-test-secret"}} t.Setenv("FLATRUN_ADMIN_PASSWORD", "testadminpass") authManager, err := auth.NewManager(dir, &cfg.Auth, true) @@ -43,6 +46,7 @@ func TestDeploymentAutoscalePolicyThroughHTTP(t *testing.T) { router.POST("/api/auth/login", middleware.Login) protected := router.Group("/api", middleware.RequireAuth()) protected.GET("/deployments/:name/autoscale", middleware.RequirePermission(auth.PermDeploymentsRead), middleware.RequireDeploymentAccess(auth.AccessLevelRead), server.getDeploymentAutoscalePolicy) + protected.GET("/deployments/:name/autoscale/compatibility", middleware.RequirePermission(auth.PermDeploymentsRead), middleware.RequireDeploymentAccess(auth.AccessLevelRead), server.getDeploymentAutoscaleCompatibility) protected.PUT("/deployments/:name/autoscale", middleware.RequirePermission(auth.PermDeploymentsWrite), middleware.RequireDeploymentAccess(auth.AccessLevelWrite), server.updateDeploymentAutoscalePolicy) token := loginAndGetToken(t, router, "admin", "testadminpass") @@ -77,4 +81,19 @@ func TestDeploymentAutoscalePolicyThroughHTTP(t *testing.T) { if response.MaxReplicas != 6 || response.CooldownSeconds != 120 || !response.AllowFleetCapacity { t.Fatalf("unexpected response: %+v", response) } + + req = httptest.NewRequest(http.MethodGet, "/api/deployments/shop/autoscale/compatibility", nil) + req.Header.Set("Authorization", "Bearer "+token) + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + var compatibility autoscale.Compatibility + if err := json.Unmarshal(w.Body.Bytes(), &compatibility); err != nil { + t.Fatal(err) + } + if !compatibility.Compatible || compatibility.Service != "app" || compatibility.Image != "nginx:alpine" { + t.Fatalf("compatibility = %#v", compatibility) + } } diff --git a/internal/api/server.go b/internal/api/server.go index be77fad..18f046e 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -629,6 +629,7 @@ func (s *Server) setupRoutes() { protected.GET("/deployments/:name/stats", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentContainerStats) protected.GET("/deployments/:name/resources", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentResources) protected.GET("/deployments/:name/autoscale", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentAutoscalePolicy) + protected.GET("/deployments/:name/autoscale/compatibility", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentAutoscaleCompatibility) protected.PUT("/deployments/:name/autoscale", s.authMiddleware.RequirePermission(auth.PermDeploymentsWrite), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelWrite), s.updateDeploymentAutoscalePolicy) // Image endpoints @@ -2027,6 +2028,9 @@ func mergeMetadata(existing, incoming *models.ServiceMetadata, sentFields map[st if _, ok := sentFields["backup"]; ok { merged.Backup = incoming.Backup } + if _, ok := sentFields["scaling"]; ok { + merged.Scaling = incoming.Scaling + } if _, ok := sentFields["protected_mode"]; ok { merged.ProtectedMode = incoming.ProtectedMode } diff --git a/internal/autoscale/compatibility.go b/internal/autoscale/compatibility.go new file mode 100644 index 0000000..29ffdd3 --- /dev/null +++ b/internal/autoscale/compatibility.go @@ -0,0 +1,91 @@ +package autoscale + +import ( + "fmt" + "strings" + + "github.com/flatrun/agent/pkg/models" + "gopkg.in/yaml.v3" +) + +type Compatibility struct { + Compatible bool `json:"compatible"` + Service string `json:"service,omitempty"` + Image string `json:"image,omitempty"` + Blockers []string `json:"blockers"` + Warnings []string `json:"warnings"` +} + +type composeCompatibilityFile struct { + Services map[string]composeCompatibilityService `yaml:"services"` +} + +type composeCompatibilityService struct { + Image string `yaml:"image"` + Build any `yaml:"build"` + Volumes []any `yaml:"volumes"` + Configs []any `yaml:"configs"` + Secrets []any `yaml:"secrets"` + Devices []any `yaml:"devices"` + Privileged bool `yaml:"privileged"` + NetworkMode string `yaml:"network_mode"` + DependsOn map[string]any `yaml:"depends_on"` +} + +func AssessCompatibility(deployment *models.Deployment, composeContent string) Compatibility { + result := Compatibility{Blockers: []string{}, Warnings: []string{}} + if deployment == nil || deployment.Metadata == nil || deployment.Metadata.Scaling == nil { + result.Blockers = append(result.Blockers, "Add a scaling declaration to service.yml") + return result + } + scaling := deployment.Metadata.Scaling + result.Service = strings.TrimSpace(scaling.Service) + if result.Service == "" { + result.Blockers = append(result.Blockers, "Choose the Compose service that may scale") + } + if !scaling.Stateless { + result.Blockers = append(result.Blockers, "Only workloads declared stateless can scale across servers") + } + mode := strings.TrimSpace(scaling.Storage.Mode) + if mode == "" { + mode = "none" + } + if mode != "none" && mode != "shared" { + result.Blockers = append(result.Blockers, "Storage mode must be none or shared") + } + if mode == "shared" && strings.TrimSpace(scaling.Storage.Class) == "" { + result.Blockers = append(result.Blockers, "Shared storage requires a storage class") + } + + var compose composeCompatibilityFile + if err := yaml.Unmarshal([]byte(composeContent), &compose); err != nil { + result.Blockers = append(result.Blockers, fmt.Sprintf("Compose configuration cannot be read: %v", err)) + return result + } + service, exists := compose.Services[result.Service] + if !exists && result.Service != "" { + result.Blockers = append(result.Blockers, fmt.Sprintf("Compose service %q does not exist", result.Service)) + return result + } + result.Image = strings.TrimSpace(service.Image) + if result.Image == "" { + result.Blockers = append(result.Blockers, "The scale-ready service needs a portable image") + } + if len(service.Volumes) > 0 && mode != "shared" { + result.Blockers = append(result.Blockers, "The service uses volumes but shared storage is not declared") + } + if len(service.Configs) > 0 || len(service.Secrets) > 0 { + result.Blockers = append(result.Blockers, "Compose configs and secrets need a Fleet distribution policy") + } + if len(service.Devices) > 0 || service.Privileged || service.NetworkMode != "" { + result.Blockers = append(result.Blockers, "Host-specific container access cannot move between Fleet servers") + } + if service.Build != nil && result.Image != "" { + result.Warnings = append(result.Warnings, "Publish the declared image before Fleet places replicas on another server") + } + if len(service.DependsOn) > 0 { + result.Warnings = append(result.Warnings, "Dependencies must be reachable from every server allowed to run this workload") + } + result.Compatible = len(result.Blockers) == 0 + return result +} diff --git a/internal/autoscale/compatibility_test.go b/internal/autoscale/compatibility_test.go new file mode 100644 index 0000000..801070d --- /dev/null +++ b/internal/autoscale/compatibility_test.go @@ -0,0 +1,23 @@ +package autoscale + +import ( + "testing" + + "github.com/flatrun/agent/pkg/models" +) + +func TestAssessCompatibilityAcceptsDeclaredStatelessImage(t *testing.T) { + deployment := &models.Deployment{Metadata: &models.ServiceMetadata{Scaling: &models.ScalingConfig{Service: "web", Stateless: true}}} + result := AssessCompatibility(deployment, "services:\n web:\n image: registry.example.com/shop:1\n") + if !result.Compatible || result.Service != "web" || result.Image != "registry.example.com/shop:1" { + t.Fatalf("result = %#v", result) + } +} + +func TestAssessCompatibilityExplainsUnsafeComposeFeatures(t *testing.T) { + deployment := &models.Deployment{Metadata: &models.ServiceMetadata{Scaling: &models.ScalingConfig{Service: "web", Stateless: true}}} + result := AssessCompatibility(deployment, "services:\n web:\n image: shop:1\n privileged: true\n volumes:\n - ./data:/data\n") + if result.Compatible || len(result.Blockers) != 2 { + t.Fatalf("result = %#v", result) + } +} diff --git a/pkg/models/deployment.go b/pkg/models/deployment.go index deacc16..99b80b0 100644 --- a/pkg/models/deployment.go +++ b/pkg/models/deployment.go @@ -39,6 +39,7 @@ type ServiceMetadata struct { QuickActions []QuickAction `yaml:"quick_actions,omitempty" json:"quick_actions,omitempty"` Security *DeploymentSecurityConfig `yaml:"security,omitempty" json:"security,omitempty"` Backup *BackupSpec `yaml:"backup,omitempty" json:"backup,omitempty"` + Scaling *ScalingConfig `yaml:"scaling,omitempty" json:"scaling,omitempty"` ProtectedMode *ProtectedModeConfig `yaml:"protected_mode,omitempty" json:"protected_mode,omitempty"` RequirePlan bool `yaml:"require_plan,omitempty" json:"require_plan,omitempty"` CredentialID string `yaml:"credential_id,omitempty" json:"credential_id,omitempty"` @@ -47,6 +48,17 @@ type ServiceMetadata struct { Databases []DatabaseConfig `yaml:"databases,omitempty" json:"databases,omitempty"` } +type ScalingConfig struct { + Service string `yaml:"service" json:"service"` + Stateless bool `yaml:"stateless" json:"stateless"` + Storage ScalingStorageConfig `yaml:"storage,omitempty" json:"storage,omitempty"` +} + +type ScalingStorageConfig struct { + Mode string `yaml:"mode,omitempty" json:"mode,omitempty"` + Class string `yaml:"class,omitempty" json:"class,omitempty"` +} + type DomainConfig struct { ID string `yaml:"id" json:"id"` Service string `yaml:"service" json:"service"` From a142cf19c3cd3187c29aa22cf1908c5acfe31b8e Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 16:34:41 +0100 Subject: [PATCH 28/46] feat(autoscale): Configure workload declarations Operators can save a scale-ready workload independently of other deployment metadata and immediately receive an updated compatibility assessment. --- internal/api/autoscale_handlers.go | 29 ++++++++++++++++++++++++- internal/api/autoscale_handlers_test.go | 17 +++++++++++++++ internal/api/server.go | 1 + internal/autoscale/compatibility.go | 6 +++++ 4 files changed, 52 insertions(+), 1 deletion(-) diff --git a/internal/api/autoscale_handlers.go b/internal/api/autoscale_handlers.go index ec8b73f..c6c1a5e 100644 --- a/internal/api/autoscale_handlers.go +++ b/internal/api/autoscale_handlers.go @@ -5,6 +5,7 @@ import ( "time" "github.com/flatrun/agent/internal/autoscale" + "github.com/flatrun/agent/pkg/models" "github.com/gin-gonic/gin" ) @@ -49,7 +50,10 @@ func (s *Server) getDeploymentAutoscalePolicy(c *gin.Context) { } func (s *Server) getDeploymentAutoscaleCompatibility(c *gin.Context) { - name := c.Param("name") + s.writeAutoscaleCompatibility(c, c.Param("name")) +} + +func (s *Server) writeAutoscaleCompatibility(c *gin.Context, name string) { deployment, err := s.manager.GetDeployment(name) if err != nil { c.JSON(http.StatusNotFound, gin.H{"error": "Deployment not found"}) @@ -63,6 +67,29 @@ func (s *Server) getDeploymentAutoscaleCompatibility(c *gin.Context) { c.JSON(http.StatusOK, autoscale.AssessCompatibility(deployment, composeContent)) } +func (s *Server) updateDeploymentAutoscaleWorkload(c *gin.Context) { + name := c.Param("name") + deployment, err := s.manager.GetDeployment(name) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "Deployment not found"}) + return + } + var scaling models.ScalingConfig + if err := c.ShouldBindJSON(&scaling); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + if deployment.Metadata == nil { + deployment.Metadata = &models.ServiceMetadata{Name: name} + } + deployment.Metadata.Scaling = &scaling + if err := s.manager.SaveMetadata(name, deployment.Metadata); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + s.writeAutoscaleCompatibility(c, name) +} + func (s *Server) updateDeploymentAutoscalePolicy(c *gin.Context) { if s.autoscaleStore == nil { c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Autoscaling storage is unavailable"}) diff --git a/internal/api/autoscale_handlers_test.go b/internal/api/autoscale_handlers_test.go index 60221ac..56bae18 100644 --- a/internal/api/autoscale_handlers_test.go +++ b/internal/api/autoscale_handlers_test.go @@ -47,6 +47,7 @@ func TestDeploymentAutoscalePolicyThroughHTTP(t *testing.T) { protected := router.Group("/api", middleware.RequireAuth()) protected.GET("/deployments/:name/autoscale", middleware.RequirePermission(auth.PermDeploymentsRead), middleware.RequireDeploymentAccess(auth.AccessLevelRead), server.getDeploymentAutoscalePolicy) protected.GET("/deployments/:name/autoscale/compatibility", middleware.RequirePermission(auth.PermDeploymentsRead), middleware.RequireDeploymentAccess(auth.AccessLevelRead), server.getDeploymentAutoscaleCompatibility) + protected.PUT("/deployments/:name/autoscale/workload", middleware.RequirePermission(auth.PermDeploymentsWrite), middleware.RequireDeploymentAccess(auth.AccessLevelWrite), server.updateDeploymentAutoscaleWorkload) protected.PUT("/deployments/:name/autoscale", middleware.RequirePermission(auth.PermDeploymentsWrite), middleware.RequireDeploymentAccess(auth.AccessLevelWrite), server.updateDeploymentAutoscalePolicy) token := loginAndGetToken(t, router, "admin", "testadminpass") @@ -96,4 +97,20 @@ func TestDeploymentAutoscalePolicyThroughHTTP(t *testing.T) { if !compatibility.Compatible || compatibility.Service != "app" || compatibility.Image != "nginx:alpine" { t.Fatalf("compatibility = %#v", compatibility) } + + workload := bytes.NewBufferString(`{"service":"app","stateless":true,"storage":{"mode":"none"}}`) + req = httptest.NewRequest(http.MethodPut, "/api/deployments/shop/autoscale/workload", workload) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + if err := json.Unmarshal(w.Body.Bytes(), &compatibility); err != nil { + t.Fatal(err) + } + if !compatibility.Compatible || len(compatibility.Services) != 1 || compatibility.Services[0] != "app" { + t.Fatalf("compatibility = %#v", compatibility) + } } diff --git a/internal/api/server.go b/internal/api/server.go index 18f046e..501a274 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -630,6 +630,7 @@ func (s *Server) setupRoutes() { protected.GET("/deployments/:name/resources", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentResources) protected.GET("/deployments/:name/autoscale", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentAutoscalePolicy) protected.GET("/deployments/:name/autoscale/compatibility", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentAutoscaleCompatibility) + protected.PUT("/deployments/:name/autoscale/workload", s.authMiddleware.RequirePermission(auth.PermDeploymentsWrite), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelWrite), s.updateDeploymentAutoscaleWorkload) protected.PUT("/deployments/:name/autoscale", s.authMiddleware.RequirePermission(auth.PermDeploymentsWrite), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelWrite), s.updateDeploymentAutoscalePolicy) // Image endpoints diff --git a/internal/autoscale/compatibility.go b/internal/autoscale/compatibility.go index 29ffdd3..84819bd 100644 --- a/internal/autoscale/compatibility.go +++ b/internal/autoscale/compatibility.go @@ -2,6 +2,7 @@ package autoscale import ( "fmt" + "sort" "strings" "github.com/flatrun/agent/pkg/models" @@ -14,6 +15,7 @@ type Compatibility struct { Image string `json:"image,omitempty"` Blockers []string `json:"blockers"` Warnings []string `json:"warnings"` + Services []string `json:"services"` } type composeCompatibilityFile struct { @@ -62,6 +64,10 @@ func AssessCompatibility(deployment *models.Deployment, composeContent string) C result.Blockers = append(result.Blockers, fmt.Sprintf("Compose configuration cannot be read: %v", err)) return result } + for name := range compose.Services { + result.Services = append(result.Services, name) + } + sort.Strings(result.Services) service, exists := compose.Services[result.Service] if !exists && result.Service != "" { result.Blockers = append(result.Blockers, fmt.Sprintf("Compose service %q does not exist", result.Service)) From 141ea2ee2afefd250d1163017fb198c609d6fb1a Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 16:37:36 +0100 Subject: [PATCH 29/46] feat(autoscale): Return saved workload declaration Compatibility responses now include the saved workload declaration so clients can edit storage and portability settings without reconstructing them. --- internal/autoscale/compatibility.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/autoscale/compatibility.go b/internal/autoscale/compatibility.go index 84819bd..2781de4 100644 --- a/internal/autoscale/compatibility.go +++ b/internal/autoscale/compatibility.go @@ -10,12 +10,13 @@ import ( ) type Compatibility struct { - Compatible bool `json:"compatible"` - Service string `json:"service,omitempty"` - Image string `json:"image,omitempty"` - Blockers []string `json:"blockers"` - Warnings []string `json:"warnings"` - Services []string `json:"services"` + Compatible bool `json:"compatible"` + Service string `json:"service,omitempty"` + Image string `json:"image,omitempty"` + Blockers []string `json:"blockers"` + Warnings []string `json:"warnings"` + Services []string `json:"services"` + Workload *models.ScalingConfig `json:"workload,omitempty"` } type composeCompatibilityFile struct { @@ -41,6 +42,7 @@ func AssessCompatibility(deployment *models.Deployment, composeContent string) C return result } scaling := deployment.Metadata.Scaling + result.Workload = scaling result.Service = strings.TrimSpace(scaling.Service) if result.Service == "" { result.Blockers = append(result.Blockers, "Choose the Compose service that may scale") From 74245bf892fbd4b8ddceb67cf174bf47f439fa04 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 16:40:11 +0100 Subject: [PATCH 30/46] feat(autoscale): Build portable runtime workloads Scale-ready services now retain inline environment, startup commands, working directories, health paths, and routed ports across Swarm and K3s. Inputs that cannot yet move safely remain activation blockers. --- internal/autoscale/compatibility.go | 76 ++++++++++++++++++++++++ internal/autoscale/compatibility_test.go | 26 ++++++++ internal/orchestrator/k3s.go | 22 +++++++ internal/orchestrator/k3s_test.go | 9 ++- internal/orchestrator/provider.go | 20 ++++--- internal/orchestrator/swarm.go | 16 ++++- internal/orchestrator/swarm_test.go | 7 ++- 7 files changed, 165 insertions(+), 11 deletions(-) diff --git a/internal/autoscale/compatibility.go b/internal/autoscale/compatibility.go index 2781de4..1b622fd 100644 --- a/internal/autoscale/compatibility.go +++ b/internal/autoscale/compatibility.go @@ -5,6 +5,7 @@ import ( "sort" "strings" + "github.com/flatrun/agent/internal/orchestrator" "github.com/flatrun/agent/pkg/models" "gopkg.in/yaml.v3" ) @@ -33,6 +34,11 @@ type composeCompatibilityService struct { Privileged bool `yaml:"privileged"` NetworkMode string `yaml:"network_mode"` DependsOn map[string]any `yaml:"depends_on"` + EnvFile any `yaml:"env_file"` + Environment any `yaml:"environment"` + Entrypoint any `yaml:"entrypoint"` + Command any `yaml:"command"` + WorkingDir string `yaml:"working_dir"` } func AssessCompatibility(deployment *models.Deployment, composeContent string) Compatibility { @@ -60,6 +66,9 @@ func AssessCompatibility(deployment *models.Deployment, composeContent string) C if mode == "shared" && strings.TrimSpace(scaling.Storage.Class) == "" { result.Blockers = append(result.Blockers, "Shared storage requires a storage class") } + if mode == "shared" { + result.Blockers = append(result.Blockers, "Shared storage needs an installed Fleet storage adapter") + } var compose composeCompatibilityFile if err := yaml.Unmarshal([]byte(composeContent), &compose); err != nil { @@ -85,6 +94,9 @@ func AssessCompatibility(deployment *models.Deployment, composeContent string) C if len(service.Configs) > 0 || len(service.Secrets) > 0 { result.Blockers = append(result.Blockers, "Compose configs and secrets need a Fleet distribution policy") } + if service.EnvFile != nil { + result.Blockers = append(result.Blockers, "Environment files must be converted to inline deployment environment values") + } if len(service.Devices) > 0 || service.Privileged || service.NetworkMode != "" { result.Blockers = append(result.Blockers, "Host-specific container access cannot move between Fleet servers") } @@ -97,3 +109,67 @@ func AssessCompatibility(deployment *models.Deployment, composeContent string) C result.Compatible = len(result.Blockers) == 0 return result } + +func BuildWorkload(deployment *models.Deployment, composeContent string, replicas int) (orchestrator.Workload, error) { + compatibility := AssessCompatibility(deployment, composeContent) + if !compatibility.Compatible { + return orchestrator.Workload{}, fmt.Errorf("workload is not scale-ready: %s", strings.Join(compatibility.Blockers, "; ")) + } + var compose composeCompatibilityFile + if err := yaml.Unmarshal([]byte(composeContent), &compose); err != nil { + return orchestrator.Workload{}, err + } + service := compose.Services[compatibility.Service] + workload := orchestrator.Workload{ + ID: deployment.Name, Image: service.Image, Replicas: replicas, + Environment: map[string]string{}, Entrypoint: stringList(service.Entrypoint), Command: stringList(service.Command), + WorkingDir: service.WorkingDir, Labels: map[string]string{"flatrun.deployment": deployment.Name}, + } + workload.Environment = environmentMap(service.Environment) + for _, domain := range deployment.Metadata.GetDomains() { + if domain.Service == compatibility.Service { + workload.Port = domain.ContainerPort + break + } + } + workload.Health.Path = deployment.Metadata.HealthCheck.Path + return workload, nil +} + +func environmentMap(value any) map[string]string { + result := map[string]string{} + switch typed := value.(type) { + case map[string]any: + for key, item := range typed { + if item != nil { + result[key] = fmt.Sprint(item) + } + } + case []any: + for _, item := range typed { + parts := strings.SplitN(fmt.Sprint(item), "=", 2) + if len(parts) == 2 { + result[parts[0]] = parts[1] + } + } + } + return result +} + +func stringList(value any) []string { + switch typed := value.(type) { + case string: + if strings.TrimSpace(typed) == "" { + return nil + } + return []string{"/bin/sh", "-c", typed} + case []any: + result := make([]string, 0, len(typed)) + for _, item := range typed { + result = append(result, fmt.Sprint(item)) + } + return result + default: + return nil + } +} diff --git a/internal/autoscale/compatibility_test.go b/internal/autoscale/compatibility_test.go index 801070d..d14dcb0 100644 --- a/internal/autoscale/compatibility_test.go +++ b/internal/autoscale/compatibility_test.go @@ -21,3 +21,29 @@ func TestAssessCompatibilityExplainsUnsafeComposeFeatures(t *testing.T) { t.Fatalf("result = %#v", result) } } + +func TestBuildWorkloadCarriesPortableRuntimeInputs(t *testing.T) { + deployment := &models.Deployment{Name: "shop", Metadata: &models.ServiceMetadata{ + Scaling: &models.ScalingConfig{Service: "web", Stateless: true}, + Domains: []models.DomainConfig{{Service: "web", ContainerPort: 8080, Domain: "shop.example.com"}}, + HealthCheck: models.HealthCheckConfig{Path: "/ready"}, + }} + workload, err := BuildWorkload(deployment, `services: + web: + image: registry.example.com/shop:1 + environment: + APP_ENV: production + entrypoint: ["/app/entrypoint"] + command: ["serve", "--port", "8080"] + working_dir: /app +`, 2) + if err != nil { + t.Fatal(err) + } + if workload.Replicas != 2 || workload.Port != 8080 || workload.Environment["APP_ENV"] != "production" || workload.WorkingDir != "/app" || workload.Health.Path != "/ready" { + t.Fatalf("workload = %#v", workload) + } + if len(workload.Entrypoint) != 1 || len(workload.Command) != 3 { + t.Fatalf("workload commands = %#v / %#v", workload.Entrypoint, workload.Command) + } +} diff --git a/internal/orchestrator/k3s.go b/internal/orchestrator/k3s.go index ba9a96c..692691e 100644 --- a/internal/orchestrator/k3s.go +++ b/internal/orchestrator/k3s.go @@ -7,6 +7,7 @@ import ( "fmt" "net" "os/exec" + "sort" "strconv" "strings" ) @@ -197,6 +198,27 @@ func k3sManifest(workload Workload) map[string]any { labels["flatrun.port"] = strconv.Itoa(workload.Port) } container := map[string]any{"name": workload.ID, "image": workload.Image, "resources": k3sResources(workload.Resources)} + if len(workload.Environment) > 0 { + keys := make([]string, 0, len(workload.Environment)) + for key := range workload.Environment { + keys = append(keys, key) + } + sort.Strings(keys) + environment := make([]any, 0, len(keys)) + for _, key := range keys { + environment = append(environment, map[string]string{"name": key, "value": workload.Environment[key]}) + } + container["env"] = environment + } + if len(workload.Entrypoint) > 0 { + container["command"] = workload.Entrypoint + } + if len(workload.Command) > 0 { + container["args"] = workload.Command + } + if workload.WorkingDir != "" { + container["workingDir"] = workload.WorkingDir + } if workload.Port > 0 { container["ports"] = []any{map[string]any{"containerPort": workload.Port}} } diff --git a/internal/orchestrator/k3s_test.go b/internal/orchestrator/k3s_test.go index 9f628b2..91e33ba 100644 --- a/internal/orchestrator/k3s_test.go +++ b/internal/orchestrator/k3s_test.go @@ -36,7 +36,7 @@ func TestK3sApplyUsesConfiguredClusterAndNamespace(t *testing.T) { provider := NewK3sProvider("/etc/rancher/k3s.yaml", "apps") provider.runner = runner - status, err := provider.Apply(context.Background(), Workload{ID: "shop", Image: "shop:1", Port: 8080, Replicas: 2}) + status, err := provider.Apply(context.Background(), Workload{ID: "shop", Image: "shop:1", Port: 8080, Replicas: 2, Environment: map[string]string{"APP_ENV": "production"}, Command: []string{"serve"}}) if err != nil { t.Fatal(err) } @@ -54,6 +54,13 @@ func TestK3sApplyUsesConfiguredClusterAndNamespace(t *testing.T) { if manifest["kind"] != "Deployment" { t.Fatalf("manifest = %#v", manifest) } + spec := manifest["spec"].(map[string]any) + template := spec["template"].(map[string]any) + podSpec := template["spec"].(map[string]any) + container := podSpec["containers"].([]any)[0].(map[string]any) + if len(container["env"].([]any)) != 1 || len(container["args"].([]any)) != 1 { + t.Fatalf("container = %#v", container) + } } func TestK3sStatusReturnsRoutableReadyPods(t *testing.T) { diff --git a/internal/orchestrator/provider.go b/internal/orchestrator/provider.go index 4d3384a..d0300a1 100644 --- a/internal/orchestrator/provider.go +++ b/internal/orchestrator/provider.go @@ -25,14 +25,18 @@ type Health struct { } type Workload struct { - ID string `json:"id"` - Image string `json:"image"` - Port int `json:"port,omitempty"` - Replicas int `json:"replicas"` - Resources Resources `json:"resources"` - Health Health `json:"health"` - Labels map[string]string `json:"labels,omitempty"` - Stateful bool `json:"stateful"` + ID string `json:"id"` + Image string `json:"image"` + Port int `json:"port,omitempty"` + Replicas int `json:"replicas"` + Resources Resources `json:"resources"` + Health Health `json:"health"` + Labels map[string]string `json:"labels,omitempty"` + Environment map[string]string `json:"environment,omitempty"` + Entrypoint []string `json:"entrypoint,omitempty"` + Command []string `json:"command,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` + Stateful bool `json:"stateful"` } type Instance struct { diff --git a/internal/orchestrator/swarm.go b/internal/orchestrator/swarm.go index 05b57e3..f831edf 100644 --- a/internal/orchestrator/swarm.go +++ b/internal/orchestrator/swarm.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net" + "sort" "strconv" "strings" @@ -180,13 +181,26 @@ func swarmSpec(workload Workload) swarm.ServiceSpec { return swarm.ServiceSpec{ Annotations: swarm.Annotations{Name: workload.ID, Labels: labels}, TaskTemplate: swarm.TaskSpec{ - ContainerSpec: &swarm.ContainerSpec{Image: workload.Image, Labels: labels}, + ContainerSpec: &swarm.ContainerSpec{Image: workload.Image, Labels: labels, Env: workloadEnvironment(workload.Environment), Command: workload.Entrypoint, Args: workload.Command, Dir: workload.WorkingDir}, Resources: swarmResources(workload.Resources), }, Mode: swarm.ServiceMode{Replicated: &swarm.ReplicatedService{Replicas: &replicas}}, } } +func workloadEnvironment(environment map[string]string) []string { + keys := make([]string, 0, len(environment)) + for key := range environment { + keys = append(keys, key) + } + sort.Strings(keys) + values := make([]string, 0, len(keys)) + for _, key := range keys { + values = append(values, key+"="+environment[key]) + } + return values +} + func servicePort(spec swarm.ServiceSpec) int { value := spec.Annotations.Labels["flatrun.port"] port, _ := strconv.Atoi(value) diff --git a/internal/orchestrator/swarm_test.go b/internal/orchestrator/swarm_test.go index 9889dd5..8bc13de 100644 --- a/internal/orchestrator/swarm_test.go +++ b/internal/orchestrator/swarm_test.go @@ -59,7 +59,8 @@ func TestSwarmProviderCreatesReplicatedService(t *testing.T) { status, err := provider.Apply(context.Background(), Workload{ ID: "shop", Image: "example/shop:1", Port: 8080, Replicas: 1, - Resources: Resources{CPURequest: 0.5, CPULimit: 1, MemoryRequest: 256 << 20, MemoryLimit: 512 << 20}, + Resources: Resources{CPURequest: 0.5, CPULimit: 1, MemoryRequest: 256 << 20, MemoryLimit: 512 << 20}, + Environment: map[string]string{"APP_ENV": "production"}, Entrypoint: []string{"/app/entrypoint"}, Command: []string{"serve"}, WorkingDir: "/app", }) if err != nil { t.Fatalf("Apply failed: %v", err) @@ -70,6 +71,10 @@ func TestSwarmProviderCreatesReplicatedService(t *testing.T) { if client.created.TaskTemplate.Resources.Limits.NanoCPUs != 1_000_000_000 { t.Fatalf("CPU limit = %d", client.created.TaskTemplate.Resources.Limits.NanoCPUs) } + container := client.created.TaskTemplate.ContainerSpec + if len(container.Env) != 1 || container.Env[0] != "APP_ENV=production" || container.Command[0] != "/app/entrypoint" || container.Args[0] != "serve" || container.Dir != "/app" { + t.Fatalf("container spec = %#v", container) + } if status.Desired != 1 || status.Available != 1 || len(status.Instances) != 1 { t.Fatalf("status = %#v", status) } From 7f83631e47ceaa56f296fccbd94b8dfaf7bb059e Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 16:41:33 +0100 Subject: [PATCH 31/46] feat(autoscale): Attach Swarm workloads to proxy network Scale-ready Swarm services now join the configured proxy overlay so the existing FlatRun proxy can reach replicas across Fleet nodes. --- internal/autoscale/compatibility.go | 5 ++++- internal/autoscale/compatibility_test.go | 5 ++++- internal/orchestrator/provider.go | 1 + internal/orchestrator/swarm.go | 11 +++++++++++ internal/orchestrator/swarm_test.go | 4 ++++ 5 files changed, 24 insertions(+), 2 deletions(-) diff --git a/internal/autoscale/compatibility.go b/internal/autoscale/compatibility.go index 1b622fd..05bf276 100644 --- a/internal/autoscale/compatibility.go +++ b/internal/autoscale/compatibility.go @@ -110,7 +110,7 @@ func AssessCompatibility(deployment *models.Deployment, composeContent string) C return result } -func BuildWorkload(deployment *models.Deployment, composeContent string, replicas int) (orchestrator.Workload, error) { +func BuildWorkload(deployment *models.Deployment, composeContent string, replicas int, proxyNetwork string) (orchestrator.Workload, error) { compatibility := AssessCompatibility(deployment, composeContent) if !compatibility.Compatible { return orchestrator.Workload{}, fmt.Errorf("workload is not scale-ready: %s", strings.Join(compatibility.Blockers, "; ")) @@ -125,6 +125,9 @@ func BuildWorkload(deployment *models.Deployment, composeContent string, replica Environment: map[string]string{}, Entrypoint: stringList(service.Entrypoint), Command: stringList(service.Command), WorkingDir: service.WorkingDir, Labels: map[string]string{"flatrun.deployment": deployment.Name}, } + if strings.TrimSpace(proxyNetwork) != "" { + workload.Networks = []string{proxyNetwork} + } workload.Environment = environmentMap(service.Environment) for _, domain := range deployment.Metadata.GetDomains() { if domain.Service == compatibility.Service { diff --git a/internal/autoscale/compatibility_test.go b/internal/autoscale/compatibility_test.go index d14dcb0..08b15f2 100644 --- a/internal/autoscale/compatibility_test.go +++ b/internal/autoscale/compatibility_test.go @@ -36,7 +36,7 @@ func TestBuildWorkloadCarriesPortableRuntimeInputs(t *testing.T) { entrypoint: ["/app/entrypoint"] command: ["serve", "--port", "8080"] working_dir: /app -`, 2) +`, 2, "proxy") if err != nil { t.Fatal(err) } @@ -46,4 +46,7 @@ func TestBuildWorkloadCarriesPortableRuntimeInputs(t *testing.T) { if len(workload.Entrypoint) != 1 || len(workload.Command) != 3 { t.Fatalf("workload commands = %#v / %#v", workload.Entrypoint, workload.Command) } + if len(workload.Networks) != 1 || workload.Networks[0] != "proxy" { + t.Fatalf("workload networks = %#v", workload.Networks) + } } diff --git a/internal/orchestrator/provider.go b/internal/orchestrator/provider.go index d0300a1..aa59196 100644 --- a/internal/orchestrator/provider.go +++ b/internal/orchestrator/provider.go @@ -36,6 +36,7 @@ type Workload struct { Entrypoint []string `json:"entrypoint,omitempty"` Command []string `json:"command,omitempty"` WorkingDir string `json:"working_dir,omitempty"` + Networks []string `json:"networks,omitempty"` Stateful bool `json:"stateful"` } diff --git a/internal/orchestrator/swarm.go b/internal/orchestrator/swarm.go index f831edf..f05cc92 100644 --- a/internal/orchestrator/swarm.go +++ b/internal/orchestrator/swarm.go @@ -183,11 +183,22 @@ func swarmSpec(workload Workload) swarm.ServiceSpec { TaskTemplate: swarm.TaskSpec{ ContainerSpec: &swarm.ContainerSpec{Image: workload.Image, Labels: labels, Env: workloadEnvironment(workload.Environment), Command: workload.Entrypoint, Args: workload.Command, Dir: workload.WorkingDir}, Resources: swarmResources(workload.Resources), + Networks: swarmNetworks(workload.Networks), }, Mode: swarm.ServiceMode{Replicated: &swarm.ReplicatedService{Replicas: &replicas}}, } } +func swarmNetworks(networks []string) []swarm.NetworkAttachmentConfig { + result := make([]swarm.NetworkAttachmentConfig, 0, len(networks)) + for _, network := range networks { + if strings.TrimSpace(network) != "" { + result = append(result, swarm.NetworkAttachmentConfig{Target: network}) + } + } + return result +} + func workloadEnvironment(environment map[string]string) []string { keys := make([]string, 0, len(environment)) for key := range environment { diff --git a/internal/orchestrator/swarm_test.go b/internal/orchestrator/swarm_test.go index 8bc13de..77cb5af 100644 --- a/internal/orchestrator/swarm_test.go +++ b/internal/orchestrator/swarm_test.go @@ -61,6 +61,7 @@ func TestSwarmProviderCreatesReplicatedService(t *testing.T) { ID: "shop", Image: "example/shop:1", Port: 8080, Replicas: 1, Resources: Resources{CPURequest: 0.5, CPULimit: 1, MemoryRequest: 256 << 20, MemoryLimit: 512 << 20}, Environment: map[string]string{"APP_ENV": "production"}, Entrypoint: []string{"/app/entrypoint"}, Command: []string{"serve"}, WorkingDir: "/app", + Networks: []string{"proxy"}, }) if err != nil { t.Fatalf("Apply failed: %v", err) @@ -75,6 +76,9 @@ func TestSwarmProviderCreatesReplicatedService(t *testing.T) { if len(container.Env) != 1 || container.Env[0] != "APP_ENV=production" || container.Command[0] != "/app/entrypoint" || container.Args[0] != "serve" || container.Dir != "/app" { t.Fatalf("container spec = %#v", container) } + if len(client.created.TaskTemplate.Networks) != 1 || client.created.TaskTemplate.Networks[0].Target != "proxy" { + t.Fatalf("networks = %#v", client.created.TaskTemplate.Networks) + } if status.Desired != 1 || status.Available != 1 || len(status.Instances) != 1 { t.Fatalf("status = %#v", status) } From 65ccf245b9556e2c9b6a40f1a6815e5d94d7d30d Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 16:43:42 +0100 Subject: [PATCH 32/46] feat(autoscale): Add managed workload cutover Scale-ready deployments can move traffic only after managed replicas are ready. Failed cutovers restore the original route and remove the incomplete workload. --- internal/autoscale/activator.go | 88 ++++++++++++++++++++++++++++ internal/autoscale/activator_test.go | 52 ++++++++++++++++ internal/autoscale/executor_test.go | 12 +++- 3 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 internal/autoscale/activator.go create mode 100644 internal/autoscale/activator_test.go diff --git a/internal/autoscale/activator.go b/internal/autoscale/activator.go new file mode 100644 index 0000000..66a942f --- /dev/null +++ b/internal/autoscale/activator.go @@ -0,0 +1,88 @@ +package autoscale + +import ( + "context" + "fmt" + "time" + + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" +) + +type ServiceStopper interface { + StopService(string, string) (string, error) +} + +type Activator struct { + orchestrator orchestrator.Provider + routing routing.Provider + stopper ServiceStopper + pollInterval time.Duration + readyTimeout time.Duration +} + +type Activation struct { + Workload orchestrator.Status `json:"workload"` + Route routing.Route `json:"route"` +} + +func NewActivator(orchestratorProvider orchestrator.Provider, routingProvider routing.Provider, stopper ServiceStopper) *Activator { + return &Activator{orchestrator: orchestratorProvider, routing: routingProvider, stopper: stopper, pollInterval: 2 * time.Second, readyTimeout: 2 * time.Minute} +} + +func (a *Activator) Activate(ctx context.Context, deployment, service string, workload orchestrator.Workload, route routing.Route) (Activation, error) { + status, err := a.orchestrator.Apply(ctx, workload) + if err != nil { + return Activation{}, fmt.Errorf("create managed workload: %w", err) + } + rollback := func() { + _ = a.routing.Remove(context.Background(), route.ID) + _ = a.orchestrator.Remove(context.Background(), workload.ID) + } + status, err = a.waitReady(ctx, workload.ID, status) + if err != nil { + rollback() + return Activation{}, err + } + route, err = routeWithReadyInstances(route, status) + if err != nil { + rollback() + return Activation{}, err + } + if err := a.routing.Reconcile(ctx, route); err != nil { + rollback() + return Activation{}, fmt.Errorf("publish managed route: %w", err) + } + if _, err := a.stopper.StopService(deployment, service); err != nil { + rollback() + return Activation{}, fmt.Errorf("stop Compose service after cutover: %w", err) + } + return Activation{Workload: status, Route: route}, nil +} + +func (a *Activator) waitReady(ctx context.Context, workloadID string, status orchestrator.Status) (orchestrator.Status, error) { + if status.Desired > 0 && status.Available >= status.Desired { + return status, nil + } + timer := time.NewTimer(a.readyTimeout) + defer timer.Stop() + ticker := time.NewTicker(a.pollInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return status, ctx.Err() + case <-timer.C: + return status, fmt.Errorf("managed workload did not become ready within %s", a.readyTimeout) + case <-ticker.C: + var err error + status, err = a.orchestrator.Status(ctx, workloadID) + if err != nil { + return status, fmt.Errorf("check managed workload readiness: %w", err) + } + if status.Desired > 0 && status.Available >= status.Desired { + return status, nil + } + } + } +} diff --git a/internal/autoscale/activator_test.go b/internal/autoscale/activator_test.go new file mode 100644 index 0000000..a714d63 --- /dev/null +++ b/internal/autoscale/activator_test.go @@ -0,0 +1,52 @@ +package autoscale + +import ( + "context" + "errors" + "testing" + + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" +) + +type activationStopper struct { + stopped bool + err error +} + +func (s *activationStopper) StopService(_, _ string) (string, error) { + s.stopped = true + return "", s.err +} + +func TestActivatorCutsOverOnlyAfterManagedReplicasAreReady(t *testing.T) { + provider := &fakeOrchestrator{status: orchestrator.Status{Workload: "shop", Desired: 2, Available: 2, Instances: []orchestrator.Instance{ + {ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}, + {ID: "two", Address: "10.0.0.2:8080", Healthy: true, Ready: true}, + }}} + router := &fakeRouter{} + stopper := &activationStopper{} + activation, err := NewActivator(provider, router, stopper).Activate(context.Background(), "shop", "web", orchestrator.Workload{ID: "shop", Image: "shop:1", Replicas: 2}, routing.Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http"}) + if err != nil { + t.Fatal(err) + } + if !stopper.stopped || len(router.reconciled.Backends) != 2 || activation.Workload.Available != 2 { + t.Fatalf("activation = %#v, route = %#v", activation, router.reconciled) + } +} + +func TestActivatorRollsBackWhenComposeCannotStop(t *testing.T) { + provider := &fakeOrchestrator{status: orchestrator.Status{Workload: "shop", Desired: 1, Available: 1, Instances: []orchestrator.Instance{{ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}}}} + router := &fakeRouter{} + stopper := &activationStopper{err: errors.New("compose failed")} + _, err := NewActivator(provider, router, stopper).Activate(context.Background(), "shop", "web", orchestrator.Workload{ID: "shop", Image: "shop:1", Replicas: 1}, routing.Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http"}) + if err == nil { + t.Fatal("activation succeeded") + } + if !stopper.stopped { + t.Fatal("Compose stop was not attempted") + } + if provider.removed != "shop" || router.removed != "shop" { + t.Fatalf("rollback removed workload %q and route %q", provider.removed, router.removed) + } +} diff --git a/internal/autoscale/executor_test.go b/internal/autoscale/executor_test.go index 725a0cd..a96122a 100644 --- a/internal/autoscale/executor_test.go +++ b/internal/autoscale/executor_test.go @@ -12,6 +12,7 @@ type fakeOrchestrator struct { status orchestrator.Status scaledTo int resizedWith orchestrator.Resources + removed string } func (f *fakeOrchestrator) ID() orchestrator.ProviderID { return orchestrator.ProviderSwarm } @@ -31,11 +32,15 @@ func (f *fakeOrchestrator) Scale(_ context.Context, _ string, replicas int) (orc func (f *fakeOrchestrator) Status(context.Context, string) (orchestrator.Status, error) { return f.status, nil } -func (f *fakeOrchestrator) Remove(context.Context, string) error { return nil } +func (f *fakeOrchestrator) Remove(_ context.Context, id string) error { + f.removed = id + return nil +} type fakeRouter struct { reconciled routing.Route drained string + removed string } func (f *fakeRouter) ID() routing.ProviderID { return routing.ProviderNginx } @@ -48,7 +53,10 @@ func (f *fakeRouter) Drain(_ context.Context, _, backendID string) error { f.drained = backendID return nil } -func (f *fakeRouter) Remove(context.Context, string) error { return nil } +func (f *fakeRouter) Remove(_ context.Context, id string) error { + f.removed = id + return nil +} func TestExecutorPublishesOnlyReadyScaledReplicas(t *testing.T) { orchestratorProvider := &fakeOrchestrator{status: orchestrator.Status{ From 095f4118fc65e1f10a65389efbf725ca4b32d3f1 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 16:47:10 +0100 Subject: [PATCH 33/46] feat(autoscale): Activate Swarm workloads Scale-ready deployments can cut over from Compose to Swarm through the authenticated deployment API. Runtime state remains available when policy evaluation is disabled. --- internal/api/autoscale_handlers.go | 91 +++++++++++++++++++++++++ internal/api/autoscale_handlers_test.go | 24 ++++++- internal/api/server.go | 4 ++ internal/autoscale/activator.go | 6 ++ internal/autoscale/controller.go | 14 ++-- internal/autoscale/controller_test.go | 15 ++++ 6 files changed, 149 insertions(+), 5 deletions(-) diff --git a/internal/api/autoscale_handlers.go b/internal/api/autoscale_handlers.go index c6c1a5e..d57c3ae 100644 --- a/internal/api/autoscale_handlers.go +++ b/internal/api/autoscale_handlers.go @@ -1,10 +1,15 @@ package api import ( + "context" + "fmt" "net/http" + "strings" "time" "github.com/flatrun/agent/internal/autoscale" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" "github.com/flatrun/agent/pkg/models" "github.com/gin-gonic/gin" ) @@ -21,6 +26,92 @@ type autoscalePolicyRequest struct { AllowFleetCapacity bool `json:"allow_fleet_capacity"` } +func (s *Server) activateDeploymentAutoscale(c *gin.Context) { + if s.runAutoscaleActivation == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Autoscaling activation is unavailable"}) + return + } + activation, err := s.runAutoscaleActivation(c, c.Param("name")) + if err != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, activation) +} + +func (s *Server) defaultRunAutoscaleActivation(ctx context.Context, name string) (autoscale.Activation, error) { + if s.autoscaleStore == nil { + return autoscale.Activation{}, fmt.Errorf("Autoscaling storage is unavailable") + } + if s.config.Cluster.Orchestrator != string(orchestrator.ProviderSwarm) { + return autoscale.Activation{}, fmt.Errorf("Autoscaling activation requires the Swarm orchestrator") + } + if s.config.Cluster.Routing != "" && s.config.Cluster.Routing != string(routing.ProviderNginx) { + return autoscale.Activation{}, fmt.Errorf("Autoscaling activation requires Nginx routing") + } + deployment, err := s.manager.GetDeployment(name) + if err != nil { + return autoscale.Activation{}, fmt.Errorf("Deployment not found") + } + composeContent, _, err := s.manager.GetComposeFile(name) + if err != nil { + return autoscale.Activation{}, fmt.Errorf("Compose configuration is unavailable") + } + policy, err := s.autoscaleStore.Policy(name) + if err != nil { + return autoscale.Activation{}, err + } + workload, err := autoscale.BuildWorkload(deployment, composeContent, policy.MinReplicas, s.config.Infrastructure.DefaultProxyNetwork) + if err != nil { + return autoscale.Activation{}, err + } + domain, err := autoscaleDomain(deployment, workload) + if err != nil { + return autoscale.Activation{}, err + } + swarmProvider, err := orchestrator.NewSwarmProviderFromEnv() + if err != nil { + return autoscale.Activation{}, fmt.Errorf("create Swarm provider: %w", err) + } + defer swarmProvider.Close() + routeProvider := routing.NewManagedNginxProvider(s.proxyOrchestrator.NginxManager(), s.manager) + stopper := autoscale.ServiceStopperFunc(func(deployment, service string) (string, error) { + return s.manager.StopService(deployment, service) + }) + activation, err := autoscale.NewActivator(swarmProvider, routeProvider, stopper).Activate(ctx, name, deployment.Metadata.Scaling.Service, workload, routing.Route{ + ID: name, Service: deployment.Metadata.Scaling.Service, Domain: domain.Domain, Path: domain.PathPrefix, Protocol: "http", + }) + if err != nil { + return autoscale.Activation{}, err + } + state, err := s.autoscaleStore.State(name) + if err != nil { + return autoscale.Activation{}, err + } + state.Active = true + state.Provider = orchestrator.ProviderSwarm + state.Service = deployment.Metadata.Scaling.Service + state.Replicas = activation.Workload.Desired + state.LastAction = time.Now() + if err := s.autoscaleStore.SetState(name, state); err != nil { + return autoscale.Activation{}, err + } + return activation, nil +} + +func autoscaleDomain(deployment *models.Deployment, workload orchestrator.Workload) (models.DomainConfig, error) { + if deployment.Metadata == nil { + return models.DomainConfig{}, fmt.Errorf("Scale-ready service must have an exposed domain") + } + service := deployment.Metadata.Scaling.Service + for _, domain := range deployment.Metadata.GetDomains() { + if domain.Service == service && strings.TrimSpace(domain.Domain) != "" && domain.ContainerPort == workload.Port { + return domain, nil + } + } + return models.DomainConfig{}, fmt.Errorf("Scale-ready service must have an exposed domain") +} + type autoscalePolicyResponse struct { autoscalePolicyRequest State autoscale.State `json:"state"` diff --git a/internal/api/autoscale_handlers_test.go b/internal/api/autoscale_handlers_test.go index 56bae18..ba07ec4 100644 --- a/internal/api/autoscale_handlers_test.go +++ b/internal/api/autoscale_handlers_test.go @@ -2,6 +2,7 @@ package api import ( "bytes" + "context" "encoding/json" "net/http" "net/http/httptest" @@ -12,6 +13,8 @@ import ( "github.com/flatrun/agent/internal/auth" "github.com/flatrun/agent/internal/autoscale" "github.com/flatrun/agent/internal/docker" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" "github.com/flatrun/agent/pkg/config" "github.com/gin-gonic/gin" ) @@ -40,7 +43,17 @@ func TestDeploymentAutoscalePolicyThroughHTTP(t *testing.T) { t.Fatal(err) } defer store.Close() - server := &Server{config: cfg, manager: docker.NewManager(dir), authManager: authManager, autoscaleStore: store} + activated := "" + server := &Server{ + config: cfg, manager: docker.NewManager(dir), authManager: authManager, autoscaleStore: store, + runAutoscaleActivation: func(_ context.Context, name string) (autoscale.Activation, error) { + activated = name + return autoscale.Activation{ + Workload: orchestrator.Status{Workload: name, Desired: 2, Available: 2}, + Route: routing.Route{ID: name, Domain: "shop.example.com", Protocol: "http"}, + }, nil + }, + } middleware := auth.NewMiddlewareWithManager(&cfg.Auth, authManager) router := gin.New() router.POST("/api/auth/login", middleware.Login) @@ -49,6 +62,7 @@ func TestDeploymentAutoscalePolicyThroughHTTP(t *testing.T) { protected.GET("/deployments/:name/autoscale/compatibility", middleware.RequirePermission(auth.PermDeploymentsRead), middleware.RequireDeploymentAccess(auth.AccessLevelRead), server.getDeploymentAutoscaleCompatibility) protected.PUT("/deployments/:name/autoscale/workload", middleware.RequirePermission(auth.PermDeploymentsWrite), middleware.RequireDeploymentAccess(auth.AccessLevelWrite), server.updateDeploymentAutoscaleWorkload) protected.PUT("/deployments/:name/autoscale", middleware.RequirePermission(auth.PermDeploymentsWrite), middleware.RequireDeploymentAccess(auth.AccessLevelWrite), server.updateDeploymentAutoscalePolicy) + protected.POST("/deployments/:name/autoscale/activate", middleware.RequirePermission(auth.PermDeploymentsWrite), middleware.RequireDeploymentAccess(auth.AccessLevelWrite), server.activateDeploymentAutoscale) token := loginAndGetToken(t, router, "admin", "testadminpass") payload := autoscalePolicyRequest{ @@ -113,4 +127,12 @@ func TestDeploymentAutoscalePolicyThroughHTTP(t *testing.T) { if !compatibility.Compatible || len(compatibility.Services) != 1 || compatibility.Services[0] != "app" { t.Fatalf("compatibility = %#v", compatibility) } + + req = httptest.NewRequest(http.MethodPost, "/api/deployments/shop/autoscale/activate", nil) + req.Header.Set("Authorization", "Bearer "+token) + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK || activated != "shop" { + t.Fatalf("activation returned %d for %q: %s", w.Code, activated, w.Body.String()) + } } diff --git a/internal/api/server.go b/internal/api/server.go index 501a274..7c1511c 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -116,6 +116,8 @@ type Server struct { aiAgents *ai.AgentStore mcpHandler http.Handler + runAutoscaleActivation func(context.Context, string) (autoscale.Activation, error) + jobs *jobRegistry // runDeploymentAction runs a deployment action and streams each output // line to emit. Overridable in tests so they need not shell out to docker. @@ -388,6 +390,7 @@ func New(cfg *config.Config, configPath string) *Server { } s.runDeploymentAction = s.defaultRunDeploymentAction s.runServiceAction = s.defaultRunServiceAction + s.runAutoscaleActivation = s.defaultRunAutoscaleActivation // Built unconditionally: it is stateless and starts nothing, so requests are // gated on the live config flag instead, letting mcp.enabled toggle at runtime. @@ -632,6 +635,7 @@ func (s *Server) setupRoutes() { protected.GET("/deployments/:name/autoscale/compatibility", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentAutoscaleCompatibility) protected.PUT("/deployments/:name/autoscale/workload", s.authMiddleware.RequirePermission(auth.PermDeploymentsWrite), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelWrite), s.updateDeploymentAutoscaleWorkload) protected.PUT("/deployments/:name/autoscale", s.authMiddleware.RequirePermission(auth.PermDeploymentsWrite), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelWrite), s.updateDeploymentAutoscalePolicy) + protected.POST("/deployments/:name/autoscale/activate", s.authMiddleware.RequirePermission(auth.PermDeploymentsWrite), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelWrite), s.activateDeploymentAutoscale) // Image endpoints protected.GET("/images", s.authMiddleware.RequirePermission(auth.PermImagesRead), s.listImages) diff --git a/internal/autoscale/activator.go b/internal/autoscale/activator.go index 66a942f..fa389da 100644 --- a/internal/autoscale/activator.go +++ b/internal/autoscale/activator.go @@ -13,6 +13,12 @@ type ServiceStopper interface { StopService(string, string) (string, error) } +type ServiceStopperFunc func(string, string) (string, error) + +func (f ServiceStopperFunc) StopService(deployment, service string) (string, error) { + return f(deployment, service) +} + type Activator struct { orchestrator orchestrator.Provider routing routing.Provider diff --git a/internal/autoscale/controller.go b/internal/autoscale/controller.go index a653316..c428792 100644 --- a/internal/autoscale/controller.go +++ b/internal/autoscale/controller.go @@ -33,9 +33,13 @@ type Policy struct { } type State struct { - HighWindows int `json:"high_windows"` - LowWindows int `json:"low_windows"` - LastAction time.Time `json:"last_action,omitempty"` + HighWindows int `json:"high_windows"` + LowWindows int `json:"low_windows"` + LastAction time.Time `json:"last_action,omitempty"` + Active bool `json:"active"` + Provider orchestrator.ProviderID `json:"provider,omitempty"` + Service string `json:"service,omitempty"` + Replicas int `json:"replicas,omitempty"` } type Input struct { @@ -71,7 +75,9 @@ func Reconcile(policy Policy, state State, input Input) (State, Decision) { return state, Decision{Action: ActionNotify, Reason: err.Error()} } if !policy.Enabled { - return State{}, Decision{Action: ActionNone, Reason: "Autoscaling is disabled"} + state.HighWindows = 0 + state.LowWindows = 0 + return state, Decision{Action: ActionNone, Reason: "Autoscaling is disabled"} } if input.Now.IsZero() { input.Now = time.Now() diff --git a/internal/autoscale/controller_test.go b/internal/autoscale/controller_test.go index 7bb202e..45ce745 100644 --- a/internal/autoscale/controller_test.go +++ b/internal/autoscale/controller_test.go @@ -72,3 +72,18 @@ func TestReconcileHonorsCooldown(t *testing.T) { t.Fatalf("state = %#v, decision = %#v", unchanged, decision) } } + +func TestReconcileDisabledPreservesManagedWorkloadState(t *testing.T) { + policy := DefaultPolicy() + policy.Enabled = false + state, decision := Reconcile(policy, State{ + HighWindows: 2, LowWindows: 3, Active: true, + Provider: orchestrator.ProviderSwarm, Service: "web", Replicas: 2, + }, Input{}) + if decision.Action != ActionNone || state.HighWindows != 0 || state.LowWindows != 0 { + t.Fatalf("state = %#v, decision = %#v", state, decision) + } + if !state.Active || state.Provider != orchestrator.ProviderSwarm || state.Service != "web" || state.Replicas != 2 { + t.Fatalf("runtime state was cleared: %#v", state) + } +} From c0123f8ac46385e1c8ea1a83df5f09a3f70b2430 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 16:51:29 +0100 Subject: [PATCH 34/46] feat(autoscale): Persist active workload routes Managed deployments retain their provider, service, replica count, and route across agent restarts so later reconciliation can safely update traffic. --- internal/api/autoscale_handlers.go | 1 + internal/autoscale/controller.go | 2 ++ internal/autoscale/store.go | 26 ++++++++++++++++++++++++++ internal/autoscale/store_test.go | 16 +++++++++++++++- 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/internal/api/autoscale_handlers.go b/internal/api/autoscale_handlers.go index d57c3ae..e65de41 100644 --- a/internal/api/autoscale_handlers.go +++ b/internal/api/autoscale_handlers.go @@ -92,6 +92,7 @@ func (s *Server) defaultRunAutoscaleActivation(ctx context.Context, name string) state.Provider = orchestrator.ProviderSwarm state.Service = deployment.Metadata.Scaling.Service state.Replicas = activation.Workload.Desired + state.Route = activation.Route state.LastAction = time.Now() if err := s.autoscaleStore.SetState(name, state); err != nil { return autoscale.Activation{}, err diff --git a/internal/autoscale/controller.go b/internal/autoscale/controller.go index c428792..56c2044 100644 --- a/internal/autoscale/controller.go +++ b/internal/autoscale/controller.go @@ -7,6 +7,7 @@ import ( "github.com/flatrun/agent/internal/capacity" "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" ) type Action string @@ -40,6 +41,7 @@ type State struct { Provider orchestrator.ProviderID `json:"provider,omitempty"` Service string `json:"service,omitempty"` Replicas int `json:"replicas,omitempty"` + Route routing.Route `json:"route,omitempty"` } type Input struct { diff --git a/internal/autoscale/store.go b/internal/autoscale/store.go index bbff29a..6417abb 100644 --- a/internal/autoscale/store.go +++ b/internal/autoscale/store.go @@ -120,3 +120,29 @@ func (s *Store) SetState(deployment string, state State) error { ON CONFLICT(deployment) DO UPDATE SET state_json = excluded.state_json, updated_at = CURRENT_TIMESTAMP`, deployment, raw) return err } + +func (s *Store) ActiveStates() (map[string]State, error) { + s.mu.RLock() + defer s.mu.RUnlock() + rows, err := s.conn.Query(`SELECT deployment, state_json FROM autoscale_states`) + if err != nil { + return nil, err + } + defer rows.Close() + states := make(map[string]State) + for rows.Next() { + var deployment string + var raw string + if err := rows.Scan(&deployment, &raw); err != nil { + return nil, err + } + var state State + if err := json.Unmarshal([]byte(raw), &state); err != nil { + return nil, err + } + if state.Active { + states[deployment] = state + } + } + return states, rows.Err() +} diff --git a/internal/autoscale/store_test.go b/internal/autoscale/store_test.go index 4d59383..fd09bcb 100644 --- a/internal/autoscale/store_test.go +++ b/internal/autoscale/store_test.go @@ -3,6 +3,9 @@ package autoscale import ( "testing" "time" + + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" ) func TestStorePersistsPolicyAndState(t *testing.T) { @@ -14,7 +17,11 @@ func TestStorePersistsPolicyAndState(t *testing.T) { policy := DefaultPolicy() policy.MaxReplicas = 7 policy.AllowFleetCapacity = true - state := State{HighWindows: 2, LastAction: time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC)} + state := State{ + HighWindows: 2, LastAction: time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC), Active: true, + Provider: orchestrator.ProviderSwarm, Service: "web", Replicas: 2, + Route: routing.Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http"}, + } if err := store.SetPolicy("shop", policy); err != nil { t.Fatal(err) } @@ -44,6 +51,13 @@ func TestStorePersistsPolicyAndState(t *testing.T) { if savedState.HighWindows != 2 || !savedState.LastAction.Equal(state.LastAction) { t.Fatalf("unexpected state: %+v", savedState) } + active, err := store.ActiveStates() + if err != nil { + t.Fatal(err) + } + if active["shop"].Route.Domain != "shop.example.com" || active["shop"].Provider != orchestrator.ProviderSwarm { + t.Fatalf("unexpected active states: %+v", active) + } } func TestStoreRejectsInvalidPolicy(t *testing.T) { From 5ef317f23bc0f1e605a3bc453d16bcbf8495a573 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 17:33:35 +0100 Subject: [PATCH 35/46] fix(autoscale): Persist scaled workload routes Successful scale changes now retain the actual replica count and routable backends. Restarted agents no longer reconcile from stale scaling state. --- internal/autoscale/executor.go | 12 ++++++++++++ internal/autoscale/executor_test.go | 11 +++++++++-- internal/autoscale/runner.go | 11 +++++++++++ internal/autoscale/runner_test.go | 22 ++++++++++++++++++++++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/internal/autoscale/executor.go b/internal/autoscale/executor.go index ae5f5bd..f4bf34e 100644 --- a/internal/autoscale/executor.go +++ b/internal/autoscale/executor.go @@ -16,6 +16,7 @@ type Executor struct { type Execution struct { Decision Decision `json:"decision"` Status orchestrator.Status `json:"status"` + Route routing.Route `json:"route,omitempty"` Pending bool `json:"pending"` } @@ -51,6 +52,7 @@ func (e *Executor) Execute(ctx context.Context, workloadID string, route routing if err := e.routing.Reconcile(ctx, updated); err != nil { return result, fmt.Errorf("publish scaled route: %w", err) } + result.Route = updated return result, nil case ActionRemoveReplica: status, err := e.orchestrator.Status(ctx, workloadID) @@ -66,6 +68,16 @@ func (e *Executor) Execute(ctx context.Context, workloadID string, route routing } status, err = e.orchestrator.Scale(ctx, workloadID, decision.Replicas) result.Status = status + if err == nil { + updated, routeErr := routeWithReadyInstances(route, status) + if routeErr != nil { + return result, routeErr + } + if routeErr = e.routing.Reconcile(ctx, updated); routeErr != nil { + return result, fmt.Errorf("publish scaled route: %w", routeErr) + } + result.Route = updated + } return result, err default: return result, fmt.Errorf("Unknown autoscaling action %q", decision.Action) diff --git a/internal/autoscale/executor_test.go b/internal/autoscale/executor_test.go index a96122a..647c96b 100644 --- a/internal/autoscale/executor_test.go +++ b/internal/autoscale/executor_test.go @@ -27,6 +27,10 @@ func (f *fakeOrchestrator) Resize(_ context.Context, _ string, resources orchest func (f *fakeOrchestrator) Scale(_ context.Context, _ string, replicas int) (orchestrator.Status, error) { f.scaledTo = replicas f.status.Desired = replicas + if len(f.status.Instances) > replicas { + f.status.Instances = f.status.Instances[:replicas] + f.status.Available = replicas + } return f.status, nil } func (f *fakeOrchestrator) Status(context.Context, string) (orchestrator.Status, error) { @@ -101,7 +105,10 @@ func TestExecutorWaitsForNewReplicaBeforeRouting(t *testing.T) { func TestExecutorDrainsBeforeRemovingReplica(t *testing.T) { orchestratorProvider := &fakeOrchestrator{status: orchestrator.Status{ Workload: "shop", Desired: 2, Available: 2, - Instances: []orchestrator.Instance{{ID: "one", Ready: true}, {ID: "two", Ready: true}}, + Instances: []orchestrator.Instance{ + {ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}, + {ID: "two", Address: "10.0.0.2:8080", Healthy: true, Ready: true}, + }, }} router := &fakeRouter{} _, err := NewExecutor(orchestratorProvider, router).Execute( @@ -113,7 +120,7 @@ func TestExecutorDrainsBeforeRemovingReplica(t *testing.T) { if err != nil { t.Fatalf("Execute failed: %v", err) } - if router.drained != "two" || orchestratorProvider.scaledTo != 1 { + if router.drained != "two" || orchestratorProvider.scaledTo != 1 || len(router.reconciled.Backends) != 1 { t.Fatalf("drained = %q, scaled = %d", router.drained, orchestratorProvider.scaledTo) } } diff --git a/internal/autoscale/runner.go b/internal/autoscale/runner.go index d6cdb0f..6e432bb 100644 --- a/internal/autoscale/runner.go +++ b/internal/autoscale/runner.go @@ -67,6 +67,17 @@ func (r *Runner) Reconcile(ctx context.Context, deployment string, input Input, r.publishFailure(deployment, err.Error(), events.SeverityCritical) return result, fmt.Errorf("execute autoscaling decision: %w", err) } + if execution.Pending { + return result, nil + } + nextState.Replicas = execution.Status.Desired + if execution.Route.ID != "" { + nextState.Route = execution.Route + } + if err := r.store.SetState(deployment, nextState); err != nil { + return result, fmt.Errorf("save autoscaling execution: %w", err) + } + result.State = nextState return result, nil } diff --git a/internal/autoscale/runner_test.go b/internal/autoscale/runner_test.go index 87f9c91..d265f6c 100644 --- a/internal/autoscale/runner_test.go +++ b/internal/autoscale/runner_test.go @@ -71,3 +71,25 @@ func TestRunnerPublishesCorrelatedBlockedEvent(t *testing.T) { t.Fatalf("unexpected events: %+v", publisher.events) } } + +func TestRunnerPersistsSuccessfulExecution(t *testing.T) { + policy := DefaultPolicy() + policy.ScaleUpWindows = 1 + store := &runnerStore{policy: policy, state: State{Active: true, Replicas: 1}} + provider := &fakeOrchestrator{status: orchestrator.Status{ + Workload: "shop", Desired: 1, Available: 2, + Instances: []orchestrator.Instance{ + {ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}, + {ID: "two", Address: "10.0.0.2:8080", Healthy: true, Ready: true}, + }, + }} + router := &fakeRouter{} + runner := NewRunner(store, NewExecutor(provider, router), nil, "prod-1") + result, err := runner.Reconcile(context.Background(), "shop", Input{Now: time.Now(), Replicas: 1, CPUPercent: 95}, routing.Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http"}) + if err != nil { + t.Fatal(err) + } + if result.State.Replicas != 2 || len(result.State.Route.Backends) != 2 { + t.Fatalf("state = %#v", result.State) + } +} From c7413dcd1896cb354f9c35d412587022cca12275 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 17:34:20 +0100 Subject: [PATCH 36/46] feat(autoscale): Add workload supervisor Active managed workloads can now be reconciled on a fixed interval. Observation failures become one correlated incident per deployment instead of silent gaps. --- internal/autoscale/supervisor.go | 88 +++++++++++++++++++++++++++ internal/autoscale/supervisor_test.go | 55 +++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 internal/autoscale/supervisor.go create mode 100644 internal/autoscale/supervisor_test.go diff --git a/internal/autoscale/supervisor.go b/internal/autoscale/supervisor.go new file mode 100644 index 0000000..a584000 --- /dev/null +++ b/internal/autoscale/supervisor.go @@ -0,0 +1,88 @@ +package autoscale + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/flatrun/agent/internal/events" +) + +type ActiveStore interface { + PolicyStore + ActiveStates() (map[string]State, error) +} + +type RuntimeSession struct { + Input Input + Executor ActionExecutor +} + +type RuntimeFactory interface { + Build(context.Context, string, State) (RuntimeSession, error) +} + +type Supervisor struct { + store ActiveStore + factory RuntimeFactory + publisher EventPublisher + node string + interval time.Duration + mu sync.Mutex +} + +func NewSupervisor(store ActiveStore, factory RuntimeFactory, publisher EventPublisher, node string, interval time.Duration) *Supervisor { + if interval <= 0 { + interval = 30 * time.Second + } + return &Supervisor{store: store, factory: factory, publisher: publisher, node: node, interval: interval} +} + +func (s *Supervisor) Run(ctx context.Context) { + ticker := time.NewTicker(s.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _ = s.Tick(ctx) + } + } +} + +func (s *Supervisor) Tick(ctx context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + states, err := s.store.ActiveStates() + if err != nil { + return fmt.Errorf("load active autoscaling workloads: %w", err) + } + var firstErr error + for deployment, state := range states { + session, err := s.factory.Build(ctx, deployment, state) + if err == nil { + _, err = NewRunner(s.store, session.Executor, s.publisher, s.node).Reconcile(ctx, deployment, session.Input, state.Route) + } + if err != nil { + s.publishFailure(deployment, err) + if firstErr == nil { + firstErr = err + } + } + } + return firstErr +} + +func (s *Supervisor) publishFailure(deployment string, err error) { + if s.publisher == nil { + return + } + _, _ = s.publisher.Publish(events.Event{ + Source: "capacity", Type: "autoscale.observation_failed", Severity: events.SeverityCritical, + Title: "Autoscaling observation failed", Message: err.Error(), + Scope: events.Scope{Node: s.node, Deployment: deployment}, + CorrelationKey: "autoscale-observation:" + s.node + ":" + deployment, OccurredAt: time.Now(), + }) +} diff --git a/internal/autoscale/supervisor_test.go b/internal/autoscale/supervisor_test.go new file mode 100644 index 0000000..f1a1279 --- /dev/null +++ b/internal/autoscale/supervisor_test.go @@ -0,0 +1,55 @@ +package autoscale + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/flatrun/agent/internal/orchestrator" +) + +type supervisorStore struct { + runnerStore + states map[string]State +} + +func (s *supervisorStore) ActiveStates() (map[string]State, error) { return s.states, nil } + +type runtimeFactoryStub struct { + err error + builtFor string + executor *runnerExecutor +} + +func (f *runtimeFactoryStub) Build(_ context.Context, deployment string, state State) (RuntimeSession, error) { + f.builtFor = deployment + return RuntimeSession{Input: Input{Now: time.Now(), Replicas: state.Replicas}, Executor: f.executor}, f.err +} + +func TestSupervisorReconcilesActiveWorkloads(t *testing.T) { + policy := DefaultPolicy() + store := &supervisorStore{ + runnerStore: runnerStore{policy: policy, state: State{Active: true, Replicas: 1}}, + states: map[string]State{"shop": {Active: true, Replicas: 1}}, + } + factory := &runtimeFactoryStub{executor: &runnerExecutor{}} + if err := NewSupervisor(store, factory, nil, "prod-1", time.Second).Tick(context.Background()); err != nil { + t.Fatal(err) + } + if factory.builtFor != "shop" { + t.Fatalf("built for %q", factory.builtFor) + } +} + +func TestSupervisorPublishesObservationFailure(t *testing.T) { + store := &supervisorStore{ + runnerStore: runnerStore{policy: DefaultPolicy()}, + states: map[string]State{"shop": {Active: true, Provider: orchestrator.ProviderSwarm}}, + } + publisher := &runnerPublisher{} + err := NewSupervisor(store, &runtimeFactoryStub{err: errors.New("stats unavailable")}, publisher, "prod-1", time.Second).Tick(context.Background()) + if err == nil || len(publisher.events) != 1 || publisher.events[0].CorrelationKey != "autoscale-observation:prod-1:shop" { + t.Fatalf("error = %v, events = %#v", err, publisher.events) + } +} From fe0c7e33ea8e29322035bcd8c5957fc762d3be96 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 17:37:54 +0100 Subject: [PATCH 37/46] feat(autoscale): Reconcile live Swarm workloads Managed workloads now react to sustained container and host pressure on a fixed interval. Partial observations block scaling, and default placement remains on the local node until Fleet capacity is explicitly granted. --- internal/api/autoscale_handlers.go | 6 ++ internal/api/autoscale_runtime.go | 110 +++++++++++++++++++++++++ internal/api/autoscale_runtime_test.go | 27 ++++++ internal/api/server.go | 14 ++++ internal/autoscale/supervisor.go | 6 ++ internal/docker/stats.go | 37 +++++++-- internal/docker/stats_test.go | 13 +++ internal/orchestrator/provider.go | 5 ++ internal/orchestrator/swarm.go | 1 + internal/orchestrator/swarm_test.go | 5 +- 10 files changed, 217 insertions(+), 7 deletions(-) create mode 100644 internal/api/autoscale_runtime.go create mode 100644 internal/api/autoscale_runtime_test.go create mode 100644 internal/docker/stats_test.go diff --git a/internal/api/autoscale_handlers.go b/internal/api/autoscale_handlers.go index e65de41..c47e5c1 100644 --- a/internal/api/autoscale_handlers.go +++ b/internal/api/autoscale_handlers.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "os" "strings" "time" @@ -65,6 +66,11 @@ func (s *Server) defaultRunAutoscaleActivation(ctx context.Context, name string) if err != nil { return autoscale.Activation{}, err } + hostname, err := os.Hostname() + if err != nil { + return autoscale.Activation{}, fmt.Errorf("resolve local Swarm node: %w", err) + } + workload.Placement.Constraints = []string{"node.hostname==" + hostname} domain, err := autoscaleDomain(deployment, workload) if err != nil { return autoscale.Activation{}, err diff --git a/internal/api/autoscale_runtime.go b/internal/api/autoscale_runtime.go new file mode 100644 index 0000000..859aee2 --- /dev/null +++ b/internal/api/autoscale_runtime.go @@ -0,0 +1,110 @@ +package api + +import ( + "context" + "fmt" + "math" + + "github.com/flatrun/agent/internal/autoscale" + "github.com/flatrun/agent/internal/capacity" + "github.com/flatrun/agent/internal/docker" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" + "github.com/flatrun/agent/internal/system" + "github.com/flatrun/agent/pkg/config" +) + +type autoscaleRuntimeFactory struct { + server *Server +} + +type autoscaleReplicaObservation struct { + Stats docker.ContainerStats + Limits docker.ResourceLimits +} + +func (f autoscaleRuntimeFactory) Build(ctx context.Context, deployment string, state autoscale.State) (autoscale.RuntimeSession, error) { + if state.Provider != orchestrator.ProviderSwarm { + return autoscale.RuntimeSession{}, fmt.Errorf("orchestrator %q does not support active reconciliation", state.Provider) + } + provider, err := orchestrator.NewSwarmProviderFromEnv() + if err != nil { + return autoscale.RuntimeSession{}, err + } + fail := func(err error) (autoscale.RuntimeSession, error) { + _ = provider.Close() + return autoscale.RuntimeSession{}, err + } + status, err := provider.Status(ctx, deployment) + if err != nil { + return fail(err) + } + stats, err := docker.GetManagedDeploymentStats(deployment) + if err != nil { + return fail(fmt.Errorf("read managed workload statistics: %w", err)) + } + if status.Available == 0 || len(stats) != status.Available { + return fail(fmt.Errorf("statistics are available for %d of %d running replicas", len(stats), status.Available)) + } + hostStats, err := system.GetSystemStats() + if err != nil { + return fail(fmt.Errorf("read host capacity: %w", err)) + } + observations := make([]autoscaleReplicaObservation, 0, len(stats)) + for _, stat := range stats { + limits, err := docker.GetContainerResources(stat.ContainerID) + if err != nil { + return fail(fmt.Errorf("read resources for replica %s: %w", stat.ContainerID, err)) + } + observations = append(observations, autoscaleReplicaObservation{Stats: stat, Limits: *limits}) + } + input := autoscaleInput(observations, status, hostStats, f.server.config.Capacity) + routeProvider := routing.NewManagedNginxProvider(f.server.proxyOrchestrator.NginxManager(), f.server.manager) + if err := routeProvider.Reconcile(ctx, state.Route); err != nil { + return fail(fmt.Errorf("restore managed route state: %w", err)) + } + return autoscale.RuntimeSession{ + Input: input, Executor: autoscale.NewExecutor(provider, routeProvider), Close: provider.Close, + }, nil +} + +func autoscaleInput(observations []autoscaleReplicaObservation, status orchestrator.Status, hostStats *system.SystemStats, configPolicy config.CapacityConfig) autoscale.Input { + var selected capacity.Container + var cpuPercent float64 + var memoryPercent float64 + var score float64 + for _, observation := range observations { + stat := observation.Stats + limits := observation.Limits + container := capacity.Container{ + ID: stat.ContainerID, Name: stat.Name, CPUPercent: stat.CPUPercent, CPULimit: limits.CPUs, + MemoryUsage: stat.MemoryUsage, MemoryLimit: uint64(max(limits.MemoryLimit, 0)), + } + containerScore := math.Max(stat.CPUPercent, stat.MemoryPercent) + if selected.ID == "" || containerScore > score { + selected = container + score = containerScore + } + cpuPercent = math.Max(cpuPercent, stat.CPUPercent) + memoryPercent = math.Max(memoryPercent, stat.MemoryPercent) + } + host := capacity.Host{ + CPUCores: float64(hostStats.CPU.Cores), CPUUsagePercent: hostStats.CPU.UsagePercent, + MemoryTotal: hostStats.Memory.Total, MemoryAvailable: hostStats.Memory.Available, + } + policy := capacity.PolicyFromConfig(configPolicy) + diagnosis := capacity.Diagnose(host, selected, policy) + resources := orchestrator.Resources{CPULimit: selected.CPULimit, MemoryLimit: selected.MemoryLimit} + suggested := resources + if diagnosis.Action == capacity.ActionIncreaseCPU { + suggested.CPULimit = diagnosis.RecommendedLimit + } + if diagnosis.Action == capacity.ActionIncreaseMemory { + suggested.MemoryLimit = uint64(diagnosis.RecommendedLimit) + } + return autoscale.Input{ + Replicas: status.Desired, CPUPercent: cpuPercent, MemoryPercent: memoryPercent, + Diagnosis: diagnosis, RequiresFleet: diagnosis.Action == capacity.ActionAddReplica, + CurrentResources: resources, SuggestedResource: suggested, + } +} diff --git a/internal/api/autoscale_runtime_test.go b/internal/api/autoscale_runtime_test.go new file mode 100644 index 0000000..dd0fa62 --- /dev/null +++ b/internal/api/autoscale_runtime_test.go @@ -0,0 +1,27 @@ +package api + +import ( + "testing" + + "github.com/flatrun/agent/internal/capacity" + "github.com/flatrun/agent/internal/docker" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/system" + "github.com/flatrun/agent/pkg/config" +) + +func TestAutoscaleInputUsesMostConstrainedReplica(t *testing.T) { + input := autoscaleInput([]autoscaleReplicaObservation{ + {Stats: docker.ContainerStats{ContainerID: "one", CPUPercent: 25, MemoryUsage: 100, MemoryPercent: 10}, Limits: docker.ResourceLimits{CPUs: 1, MemoryLimit: 1000}}, + {Stats: docker.ContainerStats{ContainerID: "two", CPUPercent: 95, MemoryUsage: 900, MemoryPercent: 90}, Limits: docker.ResourceLimits{CPUs: 1, MemoryLimit: 1000}}, + }, orchestrator.Status{Desired: 2}, &system.SystemStats{ + CPU: system.CPUStats{Cores: 8, UsagePercent: 20}, + Memory: system.MemoryStats{Total: 16 << 30, Available: 8 << 30}, + }, config.CapacityConfig{}) + if input.Replicas != 2 || input.CPUPercent != 95 || input.MemoryPercent != 90 { + t.Fatalf("input = %#v", input) + } + if input.Diagnosis.Action != capacity.ActionIncreaseMemory || input.SuggestedResource.MemoryLimit <= input.CurrentResources.MemoryLimit { + t.Fatalf("diagnosis = %#v, input = %#v", input.Diagnosis, input) + } +} diff --git a/internal/api/server.go b/internal/api/server.go index 7c1511c..b46525e 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -117,6 +117,7 @@ type Server struct { mcpHandler http.Handler runAutoscaleActivation func(context.Context, string) (autoscale.Activation, error) + autoscaleCancel context.CancelFunc jobs *jobRegistry // runDeploymentAction runs a deployment action and streams each output @@ -391,6 +392,16 @@ func New(cfg *config.Config, configPath string) *Server { s.runDeploymentAction = s.defaultRunDeploymentAction s.runServiceAction = s.defaultRunServiceAction s.runAutoscaleActivation = s.defaultRunAutoscaleActivation + if s.autoscaleStore != nil { + autoscaleContext, cancelAutoscale := context.WithCancel(context.Background()) + s.autoscaleCancel = cancelAutoscale + node := cfg.Cluster.ServerName + if node == "" { + node, _ = os.Hostname() + } + supervisor := autoscale.NewSupervisor(s.autoscaleStore, autoscaleRuntimeFactory{server: s}, s.notify, node, 30*time.Second) + go supervisor.Run(autoscaleContext) + } // Built unconditionally: it is stateless and starts nothing, so requests are // gated on the live config flag instead, letting mcp.enabled toggle at runtime. @@ -979,6 +990,9 @@ func (s *Server) Start() error { } func (s *Server) Stop() error { + if s.autoscaleCancel != nil { + s.autoscaleCancel() + } if s.pluginHost != nil { s.pluginHost.Stop() } diff --git a/internal/autoscale/supervisor.go b/internal/autoscale/supervisor.go index a584000..8590723 100644 --- a/internal/autoscale/supervisor.go +++ b/internal/autoscale/supervisor.go @@ -17,6 +17,7 @@ type ActiveStore interface { type RuntimeSession struct { Input Input Executor ActionExecutor + Close func() error } type RuntimeFactory interface { @@ -64,6 +65,11 @@ func (s *Supervisor) Tick(ctx context.Context) error { session, err := s.factory.Build(ctx, deployment, state) if err == nil { _, err = NewRunner(s.store, session.Executor, s.publisher, s.node).Reconcile(ctx, deployment, session.Input, state.Route) + if session.Close != nil { + if closeErr := session.Close(); err == nil { + err = closeErr + } + } } if err != nil { s.publishFailure(deployment, err) diff --git a/internal/docker/stats.go b/internal/docker/stats.go index 6910f94..cf1a602 100644 --- a/internal/docker/stats.go +++ b/internal/docker/stats.go @@ -81,26 +81,51 @@ func GetAllContainerStats() ([]ContainerStats, error) { func listContainerDeploymentLabels() map[string]string { labels := make(map[string]string) - cmd := exec.Command("docker", "ps", "-a", "--format", "{{.ID}}|{{.Names}}|{{.Label \"com.docker.compose.project\"}}") + cmd := exec.Command("docker", "ps", "-a", "--format", "{{.ID}}|{{.Names}}|{{.Label \"com.docker.compose.project\"}}|{{.Label \"flatrun.deployment\"}}") output, err := cmd.Output() if err != nil { log.Printf("warning: failed to list container deployment labels: %v", err) return labels } - for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") { + return parseContainerDeploymentLabels(string(output)) +} + +func parseContainerDeploymentLabels(output string) map[string]string { + labels := make(map[string]string) + for _, line := range strings.Split(strings.TrimSpace(output), "\n") { if line == "" { continue } - parts := strings.SplitN(line, "|", 3) - if len(parts) != 3 || parts[2] == "" { + parts := strings.SplitN(line, "|", 4) + if len(parts) != 4 { continue } - labels[parts[0]] = parts[2] - labels[parts[1]] = parts[2] + deployment := parts[3] + if deployment == "" { + deployment = parts[2] + } + if deployment != "" { + labels[parts[0]] = deployment + labels[parts[1]] = deployment + } } return labels } +func GetManagedDeploymentStats(deployment string) ([]ContainerStats, error) { + stats, err := GetAllContainerStats() + if err != nil { + return nil, err + } + result := make([]ContainerStats, 0, len(stats)) + for _, stat := range stats { + if stat.DeploymentName == deployment { + result = append(result, stat) + } + } + return result, nil +} + func GetDeploymentStats(projectName string) ([]ContainerStats, error) { if projectName == "" { return []ContainerStats{}, nil diff --git a/internal/docker/stats_test.go b/internal/docker/stats_test.go new file mode 100644 index 0000000..7db45df --- /dev/null +++ b/internal/docker/stats_test.go @@ -0,0 +1,13 @@ +package docker + +import "testing" + +func TestParseContainerDeploymentLabelsPrefersManagedWorkload(t *testing.T) { + labels := parseContainerDeploymentLabels("abc|shop.1.task|legacy|shop\ndef|db|database|\n") + if labels["abc"] != "shop" || labels["shop.1.task"] != "shop" { + t.Fatalf("managed labels = %#v", labels) + } + if labels["def"] != "database" || labels["db"] != "database" { + t.Fatalf("Compose labels = %#v", labels) + } +} diff --git a/internal/orchestrator/provider.go b/internal/orchestrator/provider.go index aa59196..822dd74 100644 --- a/internal/orchestrator/provider.go +++ b/internal/orchestrator/provider.go @@ -24,6 +24,10 @@ type Health struct { HealthyThreshold int `json:"healthy_threshold,omitempty"` } +type Placement struct { + Constraints []string `json:"constraints,omitempty"` +} + type Workload struct { ID string `json:"id"` Image string `json:"image"` @@ -37,6 +41,7 @@ type Workload struct { Command []string `json:"command,omitempty"` WorkingDir string `json:"working_dir,omitempty"` Networks []string `json:"networks,omitempty"` + Placement Placement `json:"placement,omitempty"` Stateful bool `json:"stateful"` } diff --git a/internal/orchestrator/swarm.go b/internal/orchestrator/swarm.go index f05cc92..d491ab5 100644 --- a/internal/orchestrator/swarm.go +++ b/internal/orchestrator/swarm.go @@ -184,6 +184,7 @@ func swarmSpec(workload Workload) swarm.ServiceSpec { ContainerSpec: &swarm.ContainerSpec{Image: workload.Image, Labels: labels, Env: workloadEnvironment(workload.Environment), Command: workload.Entrypoint, Args: workload.Command, Dir: workload.WorkingDir}, Resources: swarmResources(workload.Resources), Networks: swarmNetworks(workload.Networks), + Placement: &swarm.Placement{Constraints: workload.Placement.Constraints}, }, Mode: swarm.ServiceMode{Replicated: &swarm.ReplicatedService{Replicas: &replicas}}, } diff --git a/internal/orchestrator/swarm_test.go b/internal/orchestrator/swarm_test.go index 77cb5af..d38791d 100644 --- a/internal/orchestrator/swarm_test.go +++ b/internal/orchestrator/swarm_test.go @@ -61,7 +61,7 @@ func TestSwarmProviderCreatesReplicatedService(t *testing.T) { ID: "shop", Image: "example/shop:1", Port: 8080, Replicas: 1, Resources: Resources{CPURequest: 0.5, CPULimit: 1, MemoryRequest: 256 << 20, MemoryLimit: 512 << 20}, Environment: map[string]string{"APP_ENV": "production"}, Entrypoint: []string{"/app/entrypoint"}, Command: []string{"serve"}, WorkingDir: "/app", - Networks: []string{"proxy"}, + Networks: []string{"proxy"}, Placement: Placement{Constraints: []string{"node.hostname==prod-1"}}, }) if err != nil { t.Fatalf("Apply failed: %v", err) @@ -79,6 +79,9 @@ func TestSwarmProviderCreatesReplicatedService(t *testing.T) { if len(client.created.TaskTemplate.Networks) != 1 || client.created.TaskTemplate.Networks[0].Target != "proxy" { t.Fatalf("networks = %#v", client.created.TaskTemplate.Networks) } + if len(client.created.TaskTemplate.Placement.Constraints) != 1 || client.created.TaskTemplate.Placement.Constraints[0] != "node.hostname==prod-1" { + t.Fatalf("placement = %#v", client.created.TaskTemplate.Placement) + } if status.Desired != 1 || status.Available != 1 || len(status.Instances) != 1 { t.Fatalf("status = %#v", status) } From e261376caba4e2dc47e927ac844f1d1cc6f7e2cf Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 17:42:51 +0100 Subject: [PATCH 38/46] feat(cluster): Enforce capacity lending grants Fleet workloads stay on the local Swarm node unless a connected peer grants capacity. Consenting nodes receive an isolated placement label and enforce the configured replica ceiling. --- internal/api/autoscale_handlers.go | 58 +++++++++++++++++--- internal/api/cluster_handlers.go | 77 +++++++++++++++++++++++++++ internal/api/cluster_handlers_test.go | 44 +++++++++++++++ internal/api/server.go | 1 + internal/orchestrator/provider.go | 3 +- internal/orchestrator/swarm.go | 37 ++++++++++++- internal/orchestrator/swarm_test.go | 26 +++++++++ 7 files changed, 238 insertions(+), 8 deletions(-) diff --git a/internal/api/autoscale_handlers.go b/internal/api/autoscale_handlers.go index c47e5c1..5c4466f 100644 --- a/internal/api/autoscale_handlers.go +++ b/internal/api/autoscale_handlers.go @@ -2,13 +2,14 @@ package api import ( "context" + "encoding/json" "fmt" "net/http" - "os" "strings" "time" "github.com/flatrun/agent/internal/autoscale" + "github.com/flatrun/agent/internal/cluster" "github.com/flatrun/agent/internal/orchestrator" "github.com/flatrun/agent/internal/routing" "github.com/flatrun/agent/pkg/models" @@ -66,11 +67,6 @@ func (s *Server) defaultRunAutoscaleActivation(ctx context.Context, name string) if err != nil { return autoscale.Activation{}, err } - hostname, err := os.Hostname() - if err != nil { - return autoscale.Activation{}, fmt.Errorf("resolve local Swarm node: %w", err) - } - workload.Placement.Constraints = []string{"node.hostname==" + hostname} domain, err := autoscaleDomain(deployment, workload) if err != nil { return autoscale.Activation{}, err @@ -80,6 +76,10 @@ func (s *Server) defaultRunAutoscaleActivation(ctx context.Context, name string) return autoscale.Activation{}, fmt.Errorf("create Swarm provider: %w", err) } defer swarmProvider.Close() + workload.Placement, err = s.autoscalePlacement(ctx, swarmProvider, policy) + if err != nil { + return autoscale.Activation{}, err + } routeProvider := routing.NewManagedNginxProvider(s.proxyOrchestrator.NginxManager(), s.manager) stopper := autoscale.ServiceStopperFunc(func(deployment, service string) (string, error) { return s.manager.StopService(deployment, service) @@ -106,6 +106,52 @@ func (s *Server) defaultRunAutoscaleActivation(ctx context.Context, name string) return activation, nil } +func (s *Server) autoscalePlacement(ctx context.Context, provider *orchestrator.SwarmProvider, policy autoscale.Policy) (orchestrator.Placement, error) { + identity, err := provider.EnsureLocalNodeLabel(ctx, "flatrun.capacity.local", "true") + if err != nil { + return orchestrator.Placement{}, err + } + local := orchestrator.Placement{Constraints: []string{"node.hostname==" + identity.Hostname}} + manager := s.getClusterManager() + if !policy.AllowFleetCapacity || manager == nil { + return local, nil + } + label := capacityNodeLabel(manager.ServerName()) + if _, err := provider.EnsureLocalNodeLabel(ctx, label, "true"); err != nil { + return orchestrator.Placement{}, err + } + claims := manager.ForEachPeer(ctx, func(ctx context.Context, _ string, client *cluster.Client) ([]byte, error) { + data, status, err := client.Post(ctx, "/api/cluster/capacity/claim", nil) + if err != nil { + return nil, err + } + if status != http.StatusOK { + return nil, fmt.Errorf("capacity claim returned status %d", status) + } + return data, nil + }) + allowed := 0 + maxReplicas := 0 + constraint := "node.labels." + label + "==true" + for _, result := range claims { + if result.Error != "" { + continue + } + var claim clusterCapacityClaimResponse + if err := json.Unmarshal(result.Data, &claim); err != nil || !claim.Enabled || claim.Constraint != constraint { + continue + } + allowed++ + if claim.MaxReplicas > 0 && (maxReplicas == 0 || claim.MaxReplicas < maxReplicas) { + maxReplicas = claim.MaxReplicas + } + } + if allowed == 0 { + return local, nil + } + return orchestrator.Placement{Constraints: []string{constraint}, MaxReplicasPerNode: uint64(maxReplicas)}, nil +} + func autoscaleDomain(deployment *models.Deployment, workload orchestrator.Workload) (models.DomainConfig, error) { if deployment.Metadata == nil { return models.DomainConfig{}, fmt.Errorf("Scale-ready service must have an exposed domain") diff --git a/internal/api/cluster_handlers.go b/internal/api/cluster_handlers.go index fe7e74e..7dadd77 100644 --- a/internal/api/cluster_handlers.go +++ b/internal/api/cluster_handlers.go @@ -3,8 +3,10 @@ package api import ( "context" "crypto/rand" + "crypto/sha256" "database/sql" "encoding/base64" + "encoding/hex" "encoding/json" "fmt" "io" @@ -38,6 +40,81 @@ type clusterProvidersResponse struct { K3s config.K3sConfig `json:"k3s"` } +type clusterCapacityClaimResponse struct { + Enabled bool `json:"enabled"` + Reason string `json:"reason"` + Node orchestrator.NodeIdentity `json:"node"` + Constraint string `json:"constraint,omitempty"` + MaxCPU float64 `json:"max_cpu,omitempty"` + MaxMemory uint64 `json:"max_memory,omitempty"` + MaxReplicas int `json:"max_replicas,omitempty"` +} + +func (s *Server) clusterCapacityClaim(c *gin.Context) { + peer, err := clusterPeerFromActor(auth.GetActorFromContext(c)) + if err != nil { + c.JSON(http.StatusForbidden, gin.H{"error": err.Error()}) + return + } + mgr := s.getClusterManager() + if mgr == nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) + return + } + policy, err := mgr.DB().GetPeerPolicy(peer) + if err != nil { + c.JSON(http.StatusForbidden, gin.H{"error": "Capacity has not been granted to this peer"}) + return + } + grant, ok := capacityOfferGrant(*policy) + if !ok { + c.JSON(http.StatusOK, clusterCapacityClaimResponse{Reason: "This server has not permitted Fleet workloads from this peer"}) + return + } + provider, err := orchestrator.NewSwarmProviderFromEnv() + if err != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()}) + return + } + defer provider.Close() + label := capacityNodeLabel(peer) + node, err := provider.EnsureLocalNodeLabel(c, label, "true") + if err != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, clusterCapacityClaimResponse{ + Enabled: true, Reason: "This node permits Fleet workloads from this peer", Node: node, + Constraint: "node.labels." + label + "==true", + MaxCPU: grant.MaxCPU, MaxMemory: grant.MaxMemory, MaxReplicas: grant.MaxReplicas, + }) +} + +func clusterPeerFromActor(actor *auth.ActorContext) (string, error) { + if actor == nil || actor.APIKey == nil { + return "", fmt.Errorf("A Fleet peer credential is required") + } + peer, ok := strings.CutPrefix(actor.APIKey.Name, "cluster-peer-") + if !ok || strings.TrimSpace(peer) == "" { + return "", fmt.Errorf("A Fleet peer credential is required") + } + return peer, nil +} + +func capacityOfferGrant(policy cluster.PeerPolicy) (cluster.Grant, bool) { + for _, grant := range policy.Grants { + if grant.Capability == cluster.CapabilityCapacityOffer { + return grant, true + } + } + return cluster.Grant{}, false +} + +func capacityNodeLabel(peer string) string { + sum := sha256.Sum256([]byte(peer)) + return "flatrun.capacity." + hex.EncodeToString(sum[:6]) +} + func (s *Server) clusterProviders(c *gin.Context) { orchestratorID := s.config.Cluster.Orchestrator if orchestratorID == "" { diff --git a/internal/api/cluster_handlers_test.go b/internal/api/cluster_handlers_test.go index 7a6644b..bdd94fe 100644 --- a/internal/api/cluster_handlers_test.go +++ b/internal/api/cluster_handlers_test.go @@ -20,6 +20,23 @@ import ( "github.com/gin-gonic/gin" ) +func TestCapacityClaimUsesPeerSpecificGrant(t *testing.T) { + peer, err := clusterPeerFromActor(&auth.ActorContext{APIKey: &auth.APIKey{Name: "cluster-peer-prod1"}}) + if err != nil || peer != "prod1" { + t.Fatalf("peer = %q, error = %v", peer, err) + } + grant, ok := capacityOfferGrant(cluster.PeerPolicy{Grants: []cluster.Grant{ + {Capability: cluster.CapabilityCapacityRead}, + {Capability: cluster.CapabilityCapacityOffer, MaxCPU: 2, MaxMemory: 4 << 30, MaxReplicas: 3}, + }}) + if !ok || grant.MaxReplicas != 3 || grant.MaxCPU != 2 { + t.Fatalf("grant = %#v, found = %v", grant, ok) + } + if capacityNodeLabel("prod1") == capacityNodeLabel("prod2") { + t.Fatal("peer labels must be isolated") + } +} + type testClusterEnv struct { server *Server router *gin.Engine @@ -121,6 +138,7 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool clusterGroup.GET("/deployments", server.clusterAggregateDeployments) clusterGroup.GET("/stats", server.clusterAggregateStats) clusterGroup.GET("/capacity", server.clusterAggregateCapacity) + clusterGroup.POST("/capacity/claim", server.clusterCapacityClaim) } } @@ -377,6 +395,32 @@ func TestClusterAPIKeyEnforcesPermissionsThroughHTTP(t *testing.T) { } } +func TestCapacityClaimDeniesUnpermittedPeerThroughHTTP(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + const rawKey = "peer-capacity-key-for-test" + if err := env.server.getClusterManager().AddPeer("server-b", "http://server-b.invalid", "remote-key"); err != nil { + t.Fatal(err) + } + if err := env.server.createClusterAPIKey(rawKey, "server-b"); err != nil { + t.Fatal(err) + } + req := httptest.NewRequest(http.MethodPost, "/api/cluster/capacity/claim", nil) + req.Header.Set("Authorization", "Bearer "+rawKey) + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + var response clusterCapacityClaimResponse + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.Enabled || response.Reason == "" { + t.Fatalf("response = %#v", response) + } +} + func clusterLogin(t *testing.T, router *gin.Engine) string { t.Helper() body, _ := json.Marshal(map[string]string{ diff --git a/internal/api/server.go b/internal/api/server.go index b46525e..0039e71 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -922,6 +922,7 @@ func (s *Server) setupRoutes() { clusterGroup.GET("/deployments", s.clusterAggregateDeployments) clusterGroup.GET("/stats", s.clusterAggregateStats) clusterGroup.GET("/capacity", s.clusterAggregateCapacity) + clusterGroup.POST("/capacity/claim", s.clusterCapacityClaim) clusterGroup.GET("/providers", s.clusterProviders) clusterGroup.PUT("/providers", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.updateClusterProviders) } diff --git a/internal/orchestrator/provider.go b/internal/orchestrator/provider.go index 822dd74..fb6f60d 100644 --- a/internal/orchestrator/provider.go +++ b/internal/orchestrator/provider.go @@ -25,7 +25,8 @@ type Health struct { } type Placement struct { - Constraints []string `json:"constraints,omitempty"` + Constraints []string `json:"constraints,omitempty"` + MaxReplicasPerNode uint64 `json:"max_replicas_per_node,omitempty"` } type Workload struct { diff --git a/internal/orchestrator/swarm.go b/internal/orchestrator/swarm.go index d491ab5..d467fcb 100644 --- a/internal/orchestrator/swarm.go +++ b/internal/orchestrator/swarm.go @@ -15,6 +15,9 @@ import ( type swarmClient interface { SwarmInspect(context.Context, client.SwarmInspectOptions) (client.SwarmInspectResult, error) + Info(context.Context, client.InfoOptions) (client.SystemInfoResult, error) + NodeInspect(context.Context, string, client.NodeInspectOptions) (client.NodeInspectResult, error) + NodeUpdate(context.Context, string, client.NodeUpdateOptions) (client.NodeUpdateResult, error) ServiceCreate(context.Context, client.ServiceCreateOptions) (client.ServiceCreateResult, error) ServiceInspect(context.Context, string, client.ServiceInspectOptions) (client.ServiceInspectResult, error) ServiceUpdate(context.Context, string, client.ServiceUpdateOptions) (client.ServiceUpdateResult, error) @@ -22,6 +25,11 @@ type swarmClient interface { TaskList(context.Context, client.TaskListOptions) (client.TaskListResult, error) } +type NodeIdentity struct { + ID string `json:"id"` + Hostname string `json:"hostname"` +} + type SwarmProvider struct { client swarmClient } @@ -52,6 +60,30 @@ func (p *SwarmProvider) Close() error { return nil } +func (p *SwarmProvider) EnsureLocalNodeLabel(ctx context.Context, key, value string) (NodeIdentity, error) { + info, err := p.client.Info(ctx, client.InfoOptions{}) + if err != nil { + return NodeIdentity{}, fmt.Errorf("inspect Docker host: %w", err) + } + if info.Info.Swarm.NodeID == "" { + return NodeIdentity{}, fmt.Errorf("Docker host is not a Swarm node") + } + inspected, err := p.client.NodeInspect(ctx, info.Info.Swarm.NodeID, client.NodeInspectOptions{}) + if err != nil { + return NodeIdentity{}, fmt.Errorf("inspect local Swarm node: %w", err) + } + if inspected.Node.Spec.Labels == nil { + inspected.Node.Spec.Labels = make(map[string]string) + } + if inspected.Node.Spec.Labels[key] != value { + inspected.Node.Spec.Labels[key] = value + if _, err := p.client.NodeUpdate(ctx, inspected.Node.ID, client.NodeUpdateOptions{Version: inspected.Node.Version, Spec: inspected.Node.Spec}); err != nil { + return NodeIdentity{}, fmt.Errorf("label local Swarm node: %w", err) + } + } + return NodeIdentity{ID: inspected.Node.ID, Hostname: inspected.Node.Description.Hostname}, nil +} + func (p *SwarmProvider) ID() ProviderID { return ProviderSwarm } @@ -184,7 +216,10 @@ func swarmSpec(workload Workload) swarm.ServiceSpec { ContainerSpec: &swarm.ContainerSpec{Image: workload.Image, Labels: labels, Env: workloadEnvironment(workload.Environment), Command: workload.Entrypoint, Args: workload.Command, Dir: workload.WorkingDir}, Resources: swarmResources(workload.Resources), Networks: swarmNetworks(workload.Networks), - Placement: &swarm.Placement{Constraints: workload.Placement.Constraints}, + Placement: &swarm.Placement{ + Constraints: workload.Placement.Constraints, + MaxReplicas: workload.Placement.MaxReplicasPerNode, + }, }, Mode: swarm.ServiceMode{Replicated: &swarm.ReplicatedService{Replicas: &replicas}}, } diff --git a/internal/orchestrator/swarm_test.go b/internal/orchestrator/swarm_test.go index d38791d..d8fcac8 100644 --- a/internal/orchestrator/swarm_test.go +++ b/internal/orchestrator/swarm_test.go @@ -7,6 +7,7 @@ import ( "github.com/containerd/errdefs" "github.com/moby/moby/api/types/swarm" + "github.com/moby/moby/api/types/system" "github.com/moby/moby/client" ) @@ -15,6 +16,20 @@ type fakeSwarmClient struct { tasks []swarm.Task created *swarm.ServiceSpec updated *swarm.ServiceSpec + node swarm.Node +} + +func (f *fakeSwarmClient) Info(_ context.Context, _ client.InfoOptions) (client.SystemInfoResult, error) { + return client.SystemInfoResult{Info: system.Info{Swarm: swarm.Info{NodeID: f.node.ID}}}, nil +} + +func (f *fakeSwarmClient) NodeInspect(_ context.Context, _ string, _ client.NodeInspectOptions) (client.NodeInspectResult, error) { + return client.NodeInspectResult{Node: f.node}, nil +} + +func (f *fakeSwarmClient) NodeUpdate(_ context.Context, _ string, options client.NodeUpdateOptions) (client.NodeUpdateResult, error) { + f.node.Spec = options.Spec + return client.NodeUpdateResult{}, nil } func (f *fakeSwarmClient) SwarmInspect(_ context.Context, _ client.SwarmInspectOptions) (client.SwarmInspectResult, error) { @@ -117,3 +132,14 @@ func TestSwarmProviderRejectsUnsafeStatefulReplication(t *testing.T) { t.Fatal("stateful replication should require a storage policy") } } + +func TestSwarmProviderLabelsLocalNodeForCapacityGrant(t *testing.T) { + client := &fakeSwarmClient{node: swarm.Node{ID: "node-1", Description: swarm.NodeDescription{Hostname: "prod-1"}}} + identity, err := NewSwarmProvider(client).EnsureLocalNodeLabel(context.Background(), "flatrun.capacity.origin", "true") + if err != nil { + t.Fatal(err) + } + if identity.ID != "node-1" || identity.Hostname != "prod-1" || client.node.Spec.Labels["flatrun.capacity.origin"] != "true" { + t.Fatalf("identity = %#v, node = %#v", identity, client.node) + } +} From c29c6670ede16e69eb0b4ad16a3d87937cd104d4 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 17:47:22 +0100 Subject: [PATCH 39/46] feat(autoscale): Activate K3s workloads K3s deployments now receive a stable Service, native Traefik ingress, and recurring utilization checks through the Kubernetes Metrics API. --- internal/api/autoscale_handlers.go | 44 +++++--- internal/api/autoscale_runtime.go | 22 ++++ internal/api/cluster_handlers.go | 18 +++- internal/api/cluster_handlers_test.go | 4 +- internal/autoscale/executor.go | 4 + internal/orchestrator/k3s.go | 112 ++++++++++++++++++++- internal/orchestrator/k3s_test.go | 26 ++++- internal/orchestrator/provider.go | 9 ++ internal/routing/k3s_ingress.go | 140 ++++++++++++++++++++++++++ internal/routing/k3s_ingress_test.go | 37 +++++++ 10 files changed, 393 insertions(+), 23 deletions(-) create mode 100644 internal/routing/k3s_ingress.go create mode 100644 internal/routing/k3s_ingress_test.go diff --git a/internal/api/autoscale_handlers.go b/internal/api/autoscale_handlers.go index 5c4466f..2481841 100644 --- a/internal/api/autoscale_handlers.go +++ b/internal/api/autoscale_handlers.go @@ -45,11 +45,17 @@ func (s *Server) defaultRunAutoscaleActivation(ctx context.Context, name string) if s.autoscaleStore == nil { return autoscale.Activation{}, fmt.Errorf("Autoscaling storage is unavailable") } - if s.config.Cluster.Orchestrator != string(orchestrator.ProviderSwarm) { - return autoscale.Activation{}, fmt.Errorf("Autoscaling activation requires the Swarm orchestrator") + orchestratorID := orchestrator.ProviderID(s.config.Cluster.Orchestrator) + routingID := routing.ProviderID(s.config.Cluster.Routing) + if routingID == "" { + routingID = routing.ProviderNginx } - if s.config.Cluster.Routing != "" && s.config.Cluster.Routing != string(routing.ProviderNginx) { - return autoscale.Activation{}, fmt.Errorf("Autoscaling activation requires Nginx routing") + if orchestratorID != orchestrator.ProviderSwarm && orchestratorID != orchestrator.ProviderK3s { + return autoscale.Activation{}, fmt.Errorf("Autoscaling activation requires Swarm or K3s") + } + if (orchestratorID == orchestrator.ProviderSwarm && routingID != routing.ProviderNginx) || + (orchestratorID == orchestrator.ProviderK3s && routingID != routing.ProviderTraefik) { + return autoscale.Activation{}, fmt.Errorf("The selected orchestrator and routing providers are incompatible") } deployment, err := s.manager.GetDeployment(name) if err != nil { @@ -71,20 +77,28 @@ func (s *Server) defaultRunAutoscaleActivation(ctx context.Context, name string) if err != nil { return autoscale.Activation{}, err } - swarmProvider, err := orchestrator.NewSwarmProviderFromEnv() - if err != nil { - return autoscale.Activation{}, fmt.Errorf("create Swarm provider: %w", err) - } - defer swarmProvider.Close() - workload.Placement, err = s.autoscalePlacement(ctx, swarmProvider, policy) - if err != nil { - return autoscale.Activation{}, err + var orchestratorProvider orchestrator.Provider + var routeProvider routing.Provider + if orchestratorID == orchestrator.ProviderSwarm { + swarmProvider, err := orchestrator.NewSwarmProviderFromEnv() + if err != nil { + return autoscale.Activation{}, fmt.Errorf("create Swarm provider: %w", err) + } + defer swarmProvider.Close() + workload.Placement, err = s.autoscalePlacement(ctx, swarmProvider, policy) + if err != nil { + return autoscale.Activation{}, err + } + orchestratorProvider = swarmProvider + routeProvider = routing.NewManagedNginxProvider(s.proxyOrchestrator.NginxManager(), s.manager) + } else { + orchestratorProvider = orchestrator.NewK3sProvider(s.config.Cluster.K3s.Kubeconfig, s.config.Cluster.K3s.Namespace) + routeProvider = routing.NewK3sIngressProvider(s.config.Cluster.K3s.Kubeconfig, s.config.Cluster.K3s.Namespace) } - routeProvider := routing.NewManagedNginxProvider(s.proxyOrchestrator.NginxManager(), s.manager) stopper := autoscale.ServiceStopperFunc(func(deployment, service string) (string, error) { return s.manager.StopService(deployment, service) }) - activation, err := autoscale.NewActivator(swarmProvider, routeProvider, stopper).Activate(ctx, name, deployment.Metadata.Scaling.Service, workload, routing.Route{ + activation, err := autoscale.NewActivator(orchestratorProvider, routeProvider, stopper).Activate(ctx, name, deployment.Metadata.Scaling.Service, workload, routing.Route{ ID: name, Service: deployment.Metadata.Scaling.Service, Domain: domain.Domain, Path: domain.PathPrefix, Protocol: "http", }) if err != nil { @@ -95,7 +109,7 @@ func (s *Server) defaultRunAutoscaleActivation(ctx context.Context, name string) return autoscale.Activation{}, err } state.Active = true - state.Provider = orchestrator.ProviderSwarm + state.Provider = orchestratorID state.Service = deployment.Metadata.Scaling.Service state.Replicas = activation.Workload.Desired state.Route = activation.Route diff --git a/internal/api/autoscale_runtime.go b/internal/api/autoscale_runtime.go index 859aee2..842564f 100644 --- a/internal/api/autoscale_runtime.go +++ b/internal/api/autoscale_runtime.go @@ -24,6 +24,28 @@ type autoscaleReplicaObservation struct { } func (f autoscaleRuntimeFactory) Build(ctx context.Context, deployment string, state autoscale.State) (autoscale.RuntimeSession, error) { + if state.Provider == orchestrator.ProviderK3s { + provider := orchestrator.NewK3sProvider(f.server.config.Cluster.K3s.Kubeconfig, f.server.config.Cluster.K3s.Namespace) + status, err := provider.Status(ctx, deployment) + if err != nil { + return autoscale.RuntimeSession{}, err + } + usage, err := provider.Metrics(ctx, deployment) + if err != nil { + return autoscale.RuntimeSession{}, err + } + routeProvider := routing.NewK3sIngressProvider(f.server.config.Cluster.K3s.Kubeconfig, f.server.config.Cluster.K3s.Namespace) + if err := routeProvider.Reconcile(ctx, state.Route); err != nil { + return autoscale.RuntimeSession{}, fmt.Errorf("restore managed route state: %w", err) + } + return autoscale.RuntimeSession{ + Input: autoscale.Input{ + Replicas: status.Desired, CPUPercent: usage.CPUPercent, MemoryPercent: usage.MemoryPercent, + Diagnosis: capacity.Diagnosis{Pressure: capacity.PressureNone, Action: capacity.ActionNone, Reason: "K3s workload metrics are within policy"}, + }, + Executor: autoscale.NewExecutor(provider, routeProvider), + }, nil + } if state.Provider != orchestrator.ProviderSwarm { return autoscale.RuntimeSession{}, fmt.Errorf("orchestrator %q does not support active reconciliation", state.Provider) } diff --git a/internal/api/cluster_handlers.go b/internal/api/cluster_handlers.go index 7dadd77..7897bdb 100644 --- a/internal/api/cluster_handlers.go +++ b/internal/api/cluster_handlers.go @@ -192,7 +192,7 @@ func (s *Server) checkRouting(ctx context.Context, id routing.ProviderID) error } return nil case routing.ProviderTraefik: - return fmt.Errorf("Traefik adapter is not configured") + return orchestrator.NewK3sProvider(s.config.Cluster.K3s.Kubeconfig, s.config.Cluster.K3s.Namespace).Ready(ctx) default: return fmt.Errorf("routing provider %q is not supported", id) } @@ -212,6 +212,12 @@ func (s *Server) updateClusterProviders(c *gin.Context) { } orchestratorID := orchestrator.ProviderID(strings.TrimSpace(req.Orchestrator)) routingID := routing.ProviderID(strings.TrimSpace(req.Routing)) + if (orchestratorID == orchestrator.ProviderK3s && routingID != routing.ProviderTraefik) || + (orchestratorID == orchestrator.ProviderSwarm && routingID != routing.ProviderNginx) || + (orchestratorID == orchestrator.ProviderStandalone && routingID != routing.ProviderNginx) { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": "The selected orchestrator and routing providers are incompatible"}) + return + } var orchestratorErr error if orchestratorID == orchestrator.ProviderK3s && s.probeOrchestrator == nil { orchestratorErr = orchestrator.NewK3sProvider(req.K3s.Kubeconfig, req.K3s.Namespace).Ready(c) @@ -222,8 +228,14 @@ func (s *Server) updateClusterProviders(c *gin.Context) { c.JSON(http.StatusUnprocessableEntity, gin.H{"error": orchestratorErr.Error()}) return } - if err := s.checkRouting(c, routingID); err != nil { - c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()}) + var routingErr error + if routingID == routing.ProviderTraefik && s.probeRouting == nil { + routingErr = orchestrator.NewK3sProvider(req.K3s.Kubeconfig, req.K3s.Namespace).Ready(c) + } else { + routingErr = s.checkRouting(c, routingID) + } + if routingErr != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": routingErr.Error()}) return } diff --git a/internal/api/cluster_handlers_test.go b/internal/api/cluster_handlers_test.go index bdd94fe..735fae7 100644 --- a/internal/api/cluster_handlers_test.go +++ b/internal/api/cluster_handlers_test.go @@ -867,7 +867,7 @@ func TestUpdateClusterProvidersPersistsK3sConnection(t *testing.T) { env.server.probeOrchestrator = func(_ context.Context, _ orchestrator.ProviderID) error { return nil } env.server.probeRouting = func(_ context.Context, _ routing.ProviderID) error { return nil } - body := bytes.NewBufferString(`{"orchestrator":"k3s","routing":"nginx","k3s":{"kubeconfig":"/etc/rancher/k3s/k3s.yaml","namespace":"flatrun"}}`) + body := bytes.NewBufferString(`{"orchestrator":"k3s","routing":"traefik","k3s":{"kubeconfig":"/etc/rancher/k3s/k3s.yaml","namespace":"flatrun"}}`) req := httptest.NewRequest(http.MethodPut, "/api/cluster/providers", body) req.Header.Set("Authorization", "Bearer "+clusterLogin(t, env.router)) req.Header.Set("Content-Type", "application/json") @@ -881,7 +881,7 @@ func TestUpdateClusterProvidersPersistsK3sConnection(t *testing.T) { if err != nil { t.Fatal(err) } - if saved.Cluster.K3s.Kubeconfig != "/etc/rancher/k3s/k3s.yaml" || saved.Cluster.K3s.Namespace != "flatrun" { + if saved.Cluster.Routing != "traefik" || saved.Cluster.K3s.Kubeconfig != "/etc/rancher/k3s/k3s.yaml" || saved.Cluster.K3s.Namespace != "flatrun" { t.Fatalf("unexpected K3s connection: %+v", saved.Cluster.K3s) } } diff --git a/internal/autoscale/executor.go b/internal/autoscale/executor.go index f4bf34e..4040ade 100644 --- a/internal/autoscale/executor.go +++ b/internal/autoscale/executor.go @@ -68,6 +68,10 @@ func (e *Executor) Execute(ctx context.Context, workloadID string, route routing } status, err = e.orchestrator.Scale(ctx, workloadID, decision.Replicas) result.Status = status + if err == nil && e.routing.ID() == routing.ProviderTraefik { + result.Route = route + return result, nil + } if err == nil { updated, routeErr := routeWithReadyInstances(route, status) if routeErr != nil { diff --git a/internal/orchestrator/k3s.go b/internal/orchestrator/k3s.go index 692691e..cd580d1 100644 --- a/internal/orchestrator/k3s.go +++ b/internal/orchestrator/k3s.go @@ -165,6 +165,104 @@ func (p *K3sProvider) Status(ctx context.Context, id string) (Status, error) { return status, nil } +func (p *K3sProvider) Metrics(ctx context.Context, id string) (Usage, error) { + deploymentRaw, err := p.run(ctx, nil, "get", "deployment", id, "-o", "json") + if err != nil { + return Usage{}, fmt.Errorf("inspect K3s workload resources: %w", err) + } + var deployment struct { + Spec struct { + Replicas int `json:"replicas"` + Template struct { + Spec struct { + Containers []struct { + Resources struct { + Limits map[string]string `json:"limits"` + } `json:"resources"` + } `json:"containers"` + } `json:"spec"` + } `json:"template"` + } `json:"spec"` + } + if err := json.Unmarshal(deploymentRaw, &deployment); err != nil { + return Usage{}, fmt.Errorf("decode K3s workload resources: %w", err) + } + if deployment.Spec.Replicas < 1 || len(deployment.Spec.Template.Spec.Containers) == 0 { + return Usage{}, fmt.Errorf("K3s workload has no measurable replicas") + } + limits := deployment.Spec.Template.Spec.Containers[0].Resources.Limits + cpuLimit, err := parseCPUQuantity(limits["cpu"]) + if err != nil || cpuLimit <= 0 { + return Usage{}, fmt.Errorf("K3s workload needs a CPU limit for autoscaling") + } + memoryLimit, err := parseMemoryQuantity(limits["memory"]) + if err != nil || memoryLimit <= 0 { + return Usage{}, fmt.Errorf("K3s workload needs a memory limit for autoscaling") + } + metricsRaw, err := p.run(ctx, nil, "get", "--raw", "/apis/metrics.k8s.io/v1beta1/namespaces/"+p.namespace+"/pods?labelSelector=flatrun.workload%3D"+id) + if err != nil { + return Usage{}, fmt.Errorf("read K3s Metrics API: %w", err) + } + var metrics struct { + Items []struct { + Containers []struct { + Usage map[string]string `json:"usage"` + } `json:"containers"` + } `json:"items"` + } + if err := json.Unmarshal(metricsRaw, &metrics); err != nil { + return Usage{}, fmt.Errorf("decode K3s Metrics API: %w", err) + } + var cpuUsed float64 + var memoryUsed float64 + for _, pod := range metrics.Items { + for _, container := range pod.Containers { + cpu, cpuErr := parseCPUQuantity(container.Usage["cpu"]) + memory, memoryErr := parseMemoryQuantity(container.Usage["memory"]) + if cpuErr != nil || memoryErr != nil { + return Usage{}, fmt.Errorf("decode K3s container usage") + } + cpuUsed += cpu + memoryUsed += memory + } + } + replicas := float64(deployment.Spec.Replicas) + return Usage{ + CPUPercent: cpuUsed / (cpuLimit * replicas) * 100, + MemoryPercent: memoryUsed / (memoryLimit * replicas) * 100, + }, nil +} + +func parseCPUQuantity(value string) (float64, error) { + multiplier := 1.0 + for suffix, factor := range map[string]float64{"n": 1e-9, "u": 1e-6, "m": 1e-3} { + if strings.HasSuffix(value, suffix) { + multiplier = factor + value = strings.TrimSuffix(value, suffix) + break + } + } + parsed, err := strconv.ParseFloat(value, 64) + return parsed * multiplier, err +} + +func parseMemoryQuantity(value string) (float64, error) { + multipliers := map[string]float64{ + "Ki": 1 << 10, "Mi": 1 << 20, "Gi": 1 << 30, "Ti": 1 << 40, + "K": 1e3, "M": 1e6, "G": 1e9, "T": 1e12, + } + multiplier := 1.0 + for suffix, factor := range multipliers { + if strings.HasSuffix(value, suffix) { + multiplier = factor + value = strings.TrimSuffix(value, suffix) + break + } + } + parsed, err := strconv.ParseFloat(value, 64) + return parsed * multiplier, err +} + func (p *K3sProvider) Remove(ctx context.Context, id string) error { if _, err := p.run(ctx, nil, "delete", "deployment", id, "--ignore-not-found=true"); err != nil { return fmt.Errorf("remove K3s workload: %w", err) @@ -222,13 +320,25 @@ func k3sManifest(workload Workload) map[string]any { if workload.Port > 0 { container["ports"] = []any{map[string]any{"containerPort": workload.Port}} } - return map[string]any{ + deployment := map[string]any{ "apiVersion": "apps/v1", "kind": "Deployment", "metadata": map[string]any{"name": workload.ID, "labels": labels}, "spec": map[string]any{"replicas": workload.Replicas, "selector": map[string]any{"matchLabels": map[string]string{"flatrun.workload": workload.ID}}, "template": map[string]any{ "metadata": map[string]any{"labels": labels}, "spec": map[string]any{"containers": []any{container}}, }}, } + items := []any{deployment} + if workload.Port > 0 { + items = append(items, map[string]any{ + "apiVersion": "v1", "kind": "Service", + "metadata": map[string]any{"name": workload.ID, "labels": labels}, + "spec": map[string]any{ + "selector": map[string]string{"flatrun.workload": workload.ID}, + "ports": []any{map[string]any{"name": "http", "port": workload.Port, "targetPort": workload.Port}}, + }, + }) + } + return map[string]any{"apiVersion": "v1", "kind": "List", "items": items} } func k3sResources(resources Resources) map[string]any { diff --git a/internal/orchestrator/k3s_test.go b/internal/orchestrator/k3s_test.go index 91e33ba..93aec92 100644 --- a/internal/orchestrator/k3s_test.go +++ b/internal/orchestrator/k3s_test.go @@ -51,10 +51,16 @@ func TestK3sApplyUsesConfiguredClusterAndNamespace(t *testing.T) { if err := json.Unmarshal(runner.calls[0].input, &manifest); err != nil { t.Fatal(err) } - if manifest["kind"] != "Deployment" { + if manifest["kind"] != "List" { t.Fatalf("manifest = %#v", manifest) } - spec := manifest["spec"].(map[string]any) + items := manifest["items"].([]any) + deployment := items[0].(map[string]any) + service := items[1].(map[string]any) + if deployment["kind"] != "Deployment" || service["kind"] != "Service" { + t.Fatalf("items = %#v", items) + } + spec := deployment["spec"].(map[string]any) template := spec["template"].(map[string]any) podSpec := template["spec"].(map[string]any) container := podSpec["containers"].([]any)[0].(map[string]any) @@ -79,3 +85,19 @@ func TestK3sStatusReturnsRoutableReadyPods(t *testing.T) { t.Fatalf("status = %#v", status) } } + +func TestK3sMetricsReturnsLimitUtilization(t *testing.T) { + runner := &fakeKubectl{responses: [][]byte{ + []byte(`{"spec":{"replicas":2,"template":{"spec":{"containers":[{"resources":{"limits":{"cpu":"500m","memory":"256Mi"}}}]}}}}`), + []byte(`{"items":[{"containers":[{"usage":{"cpu":"250m","memory":"128Mi"}}]},{"containers":[{"usage":{"cpu":"500m","memory":"256Mi"}}]}]}`), + }} + provider := NewK3sProvider("", "apps") + provider.runner = runner + usage, err := provider.Metrics(context.Background(), "shop") + if err != nil { + t.Fatal(err) + } + if usage.CPUPercent != 75 || usage.MemoryPercent != 75 { + t.Fatalf("usage = %#v", usage) + } +} diff --git a/internal/orchestrator/provider.go b/internal/orchestrator/provider.go index fb6f60d..5e22f8c 100644 --- a/internal/orchestrator/provider.go +++ b/internal/orchestrator/provider.go @@ -61,6 +61,15 @@ type Status struct { Instances []Instance `json:"instances"` } +type Usage struct { + CPUPercent float64 `json:"cpu_percent"` + MemoryPercent float64 `json:"memory_percent"` +} + +type MetricsProvider interface { + Metrics(context.Context, string) (Usage, error) +} + type Provider interface { ID() ProviderID Validate(context.Context, Workload) error diff --git a/internal/routing/k3s_ingress.go b/internal/routing/k3s_ingress.go new file mode 100644 index 0000000..76098e2 --- /dev/null +++ b/internal/routing/k3s_ingress.go @@ -0,0 +1,140 @@ +package routing + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net" + "os/exec" + "strconv" + "strings" + "sync" +) + +type KubernetesRunner interface { + Run(context.Context, []byte, ...string) ([]byte, error) +} + +type kubectlCommand struct{} + +func (kubectlCommand) Run(ctx context.Context, input []byte, args ...string) ([]byte, error) { + command := exec.CommandContext(ctx, "kubectl", args...) + command.Stdin = bytes.NewReader(input) + return command.CombinedOutput() +} + +type K3sIngressProvider struct { + runner KubernetesRunner + kubeconfig string + namespace string + mu sync.RWMutex + routes map[string]Route +} + +func NewK3sIngressProvider(kubeconfig, namespace string) Provider { + if strings.TrimSpace(namespace) == "" { + namespace = "default" + } + return &K3sIngressProvider{ + runner: kubectlCommand{}, kubeconfig: strings.TrimSpace(kubeconfig), + namespace: strings.TrimSpace(namespace), routes: make(map[string]Route), + } +} + +func NewK3sIngressProviderWithRunner(kubeconfig, namespace string, runner KubernetesRunner) Provider { + provider := NewK3sIngressProvider(kubeconfig, namespace).(*K3sIngressProvider) + provider.runner = runner + return provider +} + +func (p *K3sIngressProvider) ID() ProviderID { return ProviderTraefik } + +func (p *K3sIngressProvider) Validate(_ context.Context, route Route) error { + if err := validateRoute(route); err != nil { + return err + } + if strings.TrimSpace(route.Service) == "" { + return fmt.Errorf("Route service is required") + } + return nil +} + +func (p *K3sIngressProvider) Reconcile(ctx context.Context, route Route) error { + if err := p.Validate(ctx, route); err != nil { + return err + } + _, portValue, err := net.SplitHostPort(route.Backends[0].Address) + if err != nil { + return fmt.Errorf("resolve route service port: %w", err) + } + port, err := strconv.Atoi(portValue) + if err != nil { + return fmt.Errorf("resolve route service port: %w", err) + } + path := route.Path + if path == "" { + path = "/" + } + manifest := map[string]any{ + "apiVersion": "networking.k8s.io/v1", "kind": "Ingress", + "metadata": map[string]any{"name": route.ID, "labels": map[string]string{"flatrun.route": route.ID}}, + "spec": map[string]any{"rules": []any{map[string]any{ + "host": route.Domain, "http": map[string]any{"paths": []any{map[string]any{ + "path": path, "pathType": "Prefix", "backend": map[string]any{"service": map[string]any{ + "name": route.Service, "port": map[string]any{"number": port}, + }}, + }}}, + }}}, + } + content, err := json.Marshal(manifest) + if err != nil { + return err + } + if _, err := p.run(ctx, content, "apply", "-f", "-"); err != nil { + return fmt.Errorf("apply K3s ingress: %w", err) + } + p.mu.Lock() + p.routes[route.ID] = route + p.mu.Unlock() + return nil +} + +func (p *K3sIngressProvider) Drain(_ context.Context, routeID, backendID string) error { + p.mu.RLock() + _, exists := p.routes[routeID] + p.mu.RUnlock() + if !exists { + return fmt.Errorf("Route %q is not managed", routeID) + } + return nil +} + +func (p *K3sIngressProvider) Remove(ctx context.Context, routeID string) error { + if !safeRouteID.MatchString(routeID) { + return fmt.Errorf("Route ID is invalid") + } + if _, err := p.run(ctx, nil, "delete", "ingress", routeID, "--ignore-not-found=true"); err != nil { + return fmt.Errorf("remove K3s ingress: %w", err) + } + p.mu.Lock() + delete(p.routes, routeID) + p.mu.Unlock() + return nil +} + +func (p *K3sIngressProvider) run(ctx context.Context, input []byte, args ...string) ([]byte, error) { + base := make([]string, 0, len(args)+4) + if p.kubeconfig != "" { + base = append(base, "--kubeconfig", p.kubeconfig) + } + base = append(base, "--namespace", p.namespace) + output, err := p.runner.Run(ctx, input, append(base, args...)...) + if err != nil { + if message := strings.TrimSpace(string(output)); message != "" { + return nil, fmt.Errorf("%s", message) + } + return nil, err + } + return output, nil +} diff --git a/internal/routing/k3s_ingress_test.go b/internal/routing/k3s_ingress_test.go new file mode 100644 index 0000000..4d263c9 --- /dev/null +++ b/internal/routing/k3s_ingress_test.go @@ -0,0 +1,37 @@ +package routing + +import ( + "context" + "encoding/json" + "testing" +) + +type kubernetesRunnerStub struct { + input []byte + args []string +} + +func (r *kubernetesRunnerStub) Run(_ context.Context, input []byte, args ...string) ([]byte, error) { + r.input = append([]byte(nil), input...) + r.args = append([]string(nil), args...) + return nil, nil +} + +func TestK3sIngressRoutesThroughWorkloadService(t *testing.T) { + runner := &kubernetesRunnerStub{} + provider := NewK3sIngressProviderWithRunner("/etc/rancher/k3s.yaml", "apps", runner) + err := provider.Reconcile(context.Background(), Route{ + ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http", + Backends: []Backend{{ID: "pod-one", Address: "10.42.0.8:8080", Healthy: true}}, + }) + if err != nil { + t.Fatal(err) + } + var manifest map[string]any + if err := json.Unmarshal(runner.input, &manifest); err != nil { + t.Fatal(err) + } + if manifest["kind"] != "Ingress" || runner.args[0] != "--kubeconfig" || runner.args[4] != "apply" { + t.Fatalf("manifest = %#v, args = %#v", manifest, runner.args) + } +} From f3e5939bb404a92b97b904527f13d2881dfa20ec Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 18:05:31 +0100 Subject: [PATCH 40/46] docs(api): Publish Fleet scaling operations Schema-driven clients can discover workload compatibility, activation, policy, and capacity claim operations from the agent API description. --- internal/api/openapi.json | 435 +++++++++++++++++++++++++++++++++++++- 1 file changed, 429 insertions(+), 6 deletions(-) diff --git a/internal/api/openapi.json b/internal/api/openapi.json index 44f249d..9a4b94f 100644 --- a/internal/api/openapi.json +++ b/internal/api/openapi.json @@ -1545,6 +1545,26 @@ } } }, + "/api/cluster/capacity/claim": { + "post": { + "operationId": "post-cluster-capacity-claim", + "tags": [ + "cluster" + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/api.clusterCapacityClaimResponse" + } + } + } + } + } + } + }, "/api/cluster/deployments": { "get": { "operationId": "get-cluster-deployments", @@ -3905,6 +3925,95 @@ "x-permission": "deployments:write" } }, + "/api/deployments/{name}/autoscale/activate": { + "post": { + "operationId": "post-deployments-by-name-autoscale-activate", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/autoscale.Activation" + } + } + }, + "description": "Success" + } + }, + "tags": [ + "deployments" + ], + "x-permission": "deployments:write" + } + }, + "/api/deployments/{name}/autoscale/compatibility": { + "get": { + "operationId": "get-deployments-by-name-autoscale-compatibility", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "deployments" + ], + "x-permission": "deployments:read" + } + }, + "/api/deployments/{name}/autoscale/workload": { + "put": { + "operationId": "put-deployments-by-name-autoscale-workload", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/models.ScalingConfig" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "deployments" + ], + "x-permission": "deployments:write" + } + }, "/api/deployments/{name}/backup-config": { "get": { "operationId": "get-deployments-by-name-backup-config", @@ -6867,16 +6976,21 @@ "application/json": { "schema": { "properties": { + "target_id": { + "type": "string" + }, "url": { "type": "string" } }, "type": "object", "x-columns": [ - "url" + "url", + "target_id" ], "x-property-order": [ - "url" + "url", + "target_id" ] } } @@ -11442,6 +11556,49 @@ "allow_fleet_capacity" ] }, + "api.clusterCapacityClaimResponse": { + "type": "object", + "properties": { + "constraint": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "max_cpu": { + "type": "number" + }, + "max_memory": { + "type": "integer" + }, + "max_replicas": { + "type": "integer" + }, + "node": { + "$ref": "#/components/schemas/orchestrator.NodeIdentity" + }, + "reason": { + "type": "string" + } + }, + "x-property-order": [ + "enabled", + "reason", + "node", + "constraint", + "max_cpu", + "max_memory", + "max_replicas" + ], + "x-columns": [ + "enabled", + "reason", + "constraint", + "max_cpu", + "max_memory", + "max_replicas" + ] + }, "api.clusterProviderOption": { "type": "object", "properties": { @@ -11474,6 +11631,9 @@ "api.clusterProvidersResponse": { "type": "object", "properties": { + "k3s": { + "$ref": "#/components/schemas/config.K3sConfig" + }, "orchestrators": { "type": "array", "items": { @@ -11489,7 +11649,8 @@ }, "x-property-order": [ "orchestrators", - "routing" + "routing", + "k3s" ] }, "api.clusterSetupRequest": { @@ -11753,6 +11914,9 @@ "api.updateClusterProvidersRequest": { "type": "object", "properties": { + "k3s": { + "$ref": "#/components/schemas/config.K3sConfig" + }, "orchestrator": { "type": "string" }, @@ -11762,7 +11926,8 @@ }, "x-property-order": [ "orchestrator", - "routing" + "routing", + "k3s" ], "x-columns": [ "orchestrator", @@ -12000,9 +12165,27 @@ "count" ] }, + "autoscale.Activation": { + "type": "object", + "properties": { + "route": { + "$ref": "#/components/schemas/routing.Route" + }, + "workload": { + "$ref": "#/components/schemas/orchestrator.Status" + } + }, + "x-property-order": [ + "workload", + "route" + ] + }, "autoscale.State": { "type": "object", "properties": { + "active": { + "type": "boolean" + }, "high_windows": { "type": "integer" }, @@ -12012,17 +12195,38 @@ }, "low_windows": { "type": "integer" + }, + "provider": { + "type": "string" + }, + "replicas": { + "type": "integer" + }, + "route": { + "$ref": "#/components/schemas/routing.Route" + }, + "service": { + "type": "string" } }, "x-property-order": [ "high_windows", "low_windows", - "last_action" + "last_action", + "active", + "provider", + "service", + "replicas", + "route" ], "x-columns": [ "high_windows", "low_windows", - "last_action" + "last_action", + "active", + "provider", + "service", + "replicas" ] }, "backup.Backup": { @@ -12319,6 +12523,25 @@ "enabled" ] }, + "config.K3sConfig": { + "type": "object", + "properties": { + "kubeconfig": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "x-property-order": [ + "kubeconfig", + "namespace" + ], + "x-columns": [ + "kubeconfig", + "namespace" + ] + }, "dashboards.Dashboard": { "type": "object", "properties": { @@ -13643,6 +13866,48 @@ "auto_cert" ] }, + "models.ScalingConfig": { + "type": "object", + "properties": { + "service": { + "type": "string" + }, + "stateless": { + "type": "boolean" + }, + "storage": { + "$ref": "#/components/schemas/models.ScalingStorageConfig" + } + }, + "x-property-order": [ + "service", + "stateless", + "storage" + ], + "x-columns": [ + "service", + "stateless" + ] + }, + "models.ScalingStorageConfig": { + "type": "object", + "properties": { + "class": { + "type": "string" + }, + "mode": { + "type": "string" + } + }, + "x-property-order": [ + "mode", + "class" + ], + "x-columns": [ + "mode", + "class" + ] + }, "models.Service": { "type": "object", "properties": { @@ -13756,6 +14021,9 @@ "require_plan": { "type": "boolean" }, + "scaling": { + "$ref": "#/components/schemas/models.ScalingConfig" + }, "security": { "$ref": "#/components/schemas/models.DeploymentSecurityConfig" }, @@ -13784,6 +14052,7 @@ "quick_actions", "security", "backup", + "scaling", "protected_mode", "require_plan", "credential_id", @@ -13951,6 +14220,160 @@ "enabled" ] }, + "orchestrator.Instance": { + "type": "object", + "properties": { + "address": { + "type": "string" + }, + "healthy": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "node": { + "type": "string" + }, + "ready": { + "type": "boolean" + } + }, + "x-property-order": [ + "id", + "node", + "address", + "healthy", + "ready" + ], + "x-columns": [ + "id", + "node", + "address", + "healthy", + "ready" + ] + }, + "orchestrator.NodeIdentity": { + "type": "object", + "properties": { + "hostname": { + "type": "string" + }, + "id": { + "type": "string" + } + }, + "x-property-order": [ + "id", + "hostname" + ], + "x-columns": [ + "id", + "hostname" + ] + }, + "orchestrator.Status": { + "type": "object", + "properties": { + "available": { + "type": "integer" + }, + "desired": { + "type": "integer" + }, + "instances": { + "type": "array", + "items": { + "$ref": "#/components/schemas/orchestrator.Instance" + } + }, + "workload": { + "type": "string" + } + }, + "x-property-order": [ + "workload", + "desired", + "available", + "instances" + ], + "x-columns": [ + "workload", + "desired", + "available" + ] + }, + "routing.Backend": { + "type": "object", + "properties": { + "address": { + "type": "string" + }, + "healthy": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "weight": { + "type": "integer" + } + }, + "x-property-order": [ + "id", + "address", + "healthy", + "weight" + ], + "x-columns": [ + "id", + "address", + "healthy", + "weight" + ] + }, + "routing.Route": { + "type": "object", + "properties": { + "backends": { + "type": "array", + "items": { + "$ref": "#/components/schemas/routing.Backend" + } + }, + "domain": { + "type": "string" + }, + "id": { + "type": "string" + }, + "path": { + "type": "string" + }, + "protocol": { + "type": "string" + }, + "service": { + "type": "string" + } + }, + "x-property-order": [ + "id", + "service", + "domain", + "path", + "protocol", + "backends" + ], + "x-columns": [ + "id", + "service", + "domain", + "path", + "protocol" + ] + }, "scheduler.AgentTaskConfig": { "type": "object", "properties": { From 8555500aede5cd79ca9a1e8636afc32892bf8a44 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 18:13:43 +0100 Subject: [PATCH 41/46] feat(autoscale): Enforce borrowed resource grants Borrowed placement now requires declared workload limits and only uses peers whose CPU, memory, and replica grants can accommodate each replica. --- go.mod | 2 +- internal/api/autoscale_handlers.go | 15 +++++- internal/api/autoscale_handlers_test.go | 13 +++++ internal/autoscale/compatibility.go | 64 ++++++++++++++++++++++++ internal/autoscale/compatibility_test.go | 11 ++++ 5 files changed, 102 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c44bbc6..ac388ec 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/digitalocean/godo v1.171.0 github.com/distribution/reference v0.6.0 github.com/docker/docker v28.5.2+incompatible + github.com/docker/go-units v0.5.0 github.com/fsnotify/fsnotify v1.10.1 github.com/gin-contrib/cors v1.7.6 github.com/gin-gonic/gin v1.10.1 @@ -94,7 +95,6 @@ require ( github.com/docker/compose/v5 v5.1.0 // indirect github.com/docker/docker-credential-helpers v0.9.5 // indirect github.com/docker/go-connections v0.6.0 // indirect - github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/eclipse/paho.golang v0.23.0 // indirect diff --git a/internal/api/autoscale_handlers.go b/internal/api/autoscale_handlers.go index 2481841..1919f43 100644 --- a/internal/api/autoscale_handlers.go +++ b/internal/api/autoscale_handlers.go @@ -85,7 +85,7 @@ func (s *Server) defaultRunAutoscaleActivation(ctx context.Context, name string) return autoscale.Activation{}, fmt.Errorf("create Swarm provider: %w", err) } defer swarmProvider.Close() - workload.Placement, err = s.autoscalePlacement(ctx, swarmProvider, policy) + workload.Placement, err = s.autoscalePlacement(ctx, swarmProvider, policy, workload.Resources) if err != nil { return autoscale.Activation{}, err } @@ -120,7 +120,7 @@ func (s *Server) defaultRunAutoscaleActivation(ctx context.Context, name string) return activation, nil } -func (s *Server) autoscalePlacement(ctx context.Context, provider *orchestrator.SwarmProvider, policy autoscale.Policy) (orchestrator.Placement, error) { +func (s *Server) autoscalePlacement(ctx context.Context, provider *orchestrator.SwarmProvider, policy autoscale.Policy, resources orchestrator.Resources) (orchestrator.Placement, error) { identity, err := provider.EnsureLocalNodeLabel(ctx, "flatrun.capacity.local", "true") if err != nil { return orchestrator.Placement{}, err @@ -130,6 +130,9 @@ func (s *Server) autoscalePlacement(ctx context.Context, provider *orchestrator. if !policy.AllowFleetCapacity || manager == nil { return local, nil } + if resources.CPULimit == 0 || resources.MemoryLimit == 0 { + return orchestrator.Placement{}, fmt.Errorf("Fleet capacity requires CPU and memory limits in the Compose deployment resources") + } label := capacityNodeLabel(manager.ServerName()) if _, err := provider.EnsureLocalNodeLabel(ctx, label, "true"); err != nil { return orchestrator.Placement{}, err @@ -155,6 +158,9 @@ func (s *Server) autoscalePlacement(ctx context.Context, provider *orchestrator. if err := json.Unmarshal(result.Data, &claim); err != nil || !claim.Enabled || claim.Constraint != constraint { continue } + if !capacityClaimFits(claim, resources) { + continue + } allowed++ if claim.MaxReplicas > 0 && (maxReplicas == 0 || claim.MaxReplicas < maxReplicas) { maxReplicas = claim.MaxReplicas @@ -166,6 +172,11 @@ func (s *Server) autoscalePlacement(ctx context.Context, provider *orchestrator. return orchestrator.Placement{Constraints: []string{constraint}, MaxReplicasPerNode: uint64(maxReplicas)}, nil } +func capacityClaimFits(claim clusterCapacityClaimResponse, resources orchestrator.Resources) bool { + return (claim.MaxCPU == 0 || resources.CPULimit <= claim.MaxCPU) && + (claim.MaxMemory == 0 || resources.MemoryLimit <= claim.MaxMemory) +} + func autoscaleDomain(deployment *models.Deployment, workload orchestrator.Workload) (models.DomainConfig, error) { if deployment.Metadata == nil { return models.DomainConfig{}, fmt.Errorf("Scale-ready service must have an exposed domain") diff --git a/internal/api/autoscale_handlers_test.go b/internal/api/autoscale_handlers_test.go index ba07ec4..ef0983b 100644 --- a/internal/api/autoscale_handlers_test.go +++ b/internal/api/autoscale_handlers_test.go @@ -136,3 +136,16 @@ func TestDeploymentAutoscalePolicyThroughHTTP(t *testing.T) { t.Fatalf("activation returned %d for %q: %s", w.Code, activated, w.Body.String()) } } + +func TestCapacityClaimFitsWorkloadLimits(t *testing.T) { + resources := orchestrator.Resources{CPULimit: 2, MemoryLimit: 2 << 30} + if !capacityClaimFits(clusterCapacityClaimResponse{MaxCPU: 2, MaxMemory: 2 << 30}, resources) { + t.Fatal("matching capacity grant was rejected") + } + if capacityClaimFits(clusterCapacityClaimResponse{MaxCPU: 1, MaxMemory: 4 << 30}, resources) { + t.Fatal("CPU limit above the capacity grant was accepted") + } + if capacityClaimFits(clusterCapacityClaimResponse{MaxCPU: 4, MaxMemory: 1 << 30}, resources) { + t.Fatal("memory limit above the capacity grant was accepted") + } +} diff --git a/internal/autoscale/compatibility.go b/internal/autoscale/compatibility.go index 05bf276..4357d8e 100644 --- a/internal/autoscale/compatibility.go +++ b/internal/autoscale/compatibility.go @@ -3,8 +3,10 @@ package autoscale import ( "fmt" "sort" + "strconv" "strings" + "github.com/docker/go-units" "github.com/flatrun/agent/internal/orchestrator" "github.com/flatrun/agent/pkg/models" "gopkg.in/yaml.v3" @@ -39,6 +41,21 @@ type composeCompatibilityService struct { Entrypoint any `yaml:"entrypoint"` Command any `yaml:"command"` WorkingDir string `yaml:"working_dir"` + Deploy composeDeploy `yaml:"deploy"` +} + +type composeDeploy struct { + Resources composeResources `yaml:"resources"` +} + +type composeResources struct { + Limits composeResourceValues `yaml:"limits"` + Reservations composeResourceValues `yaml:"reservations"` +} + +type composeResourceValues struct { + CPUs any `yaml:"cpus"` + Memory any `yaml:"memory"` } func AssessCompatibility(deployment *models.Deployment, composeContent string) Compatibility { @@ -125,6 +142,11 @@ func BuildWorkload(deployment *models.Deployment, composeContent string, replica Environment: map[string]string{}, Entrypoint: stringList(service.Entrypoint), Command: stringList(service.Command), WorkingDir: service.WorkingDir, Labels: map[string]string{"flatrun.deployment": deployment.Name}, } + parsedResources, err := workloadResources(service.Deploy.Resources) + if err != nil { + return orchestrator.Workload{}, err + } + workload.Resources = parsedResources if strings.TrimSpace(proxyNetwork) != "" { workload.Networks = []string{proxyNetwork} } @@ -139,6 +161,48 @@ func BuildWorkload(deployment *models.Deployment, composeContent string, replica return workload, nil } +func workloadResources(resources composeResources) (orchestrator.Resources, error) { + cpuLimit, err := cpuValue(resources.Limits.CPUs) + if err != nil { + return orchestrator.Resources{}, fmt.Errorf("invalid CPU limit: %w", err) + } + cpuRequest, err := cpuValue(resources.Reservations.CPUs) + if err != nil { + return orchestrator.Resources{}, fmt.Errorf("invalid CPU reservation: %w", err) + } + memoryLimit, err := memoryValue(resources.Limits.Memory) + if err != nil { + return orchestrator.Resources{}, fmt.Errorf("invalid memory limit: %w", err) + } + memoryRequest, err := memoryValue(resources.Reservations.Memory) + if err != nil { + return orchestrator.Resources{}, fmt.Errorf("invalid memory reservation: %w", err) + } + return orchestrator.Resources{CPURequest: cpuRequest, CPULimit: cpuLimit, MemoryRequest: memoryRequest, MemoryLimit: memoryLimit}, nil +} + +func cpuValue(value any) (float64, error) { + if value == nil { + return 0, nil + } + parsed, err := strconv.ParseFloat(strings.TrimSpace(fmt.Sprint(value)), 64) + if err != nil || parsed < 0 { + return 0, fmt.Errorf("%q", value) + } + return parsed, nil +} + +func memoryValue(value any) (uint64, error) { + if value == nil { + return 0, nil + } + parsed, err := units.RAMInBytes(strings.TrimSpace(fmt.Sprint(value))) + if err != nil || parsed < 0 { + return 0, fmt.Errorf("%q", value) + } + return uint64(parsed), nil +} + func environmentMap(value any) map[string]string { result := map[string]string{} switch typed := value.(type) { diff --git a/internal/autoscale/compatibility_test.go b/internal/autoscale/compatibility_test.go index 08b15f2..4d1fa36 100644 --- a/internal/autoscale/compatibility_test.go +++ b/internal/autoscale/compatibility_test.go @@ -36,6 +36,14 @@ func TestBuildWorkloadCarriesPortableRuntimeInputs(t *testing.T) { entrypoint: ["/app/entrypoint"] command: ["serve", "--port", "8080"] working_dir: /app + deploy: + resources: + limits: + cpus: "1.5" + memory: 512M + reservations: + cpus: "0.5" + memory: 256M `, 2, "proxy") if err != nil { t.Fatal(err) @@ -49,4 +57,7 @@ func TestBuildWorkloadCarriesPortableRuntimeInputs(t *testing.T) { if len(workload.Networks) != 1 || workload.Networks[0] != "proxy" { t.Fatalf("workload networks = %#v", workload.Networks) } + if workload.Resources.CPULimit != 1.5 || workload.Resources.CPURequest != 0.5 || workload.Resources.MemoryLimit != 512<<20 || workload.Resources.MemoryRequest != 256<<20 { + t.Fatalf("workload resources = %#v", workload.Resources) + } } From 880a2bf71bde73efc8dacf7afea891bdf0950894 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 20:48:56 +0100 Subject: [PATCH 42/46] fix(autoscale): Close Fleet activation gaps Borrowed capacity now requires a matching runtime cluster and a live peer grant. Ready replicas enter routing after asynchronous startup, capacity credentials reach their dedicated claim operation, and activation state is durable before Compose stops. --- internal/api/autoscale_handlers.go | 83 ++++++++++++++++++++----- internal/api/autoscale_handlers_test.go | 13 +++- internal/api/autoscale_runtime.go | 21 +++++++ internal/api/autoscale_runtime_test.go | 9 +++ internal/api/cluster_handlers_test.go | 9 ++- internal/api/openapi.json | 9 ++- internal/api/server.go | 2 +- internal/autoscale/activator.go | 13 +++- internal/autoscale/activator_test.go | 21 +++++++ internal/autoscale/executor.go | 22 ++++++- internal/autoscale/executor_test.go | 22 +++++++ internal/autoscale/runner.go | 8 +-- internal/autoscale/runner_test.go | 24 +++++++ internal/orchestrator/swarm.go | 57 +++++++++++++---- internal/orchestrator/swarm_test.go | 2 +- 15 files changed, 273 insertions(+), 42 deletions(-) diff --git a/internal/api/autoscale_handlers.go b/internal/api/autoscale_handlers.go index 1919f43..f8ebcef 100644 --- a/internal/api/autoscale_handlers.go +++ b/internal/api/autoscale_handlers.go @@ -98,23 +98,33 @@ func (s *Server) defaultRunAutoscaleActivation(ctx context.Context, name string) stopper := autoscale.ServiceStopperFunc(func(deployment, service string) (string, error) { return s.manager.StopService(deployment, service) }) - activation, err := autoscale.NewActivator(orchestratorProvider, routeProvider, stopper).Activate(ctx, name, deployment.Metadata.Scaling.Service, workload, routing.Route{ - ID: name, Service: deployment.Metadata.Scaling.Service, Domain: domain.Domain, Path: domain.PathPrefix, Protocol: "http", - }) + previousState, err := s.autoscaleStore.State(name) if err != nil { return autoscale.Activation{}, err } - state, err := s.autoscaleStore.State(name) + persisted := false + activation, err := autoscale.NewActivator(orchestratorProvider, routeProvider, stopper).ActivateDurably(ctx, name, deployment.Metadata.Scaling.Service, workload, routing.Route{ + ID: name, Service: deployment.Metadata.Scaling.Service, Domain: domain.Domain, Path: domain.PathPrefix, Protocol: "http", + }, func(activation autoscale.Activation) error { + state := previousState + state.Active = true + state.Provider = orchestratorID + state.Service = deployment.Metadata.Scaling.Service + state.Replicas = activation.Workload.Desired + state.Route = activation.Route + state.LastAction = time.Now() + if err := s.autoscaleStore.SetState(name, state); err != nil { + return err + } + persisted = true + return nil + }) if err != nil { - return autoscale.Activation{}, err - } - state.Active = true - state.Provider = orchestratorID - state.Service = deployment.Metadata.Scaling.Service - state.Replicas = activation.Workload.Desired - state.Route = activation.Route - state.LastAction = time.Now() - if err := s.autoscaleStore.SetState(name, state); err != nil { + if persisted { + if restoreErr := s.autoscaleStore.SetState(name, previousState); restoreErr != nil { + return autoscale.Activation{}, fmt.Errorf("%v; restore autoscaling state: %w", err, restoreErr) + } + } return autoscale.Activation{}, err } return activation, nil @@ -148,6 +158,7 @@ func (s *Server) autoscalePlacement(ctx context.Context, provider *orchestrator. return data, nil }) allowed := 0 + incompatible := 0 maxReplicas := 0 constraint := "node.labels." + label + "==true" for _, result := range claims { @@ -158,7 +169,11 @@ func (s *Server) autoscalePlacement(ctx context.Context, provider *orchestrator. if err := json.Unmarshal(result.Data, &claim); err != nil || !claim.Enabled || claim.Constraint != constraint { continue } - if !capacityClaimFits(claim, resources) { + if claim.Node.ClusterID != identity.ClusterID { + incompatible++ + continue + } + if !capacityClaimFits(claim, resources, identity.ClusterID) { continue } allowed++ @@ -167,16 +182,52 @@ func (s *Server) autoscalePlacement(ctx context.Context, provider *orchestrator. } } if allowed == 0 { + if incompatible > 0 { + return orchestrator.Placement{}, fmt.Errorf("Permitted Fleet servers must join the same Docker Swarm before they can lend capacity") + } return local, nil } return orchestrator.Placement{Constraints: []string{constraint}, MaxReplicasPerNode: uint64(maxReplicas)}, nil } -func capacityClaimFits(claim clusterCapacityClaimResponse, resources orchestrator.Resources) bool { - return (claim.MaxCPU == 0 || resources.CPULimit <= claim.MaxCPU) && +func capacityClaimFits(claim clusterCapacityClaimResponse, resources orchestrator.Resources, clusterID string) bool { + return clusterID != "" && claim.Node.ClusterID == clusterID && + (claim.MaxCPU == 0 || resources.CPULimit <= claim.MaxCPU) && (claim.MaxMemory == 0 || resources.MemoryLimit <= claim.MaxMemory) } +func (s *Server) fleetCapacityAvailable(ctx context.Context, provider *orchestrator.SwarmProvider, resources orchestrator.Resources) (bool, error) { + manager := s.getClusterManager() + if manager == nil { + return false, nil + } + identity, err := provider.LocalNodeIdentity(ctx) + if err != nil { + return false, err + } + constraint := "node.labels." + capacityNodeLabel(manager.ServerName()) + "==true" + claims := manager.ForEachPeer(ctx, func(ctx context.Context, _ string, client *cluster.Client) ([]byte, error) { + data, status, err := client.Post(ctx, "/api/cluster/capacity/claim", nil) + if err != nil { + return nil, err + } + if status != http.StatusOK { + return nil, fmt.Errorf("capacity claim returned status %d", status) + } + return data, nil + }) + for _, result := range claims { + if result.Error != "" { + continue + } + var claim clusterCapacityClaimResponse + if err := json.Unmarshal(result.Data, &claim); err == nil && claim.Enabled && claim.Constraint == constraint && capacityClaimFits(claim, resources, identity.ClusterID) { + return true, nil + } + } + return false, nil +} + func autoscaleDomain(deployment *models.Deployment, workload orchestrator.Workload) (models.DomainConfig, error) { if deployment.Metadata == nil { return models.DomainConfig{}, fmt.Errorf("Scale-ready service must have an exposed domain") diff --git a/internal/api/autoscale_handlers_test.go b/internal/api/autoscale_handlers_test.go index ef0983b..97dd21f 100644 --- a/internal/api/autoscale_handlers_test.go +++ b/internal/api/autoscale_handlers_test.go @@ -139,13 +139,20 @@ func TestDeploymentAutoscalePolicyThroughHTTP(t *testing.T) { func TestCapacityClaimFitsWorkloadLimits(t *testing.T) { resources := orchestrator.Resources{CPULimit: 2, MemoryLimit: 2 << 30} - if !capacityClaimFits(clusterCapacityClaimResponse{MaxCPU: 2, MaxMemory: 2 << 30}, resources) { + claim := clusterCapacityClaimResponse{Node: orchestrator.NodeIdentity{ClusterID: "swarm-1"}, MaxCPU: 2, MaxMemory: 2 << 30} + if !capacityClaimFits(claim, resources, "swarm-1") { t.Fatal("matching capacity grant was rejected") } - if capacityClaimFits(clusterCapacityClaimResponse{MaxCPU: 1, MaxMemory: 4 << 30}, resources) { + claim.MaxCPU, claim.MaxMemory = 1, 4<<30 + if capacityClaimFits(claim, resources, "swarm-1") { t.Fatal("CPU limit above the capacity grant was accepted") } - if capacityClaimFits(clusterCapacityClaimResponse{MaxCPU: 4, MaxMemory: 1 << 30}, resources) { + claim.MaxCPU, claim.MaxMemory = 4, 1<<30 + if capacityClaimFits(claim, resources, "swarm-1") { t.Fatal("memory limit above the capacity grant was accepted") } + claim.MaxCPU, claim.MaxMemory = 4, 4<<30 + if capacityClaimFits(claim, resources, "swarm-2") { + t.Fatal("capacity from a different Swarm was accepted") + } } diff --git a/internal/api/autoscale_runtime.go b/internal/api/autoscale_runtime.go index 842564f..45fc068 100644 --- a/internal/api/autoscale_runtime.go +++ b/internal/api/autoscale_runtime.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math" + "strings" "github.com/flatrun/agent/internal/autoscale" "github.com/flatrun/agent/internal/capacity" @@ -81,6 +82,17 @@ func (f autoscaleRuntimeFactory) Build(ctx context.Context, deployment string, s observations = append(observations, autoscaleReplicaObservation{Stats: stat, Limits: *limits}) } input := autoscaleInput(observations, status, hostStats, f.server.config.Capacity) + placement, err := provider.Placement(ctx, deployment) + if err != nil { + return fail(fmt.Errorf("read managed workload placement: %w", err)) + } + if usesFleetPlacement(placement) { + available, err := f.server.fleetCapacityAvailable(ctx, provider, input.CurrentResources) + if err != nil { + return fail(fmt.Errorf("read permitted Fleet capacity: %w", err)) + } + input.FleetOffer = capacity.Offer{Enabled: available} + } routeProvider := routing.NewManagedNginxProvider(f.server.proxyOrchestrator.NginxManager(), f.server.manager) if err := routeProvider.Reconcile(ctx, state.Route); err != nil { return fail(fmt.Errorf("restore managed route state: %w", err)) @@ -90,6 +102,15 @@ func (f autoscaleRuntimeFactory) Build(ctx context.Context, deployment string, s }, nil } +func usesFleetPlacement(placement orchestrator.Placement) bool { + for _, constraint := range placement.Constraints { + if strings.HasPrefix(constraint, "node.labels.flatrun.capacity.") { + return true + } + } + return false +} + func autoscaleInput(observations []autoscaleReplicaObservation, status orchestrator.Status, hostStats *system.SystemStats, configPolicy config.CapacityConfig) autoscale.Input { var selected capacity.Container var cpuPercent float64 diff --git a/internal/api/autoscale_runtime_test.go b/internal/api/autoscale_runtime_test.go index dd0fa62..130ec4a 100644 --- a/internal/api/autoscale_runtime_test.go +++ b/internal/api/autoscale_runtime_test.go @@ -25,3 +25,12 @@ func TestAutoscaleInputUsesMostConstrainedReplica(t *testing.T) { t.Fatalf("diagnosis = %#v, input = %#v", input.Diagnosis, input) } } + +func TestUsesFleetPlacementOnlyForCapacityLabels(t *testing.T) { + if !usesFleetPlacement(orchestrator.Placement{Constraints: []string{"node.labels.flatrun.capacity.a1b2==true"}}) { + t.Fatal("Fleet capacity placement was not detected") + } + if usesFleetPlacement(orchestrator.Placement{Constraints: []string{"node.hostname==prod-1"}}) { + t.Fatal("local placement was treated as Fleet capacity") + } +} diff --git a/internal/api/cluster_handlers_test.go b/internal/api/cluster_handlers_test.go index 735fae7..a9266bd 100644 --- a/internal/api/cluster_handlers_test.go +++ b/internal/api/cluster_handlers_test.go @@ -121,6 +121,7 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool protected.GET("/test/users", authMiddleware.RequirePermission(auth.PermUsersWrite), func(c *gin.Context) { c.Status(http.StatusNoContent) }) + protected.POST("/cluster/capacity/claim", server.clusterCapacityClaim) clusterGroup := protected.Group("/cluster") clusterGroup.Use(authMiddleware.RequirePermission(auth.PermClusterRead)) { @@ -138,7 +139,6 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool clusterGroup.GET("/deployments", server.clusterAggregateDeployments) clusterGroup.GET("/stats", server.clusterAggregateStats) clusterGroup.GET("/capacity", server.clusterAggregateCapacity) - clusterGroup.POST("/capacity/claim", server.clusterCapacityClaim) } } @@ -236,6 +236,13 @@ func TestUpdateClusterPeerPolicyThroughHTTP(t *testing.T) { t.Fatalf("capacity offer unexpectedly granted general API permissions: %#v", key.Permissions) } } + claimReq := httptest.NewRequest(http.MethodPost, "/api/cluster/capacity/claim", nil) + claimReq.Header.Set("Authorization", "Bearer server-b-inbound-key") + claimWriter := httptest.NewRecorder() + env.router.ServeHTTP(claimWriter, claimReq) + if claimWriter.Code == http.StatusForbidden || claimWriter.Code == http.StatusUnauthorized { + t.Fatalf("capacity offer credential was rejected by middleware: %d: %s", claimWriter.Code, claimWriter.Body.String()) + } } func TestClusterPolicyAccessScopesDeployments(t *testing.T) { diff --git a/internal/api/openapi.json b/internal/api/openapi.json index 9a4b94f..71daddb 100644 --- a/internal/api/openapi.json +++ b/internal/api/openapi.json @@ -14257,6 +14257,9 @@ "orchestrator.NodeIdentity": { "type": "object", "properties": { + "cluster_id": { + "type": "string" + }, "hostname": { "type": "string" }, @@ -14266,11 +14269,13 @@ }, "x-property-order": [ "id", - "hostname" + "hostname", + "cluster_id" ], "x-columns": [ "id", - "hostname" + "hostname", + "cluster_id" ] }, "orchestrator.Status": { diff --git a/internal/api/server.go b/internal/api/server.go index 0039e71..c371d95 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -907,6 +907,7 @@ func (s *Server) setupRoutes() { _ = s.firewall.RegisterRoutes(protected) // Cluster endpoints + protected.POST("/cluster/capacity/claim", s.clusterCapacityClaim) clusterGroup := protected.Group("/cluster") clusterGroup.Use(s.authMiddleware.RequirePermission(auth.PermClusterRead)) { @@ -922,7 +923,6 @@ func (s *Server) setupRoutes() { clusterGroup.GET("/deployments", s.clusterAggregateDeployments) clusterGroup.GET("/stats", s.clusterAggregateStats) clusterGroup.GET("/capacity", s.clusterAggregateCapacity) - clusterGroup.POST("/capacity/claim", s.clusterCapacityClaim) clusterGroup.GET("/providers", s.clusterProviders) clusterGroup.PUT("/providers", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.updateClusterProviders) } diff --git a/internal/autoscale/activator.go b/internal/autoscale/activator.go index fa389da..ccb8bf3 100644 --- a/internal/autoscale/activator.go +++ b/internal/autoscale/activator.go @@ -37,6 +37,10 @@ func NewActivator(orchestratorProvider orchestrator.Provider, routingProvider ro } func (a *Activator) Activate(ctx context.Context, deployment, service string, workload orchestrator.Workload, route routing.Route) (Activation, error) { + return a.ActivateDurably(ctx, deployment, service, workload, route, nil) +} + +func (a *Activator) ActivateDurably(ctx context.Context, deployment, service string, workload orchestrator.Workload, route routing.Route, persist func(Activation) error) (Activation, error) { status, err := a.orchestrator.Apply(ctx, workload) if err != nil { return Activation{}, fmt.Errorf("create managed workload: %w", err) @@ -59,11 +63,18 @@ func (a *Activator) Activate(ctx context.Context, deployment, service string, wo rollback() return Activation{}, fmt.Errorf("publish managed route: %w", err) } + activation := Activation{Workload: status, Route: route} + if persist != nil { + if err := persist(activation); err != nil { + rollback() + return Activation{}, fmt.Errorf("save managed workload state: %w", err) + } + } if _, err := a.stopper.StopService(deployment, service); err != nil { rollback() return Activation{}, fmt.Errorf("stop Compose service after cutover: %w", err) } - return Activation{Workload: status, Route: route}, nil + return activation, nil } func (a *Activator) waitReady(ctx context.Context, workloadID string, status orchestrator.Status) (orchestrator.Status, error) { diff --git a/internal/autoscale/activator_test.go b/internal/autoscale/activator_test.go index a714d63..11ef8bd 100644 --- a/internal/autoscale/activator_test.go +++ b/internal/autoscale/activator_test.go @@ -50,3 +50,24 @@ func TestActivatorRollsBackWhenComposeCannotStop(t *testing.T) { t.Fatalf("rollback removed workload %q and route %q", provider.removed, router.removed) } } + +func TestActivatorPersistsBeforeStoppingCompose(t *testing.T) { + provider := &fakeOrchestrator{status: orchestrator.Status{Workload: "shop", Desired: 1, Available: 1, Instances: []orchestrator.Instance{{ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}}}} + router := &fakeRouter{} + stopper := &activationStopper{} + persisted := false + _, err := NewActivator(provider, router, stopper).ActivateDurably( + context.Background(), "shop", "web", orchestrator.Workload{ID: "shop", Image: "shop:1", Replicas: 1}, + routing.Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http"}, + func(activation Activation) error { + persisted = activation.Workload.Available == 1 && len(activation.Route.Backends) == 1 + return errors.New("database unavailable") + }, + ) + if err == nil || !persisted { + t.Fatalf("error = %v, persisted = %t", err, persisted) + } + if stopper.stopped || provider.removed != "shop" || router.removed != "shop" { + t.Fatalf("stopped = %t, workload = %q, route = %q", stopper.stopped, provider.removed, router.removed) + } +} diff --git a/internal/autoscale/executor.go b/internal/autoscale/executor.go index 4040ade..70d287e 100644 --- a/internal/autoscale/executor.go +++ b/internal/autoscale/executor.go @@ -3,6 +3,7 @@ package autoscale import ( "context" "fmt" + "slices" "github.com/flatrun/agent/internal/orchestrator" "github.com/flatrun/agent/internal/routing" @@ -27,7 +28,26 @@ func NewExecutor(orchestrator orchestrator.Provider, routing routing.Provider) * func (e *Executor) Execute(ctx context.Context, workloadID string, route routing.Route, decision Decision) (Execution, error) { result := Execution{Decision: decision} switch decision.Action { - case ActionNone, ActionNotify: + case ActionNone: + status, err := e.orchestrator.Status(ctx, workloadID) + result.Status = status + if err != nil || status.Available < status.Desired { + result.Pending = err == nil + return result, err + } + updated, err := routeWithReadyInstances(route, status) + if err != nil { + return result, err + } + if slices.Equal(updated.Backends, route.Backends) { + return result, nil + } + if err := e.routing.Reconcile(ctx, updated); err != nil { + return result, fmt.Errorf("publish ready route: %w", err) + } + result.Route = updated + return result, nil + case ActionNotify: status, err := e.orchestrator.Status(ctx, workloadID) result.Status = status return result, err diff --git a/internal/autoscale/executor_test.go b/internal/autoscale/executor_test.go index 647c96b..0094f27 100644 --- a/internal/autoscale/executor_test.go +++ b/internal/autoscale/executor_test.go @@ -102,6 +102,28 @@ func TestExecutorWaitsForNewReplicaBeforeRouting(t *testing.T) { } } +func TestExecutorPublishesReplicaThatBecameReady(t *testing.T) { + orchestratorProvider := &fakeOrchestrator{status: orchestrator.Status{ + Workload: "shop", Desired: 2, Available: 2, + Instances: []orchestrator.Instance{ + {ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}, + {ID: "two", Address: "10.0.0.2:8080", Healthy: true, Ready: true}, + }, + }} + router := &fakeRouter{} + execution, err := NewExecutor(orchestratorProvider, router).Execute( + context.Background(), "shop", + routing.Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http", Backends: []routing.Backend{{ID: "one", Address: "10.0.0.1:8080", Healthy: true, Weight: 1}}}, + Decision{Action: ActionNone}, + ) + if err != nil { + t.Fatalf("Execute failed: %v", err) + } + if execution.Pending || len(execution.Route.Backends) != 2 || len(router.reconciled.Backends) != 2 { + t.Fatalf("execution = %#v, route = %#v", execution, router.reconciled) + } +} + func TestExecutorDrainsBeforeRemovingReplica(t *testing.T) { orchestratorProvider := &fakeOrchestrator{status: orchestrator.Status{ Workload: "shop", Desired: 2, Available: 2, diff --git a/internal/autoscale/runner.go b/internal/autoscale/runner.go index 6e432bb..11eb57f 100644 --- a/internal/autoscale/runner.go +++ b/internal/autoscale/runner.go @@ -54,7 +54,7 @@ func (r *Runner) Reconcile(ctx context.Context, deployment string, input Input, return ReconcileResult{}, fmt.Errorf("save autoscaling state: %w", err) } result := ReconcileResult{State: nextState, Decision: decision} - if decision.Action == ActionNone { + if decision.Action == ActionNone && !nextState.Active { return result, nil } if decision.Action == ActionNotify { @@ -67,9 +67,6 @@ func (r *Runner) Reconcile(ctx context.Context, deployment string, input Input, r.publishFailure(deployment, err.Error(), events.SeverityCritical) return result, fmt.Errorf("execute autoscaling decision: %w", err) } - if execution.Pending { - return result, nil - } nextState.Replicas = execution.Status.Desired if execution.Route.ID != "" { nextState.Route = execution.Route @@ -78,6 +75,9 @@ func (r *Runner) Reconcile(ctx context.Context, deployment string, input Input, return result, fmt.Errorf("save autoscaling execution: %w", err) } result.State = nextState + if execution.Pending { + return result, nil + } return result, nil } diff --git a/internal/autoscale/runner_test.go b/internal/autoscale/runner_test.go index d265f6c..8ed69a8 100644 --- a/internal/autoscale/runner_test.go +++ b/internal/autoscale/runner_test.go @@ -93,3 +93,27 @@ func TestRunnerPersistsSuccessfulExecution(t *testing.T) { t.Fatalf("state = %#v", result.State) } } + +func TestRunnerPublishesPendingReplicaWhenItBecomesReady(t *testing.T) { + policy := DefaultPolicy() + policy.ScaleUpWindows = 1 + now := time.Now() + store := &runnerStore{policy: policy, state: State{Active: true, Replicas: 1}} + provider := &fakeOrchestrator{status: orchestrator.Status{ + Workload: "shop", Desired: 1, Available: 1, + Instances: []orchestrator.Instance{{ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}}, + }} + router := &fakeRouter{} + runner := NewRunner(store, NewExecutor(provider, router), nil, "prod-1") + route := routing.Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http", Backends: []routing.Backend{{ID: "one", Address: "10.0.0.1:8080", Healthy: true, Weight: 1}}} + result, err := runner.Reconcile(context.Background(), "shop", Input{Now: now, Replicas: 1, CPUPercent: 95}, route) + if err != nil || result.Execution == nil || !result.Execution.Pending || store.state.Replicas != 2 { + t.Fatalf("result = %#v, state = %#v, error = %v", result, store.state, err) + } + provider.status.Available = 2 + provider.status.Instances = append(provider.status.Instances, orchestrator.Instance{ID: "two", Address: "10.0.0.2:8080", Healthy: true, Ready: true}) + result, err = runner.Reconcile(context.Background(), "shop", Input{Now: now.Add(time.Second), Replicas: 2}, route) + if err != nil || result.Execution == nil || result.Execution.Pending || len(store.state.Route.Backends) != 2 { + t.Fatalf("result = %#v, state = %#v, error = %v", result, store.state, err) + } +} diff --git a/internal/orchestrator/swarm.go b/internal/orchestrator/swarm.go index d467fcb..15dd377 100644 --- a/internal/orchestrator/swarm.go +++ b/internal/orchestrator/swarm.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net" + "slices" "sort" "strconv" "strings" @@ -26,8 +27,9 @@ type swarmClient interface { } type NodeIdentity struct { - ID string `json:"id"` - Hostname string `json:"hostname"` + ID string `json:"id"` + Hostname string `json:"hostname"` + ClusterID string `json:"cluster_id"` } type SwarmProvider struct { @@ -61,16 +63,9 @@ func (p *SwarmProvider) Close() error { } func (p *SwarmProvider) EnsureLocalNodeLabel(ctx context.Context, key, value string) (NodeIdentity, error) { - info, err := p.client.Info(ctx, client.InfoOptions{}) + identity, inspected, err := p.localNode(ctx) if err != nil { - return NodeIdentity{}, fmt.Errorf("inspect Docker host: %w", err) - } - if info.Info.Swarm.NodeID == "" { - return NodeIdentity{}, fmt.Errorf("Docker host is not a Swarm node") - } - inspected, err := p.client.NodeInspect(ctx, info.Info.Swarm.NodeID, client.NodeInspectOptions{}) - if err != nil { - return NodeIdentity{}, fmt.Errorf("inspect local Swarm node: %w", err) + return NodeIdentity{}, err } if inspected.Node.Spec.Labels == nil { inspected.Node.Spec.Labels = make(map[string]string) @@ -81,7 +76,45 @@ func (p *SwarmProvider) EnsureLocalNodeLabel(ctx context.Context, key, value str return NodeIdentity{}, fmt.Errorf("label local Swarm node: %w", err) } } - return NodeIdentity{ID: inspected.Node.ID, Hostname: inspected.Node.Description.Hostname}, nil + return identity, nil +} + +func (p *SwarmProvider) LocalNodeIdentity(ctx context.Context) (NodeIdentity, error) { + identity, _, err := p.localNode(ctx) + return identity, err +} + +func (p *SwarmProvider) localNode(ctx context.Context) (NodeIdentity, client.NodeInspectResult, error) { + cluster, err := p.client.SwarmInspect(ctx, client.SwarmInspectOptions{}) + if err != nil { + return NodeIdentity{}, client.NodeInspectResult{}, fmt.Errorf("inspect Docker Swarm: %w", err) + } + info, err := p.client.Info(ctx, client.InfoOptions{}) + if err != nil { + return NodeIdentity{}, client.NodeInspectResult{}, fmt.Errorf("inspect Docker host: %w", err) + } + if info.Info.Swarm.NodeID == "" { + return NodeIdentity{}, client.NodeInspectResult{}, fmt.Errorf("Docker host is not a Swarm node") + } + inspected, err := p.client.NodeInspect(ctx, info.Info.Swarm.NodeID, client.NodeInspectOptions{}) + if err != nil { + return NodeIdentity{}, client.NodeInspectResult{}, fmt.Errorf("inspect local Swarm node: %w", err) + } + return NodeIdentity{ID: inspected.Node.ID, Hostname: inspected.Node.Description.Hostname, ClusterID: cluster.Swarm.ID}, inspected, nil +} + +func (p *SwarmProvider) Placement(ctx context.Context, id string) (Placement, error) { + service, err := p.client.ServiceInspect(ctx, id, client.ServiceInspectOptions{}) + if err != nil { + return Placement{}, err + } + if service.Service.Spec.TaskTemplate.Placement == nil { + return Placement{}, nil + } + return Placement{ + Constraints: slices.Clone(service.Service.Spec.TaskTemplate.Placement.Constraints), + MaxReplicasPerNode: service.Service.Spec.TaskTemplate.Placement.MaxReplicas, + }, nil } func (p *SwarmProvider) ID() ProviderID { diff --git a/internal/orchestrator/swarm_test.go b/internal/orchestrator/swarm_test.go index d8fcac8..3d3c60f 100644 --- a/internal/orchestrator/swarm_test.go +++ b/internal/orchestrator/swarm_test.go @@ -139,7 +139,7 @@ func TestSwarmProviderLabelsLocalNodeForCapacityGrant(t *testing.T) { if err != nil { t.Fatal(err) } - if identity.ID != "node-1" || identity.Hostname != "prod-1" || client.node.Spec.Labels["flatrun.capacity.origin"] != "true" { + if identity.ID != "node-1" || identity.Hostname != "prod-1" || identity.ClusterID != "swarm-1" || client.node.Spec.Labels["flatrun.capacity.origin"] != "true" { t.Fatalf("identity = %#v, node = %#v", identity, client.node) } } From dd7bb1ac4a8102bdc1ee79da7d0f90fdac856e88 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 21:03:30 +0100 Subject: [PATCH 43/46] fix(cluster): Resolve autoscale and Fleet review findings Autoscaling now observes only managed replicas, preserves routing identity, and applies safe provider updates without unnecessary reloads. Fleet health events, capacity claims, incident fallback, advertised URLs, and request cancellation now follow their intended trust and lifecycle boundaries. --- internal/api/autoscale_handlers.go | 2 +- internal/api/autoscale_handlers_test.go | 23 +++++++++++++++++ internal/api/autoscale_runtime.go | 10 ++------ internal/api/cluster_handlers.go | 3 +++ internal/api/cluster_handlers_test.go | 18 +++++--------- internal/api/openapi.json | 13 +++++----- internal/api/server.go | 7 ++---- internal/api/server_info_handlers.go | 12 --------- internal/api/server_info_handlers_test.go | 6 ++--- internal/auth/permissions.go | 7 +++--- internal/autoscale/executor.go | 30 ++++------------------- internal/autoscale/executor_test.go | 4 +-- internal/cluster/manager.go | 4 ++- internal/docker/stats.go | 30 +++++++++++++++++++---- internal/nginx/manager.go | 22 ++++++++++------- internal/nginx/manager_test.go | 30 ++++++++++++++++++++--- internal/notify/events_test.go | 13 ++++++++++ internal/notify/notify.go | 3 +++ internal/orchestrator/k3s.go | 2 +- internal/orchestrator/k3s_test.go | 18 ++++++++++++++ internal/orchestrator/swarm.go | 2 +- internal/routing/adapters.go | 3 ++- internal/routing/adapters_test.go | 16 ++++++++++++ internal/routing/k3s_ingress.go | 10 +++++--- internal/routing/managed_nginx.go | 22 ++++++++++++++--- internal/routing/managed_nginx_test.go | 22 ++++++++++++++--- 26 files changed, 224 insertions(+), 108 deletions(-) diff --git a/internal/api/autoscale_handlers.go b/internal/api/autoscale_handlers.go index f8ebcef..51e8124 100644 --- a/internal/api/autoscale_handlers.go +++ b/internal/api/autoscale_handlers.go @@ -33,7 +33,7 @@ func (s *Server) activateDeploymentAutoscale(c *gin.Context) { c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Autoscaling activation is unavailable"}) return } - activation, err := s.runAutoscaleActivation(c, c.Param("name")) + activation, err := s.runAutoscaleActivation(c.Request.Context(), c.Param("name")) if err != nil { c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()}) return diff --git a/internal/api/autoscale_handlers_test.go b/internal/api/autoscale_handlers_test.go index 97dd21f..e9354aa 100644 --- a/internal/api/autoscale_handlers_test.go +++ b/internal/api/autoscale_handlers_test.go @@ -137,6 +137,29 @@ func TestDeploymentAutoscalePolicyThroughHTTP(t *testing.T) { } } +func TestActivateDeploymentAutoscaleUsesRequestContext(t *testing.T) { + contextCanceled := false + server := &Server{runAutoscaleActivation: func(ctx context.Context, _ string) (autoscale.Activation, error) { + contextCanceled = ctx.Err() == context.Canceled + return autoscale.Activation{}, ctx.Err() + }} + router := gin.New() + router.POST("/deployments/:name/autoscale/activate", server.activateDeploymentAutoscale) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + req := httptest.NewRequest(http.MethodPost, "/deployments/shop/autoscale/activate", nil).WithContext(ctx) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if !contextCanceled { + t.Fatal("activation did not receive the canceled request context") + } + if w.Code != http.StatusUnprocessableEntity { + t.Fatalf("status = %d", w.Code) + } +} + func TestCapacityClaimFitsWorkloadLimits(t *testing.T) { resources := orchestrator.Resources{CPULimit: 2, MemoryLimit: 2 << 30} claim := clusterCapacityClaimResponse{Node: orchestrator.NodeIdentity{ClusterID: "swarm-1"}, MaxCPU: 2, MaxMemory: 2 << 30} diff --git a/internal/api/autoscale_runtime.go b/internal/api/autoscale_runtime.go index 45fc068..72f5078 100644 --- a/internal/api/autoscale_runtime.go +++ b/internal/api/autoscale_runtime.go @@ -35,10 +35,7 @@ func (f autoscaleRuntimeFactory) Build(ctx context.Context, deployment string, s if err != nil { return autoscale.RuntimeSession{}, err } - routeProvider := routing.NewK3sIngressProvider(f.server.config.Cluster.K3s.Kubeconfig, f.server.config.Cluster.K3s.Namespace) - if err := routeProvider.Reconcile(ctx, state.Route); err != nil { - return autoscale.RuntimeSession{}, fmt.Errorf("restore managed route state: %w", err) - } + routeProvider := routing.NewK3sIngressProvider(f.server.config.Cluster.K3s.Kubeconfig, f.server.config.Cluster.K3s.Namespace, state.Route) return autoscale.RuntimeSession{ Input: autoscale.Input{ Replicas: status.Desired, CPUPercent: usage.CPUPercent, MemoryPercent: usage.MemoryPercent, @@ -93,10 +90,7 @@ func (f autoscaleRuntimeFactory) Build(ctx context.Context, deployment string, s } input.FleetOffer = capacity.Offer{Enabled: available} } - routeProvider := routing.NewManagedNginxProvider(f.server.proxyOrchestrator.NginxManager(), f.server.manager) - if err := routeProvider.Reconcile(ctx, state.Route); err != nil { - return fail(fmt.Errorf("restore managed route state: %w", err)) - } + routeProvider := routing.NewManagedNginxProvider(f.server.proxyOrchestrator.NginxManager(), f.server.manager, state.Route) return autoscale.RuntimeSession{ Input: input, Executor: autoscale.NewExecutor(provider, routeProvider), Close: provider.Close, }, nil diff --git a/internal/api/cluster_handlers.go b/internal/api/cluster_handlers.go index 7897bdb..307b5aa 100644 --- a/internal/api/cluster_handlers.go +++ b/internal/api/cluster_handlers.go @@ -339,6 +339,7 @@ func (s *Server) clusterSetup(c *gin.Context) { requestTimeout = 10 * time.Second } mgr := cluster.NewManager(clusterDB, req.ServerName, healthInterval, requestTimeout, s.config.Auth.JWTSecret) + mgr.SetEventPublisher(s.notify) if err := mgr.Start(context.Background()); err != nil { _ = clusterDB.Close() s.config.Cluster = previous @@ -710,6 +711,8 @@ func clusterPolicyAccess(policy cluster.PeerPolicy) ([]string, auth.DeploymentAc unrestrictedDeployments = mergeClusterDeploymentAccess(deployments, grant.Deployments, auth.AccessLevelWrite, unrestrictedDeployments) case cluster.CapabilityCapacityRead: permissions[auth.PermSystemRead.String()] = true + case cluster.CapabilityCapacityOffer: + permissions[auth.PermClusterCapacityClaim.String()] = true case cluster.CapabilityRoutingManage: permissions[auth.PermInfrastructureRead.String()] = true permissions[auth.PermInfrastructureWrite.String()] = true diff --git a/internal/api/cluster_handlers_test.go b/internal/api/cluster_handlers_test.go index a9266bd..2260b61 100644 --- a/internal/api/cluster_handlers_test.go +++ b/internal/api/cluster_handlers_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "os" + "slices" "testing" "time" @@ -121,7 +122,7 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool protected.GET("/test/users", authMiddleware.RequirePermission(auth.PermUsersWrite), func(c *gin.Context) { c.Status(http.StatusNoContent) }) - protected.POST("/cluster/capacity/claim", server.clusterCapacityClaim) + protected.POST("/cluster/capacity/claim", authMiddleware.RequirePermission(auth.PermClusterCapacityClaim), server.clusterCapacityClaim) clusterGroup := protected.Group("/cluster") clusterGroup.Use(authMiddleware.RequirePermission(auth.PermClusterRead)) { @@ -232,8 +233,8 @@ func TestUpdateClusterPeerPolicyThroughHTTP(t *testing.T) { t.Fatalf("list API keys: %v", err) } for _, key := range keys { - if key.Name == "cluster-peer-server-b" && len(key.Permissions) != 0 { - t.Fatalf("capacity offer unexpectedly granted general API permissions: %#v", key.Permissions) + if key.Name == "cluster-peer-server-b" && !slices.Equal(key.Permissions, []string{auth.PermClusterCapacityClaim.String()}) { + t.Fatalf("capacity offer permissions = %#v", key.Permissions) } } claimReq := httptest.NewRequest(http.MethodPost, "/api/cluster/capacity/claim", nil) @@ -402,7 +403,7 @@ func TestClusterAPIKeyEnforcesPermissionsThroughHTTP(t *testing.T) { } } -func TestCapacityClaimDeniesUnpermittedPeerThroughHTTP(t *testing.T) { +func TestCapacityClaimRejectsUnpermittedPeerThroughHTTP(t *testing.T) { env := setupClusterTestServer(t, "server-a", true) defer env.cleanup() const rawKey = "peer-capacity-key-for-test" @@ -416,16 +417,9 @@ func TestCapacityClaimDeniesUnpermittedPeerThroughHTTP(t *testing.T) { req.Header.Set("Authorization", "Bearer "+rawKey) w := httptest.NewRecorder() env.router.ServeHTTP(w, req) - if w.Code != http.StatusOK { + if w.Code != http.StatusForbidden { t.Fatalf("status = %d: %s", w.Code, w.Body.String()) } - var response clusterCapacityClaimResponse - if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { - t.Fatal(err) - } - if response.Enabled || response.Reason == "" { - t.Fatalf("response = %#v", response) - } } func clusterLogin(t *testing.T, router *gin.Engine) string { diff --git a/internal/api/openapi.json b/internal/api/openapi.json index 71daddb..81cc260 100644 --- a/internal/api/openapi.json +++ b/internal/api/openapi.json @@ -1548,21 +1548,22 @@ "/api/cluster/capacity/claim": { "post": { "operationId": "post-cluster-capacity-claim", - "tags": [ - "cluster" - ], "responses": { "200": { - "description": "Success", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/api.clusterCapacityClaimResponse" } } - } + }, + "description": "Success" } - } + }, + "tags": [ + "cluster" + ], + "x-permission": "cluster:capacity:claim" } }, "/api/cluster/deployments": { diff --git a/internal/api/server.go b/internal/api/server.go index c371d95..881b25a 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -339,16 +339,13 @@ func New(cfg *config.Config, configPath string) *Server { requestTimeout = 10 * time.Second } clusterManager = cluster.NewManager(clusterDB, cfg.Cluster.ServerName, healthInterval, requestTimeout, cfg.Auth.JWTSecret) + clusterManager.SetEventPublisher(notifyService) if startErr := clusterManager.Start(context.Background()); startErr != nil { log.Printf("Warning: Failed to start cluster manager: %v", startErr) clusterManager = nil } } } - if clusterManager != nil { - clusterManager.SetEventPublisher(notifyService) - } - s := &Server{ config: cfg, configPath: configPath, @@ -907,7 +904,7 @@ func (s *Server) setupRoutes() { _ = s.firewall.RegisterRoutes(protected) // Cluster endpoints - protected.POST("/cluster/capacity/claim", s.clusterCapacityClaim) + protected.POST("/cluster/capacity/claim", s.authMiddleware.RequirePermission(auth.PermClusterCapacityClaim), s.clusterCapacityClaim) clusterGroup := protected.Group("/cluster") clusterGroup.Use(s.authMiddleware.RequirePermission(auth.PermClusterRead)) { diff --git a/internal/api/server_info_handlers.go b/internal/api/server_info_handlers.go index b863b91..7ad1fa2 100644 --- a/internal/api/server_info_handlers.go +++ b/internal/api/server_info_handlers.go @@ -32,25 +32,13 @@ func (s *Server) agentURL(c *gin.Context) string { if c.Request.TLS != nil { scheme = "https" } - if forwarded := firstForwardedValue(c.GetHeader("X-Forwarded-Proto")); forwarded != "" { - scheme = forwarded - } - host := c.Request.Host - if forwarded := firstForwardedValue(c.GetHeader("X-Forwarded-Host")); forwarded != "" { - host = forwarded - } if host == "" { return "" } return scheme + "://" + host } -func firstForwardedValue(value string) string { - value, _, _ = strings.Cut(value, ",") - return strings.TrimSpace(value) -} - func (s *Server) getNetworkHealth(c *gin.Context) { health, err := system.CheckNetworkHealth(c.Request.Context()) if err != nil { diff --git a/internal/api/server_info_handlers_test.go b/internal/api/server_info_handlers_test.go index 2dd9a53..7fa5b74 100644 --- a/internal/api/server_info_handlers_test.go +++ b/internal/api/server_info_handlers_test.go @@ -106,7 +106,7 @@ func TestGetServerInfo(t *testing.T) { } } -func TestGetServerInfoUsesForwardedAgentURL(t *testing.T) { +func TestGetServerInfoIgnoresUntrustedForwardedAgentURL(t *testing.T) { router, token, cleanup := setupServerInfoTestServer(t) defer cleanup() @@ -124,8 +124,8 @@ func TestGetServerInfoUsesForwardedAgentURL(t *testing.T) { if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatalf("Failed to parse response: %v", err) } - if resp.Server.AgentURL != "https://agent.example.com" { - t.Errorf("agent_url = %q, want %q", resp.Server.AgentURL, "https://agent.example.com") + if resp.Server.AgentURL != "http://example.com" { + t.Errorf("agent_url = %q, want %q", resp.Server.AgentURL, "http://example.com") } } diff --git a/internal/auth/permissions.go b/internal/auth/permissions.go index 83e6b9e..1a3c31b 100644 --- a/internal/auth/permissions.go +++ b/internal/auth/permissions.go @@ -78,8 +78,9 @@ const ( PermTrafficRead Permission = "traffic:read" PermTrafficWrite Permission = "traffic:write" - PermClusterRead Permission = "cluster:read" - PermClusterWrite Permission = "cluster:write" + PermClusterRead Permission = "cluster:read" + PermClusterWrite Permission = "cluster:write" + PermClusterCapacityClaim Permission = "cluster:capacity:claim" ) var adminPermissions = []Permission{ @@ -104,7 +105,7 @@ var adminPermissions = []Permission{ PermRegistriesRead, PermRegistriesWrite, PermRegistriesDelete, PermTemplatesRead, PermTemplatesWrite, PermTrafficRead, PermTrafficWrite, - PermClusterRead, PermClusterWrite, + PermClusterRead, PermClusterWrite, PermClusterCapacityClaim, } var operatorPermissions = []Permission{ diff --git a/internal/autoscale/executor.go b/internal/autoscale/executor.go index 70d287e..a64f199 100644 --- a/internal/autoscale/executor.go +++ b/internal/autoscale/executor.go @@ -75,19 +75,12 @@ func (e *Executor) Execute(ctx context.Context, workloadID string, route routing result.Route = updated return result, nil case ActionRemoveReplica: - status, err := e.orchestrator.Status(ctx, workloadID) - if err != nil { - return result, err - } - backendID := retiringBackend(route, status) - if backendID == "" { - return result, fmt.Errorf("No routable replica is available to drain") - } - if err := e.routing.Drain(ctx, route.ID, backendID); err != nil { - return result, fmt.Errorf("drain replica: %w", err) - } - status, err = e.orchestrator.Scale(ctx, workloadID, decision.Replicas) + status, err := e.orchestrator.Scale(ctx, workloadID, decision.Replicas) result.Status = status + if err == nil && status.Available != status.Desired { + result.Pending = true + return result, nil + } if err == nil && e.routing.ID() == routing.ProviderTraefik { result.Route = route return result, nil @@ -125,16 +118,3 @@ func routeWithReadyInstances(route routing.Route, status orchestrator.Status) (r route.Backends = backends return route, nil } - -func retiringBackend(route routing.Route, status orchestrator.Status) string { - ready := make(map[string]bool, len(status.Instances)) - for _, instance := range status.Instances { - ready[instance.ID] = instance.Ready - } - for index := len(route.Backends) - 1; index >= 0; index-- { - if ready[route.Backends[index].ID] { - return route.Backends[index].ID - } - } - return "" -} diff --git a/internal/autoscale/executor_test.go b/internal/autoscale/executor_test.go index 0094f27..bc4002e 100644 --- a/internal/autoscale/executor_test.go +++ b/internal/autoscale/executor_test.go @@ -124,7 +124,7 @@ func TestExecutorPublishesReplicaThatBecameReady(t *testing.T) { } } -func TestExecutorDrainsBeforeRemovingReplica(t *testing.T) { +func TestExecutorRoutesTheReplicaSetChosenByTheOrchestrator(t *testing.T) { orchestratorProvider := &fakeOrchestrator{status: orchestrator.Status{ Workload: "shop", Desired: 2, Available: 2, Instances: []orchestrator.Instance{ @@ -142,7 +142,7 @@ func TestExecutorDrainsBeforeRemovingReplica(t *testing.T) { if err != nil { t.Fatalf("Execute failed: %v", err) } - if router.drained != "two" || orchestratorProvider.scaledTo != 1 || len(router.reconciled.Backends) != 1 { + if router.drained != "" || orchestratorProvider.scaledTo != 1 || len(router.reconciled.Backends) != 1 { t.Fatalf("drained = %q, scaled = %d", router.drained, orchestratorProvider.scaledTo) } } diff --git a/internal/cluster/manager.go b/internal/cluster/manager.go index 2867517..ce4c98b 100644 --- a/internal/cluster/manager.go +++ b/internal/cluster/manager.go @@ -26,6 +26,7 @@ type PeerStatus struct { Online bool `json:"online"` LastSeen time.Time `json:"last_seen"` Error string `json:"error,omitempty"` + Probed bool `json:"-"` } type Result struct { @@ -142,9 +143,10 @@ func (m *Manager) checkAllPeers(ctx context.Context) { m.mu.Lock() st, exists := m.status[p.name] wasOnline := exists && st.Online - wasKnown := exists && (st.Online || st.Error != "" || !st.LastSeen.IsZero()) + wasKnown := exists && st.Probed publisher := m.publisher if exists { + st.Probed = true if err != nil { st.Online = false st.Error = err.Error() diff --git a/internal/docker/stats.go b/internal/docker/stats.go index cf1a602..36d69b4 100644 --- a/internal/docker/stats.go +++ b/internal/docker/stats.go @@ -113,15 +113,35 @@ func parseContainerDeploymentLabels(output string) map[string]string { } func GetManagedDeploymentStats(deployment string) ([]ContainerStats, error) { - stats, err := GetAllContainerStats() + if strings.TrimSpace(deployment) == "" { + return []ContainerStats{}, nil + } + ps := exec.Command("docker", "ps", "-q", "--filter", "label=flatrun.deployment="+deployment) + containerIDs, err := ps.Output() + if err != nil { + return nil, err + } + ids := strings.Fields(string(containerIDs)) + if len(ids) == 0 { + return []ContainerStats{}, nil + } + args := append([]string{"stats", "--no-stream", "--format", "{{json .}}"}, ids...) + output, err := exec.Command("docker", args...).Output() if err != nil { return nil, err } - result := make([]ContainerStats, 0, len(stats)) - for _, stat := range stats { - if stat.DeploymentName == deployment { - result = append(result, stat) + result := make([]ContainerStats, 0, len(ids)) + for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") { + if line == "" { + continue + } + var raw dockerStatsJSON + if err := json.Unmarshal([]byte(line), &raw); err != nil { + continue } + stat := parseStats(&raw) + stat.DeploymentName = deployment + result = append(result, *stat) } return result, nil } diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index c8dd6b8..6fad5b8 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -752,12 +752,10 @@ func assignUpstreams(servers []serverData, keepalive bool, backendOverrides map[ for li := range servers[si].Locations { loc := &servers[si].Locations[li] target := fmt.Sprintf("%s:%d", loc.Service, loc.ContainerPort) - overrides := backendOverrides[loc.RouteService] - if len(overrides) == 0 { - overrides = backendOverrides[loc.Service] - } + overrideKey := fmt.Sprintf("%s:%d", loc.RouteService, loc.ContainerPort) + overrides := backendOverrides[overrideKey] if len(overrides) > 0 { - target = "override:" + loc.Service + target = "override:" + overrideKey } name, ok := byTarget[target] if !ok { @@ -767,7 +765,7 @@ func assignUpstreams(servers []serverData, keepalive bool, backendOverrides map[ if len(overrides) > 0 { targets = append([]UpstreamBackend(nil), overrides...) } - upstreams = append(upstreams, upstreamData{Name: name, Targets: targets}) + upstreams = append(upstreams, upstreamData{Name: name, Targets: targets, Keepalive: keepalive}) } loc.Upstream = name } @@ -972,8 +970,9 @@ type multiRouteTemplateData struct { } type upstreamData struct { - Name string - Targets []UpstreamBackend + Name string + Targets []UpstreamBackend + Keepalive bool } type serverData struct { @@ -1158,15 +1157,20 @@ server { // container-restart rediscovery that the variable proxy_pass path provided, // while keepalive reuses connections across ordinary requests. const upstreamBlocks = `{{- range .Upstreams}} +{{- $pool := .}} upstream {{.Name}} { +{{- if .Keepalive}} zone {{.Name}} 64k; resolver 127.0.0.11 valid=30s ipv6=off; +{{- end}} {{- range .Targets}} - server {{.Address}}{{if .Weight}} weight={{.Weight}}{{end}}{{if not .Healthy}} down{{end}} resolve; + server {{.Address}}{{if .Weight}} weight={{.Weight}}{{end}}{{if not .Healthy}} down{{end}}{{if $pool.Keepalive}} resolve{{end}}; {{- end}} +{{- if .Keepalive}} keepalive 16; keepalive_timeout 60s; keepalive_requests 1000; +{{- end}} } {{end -}} ` diff --git a/internal/nginx/manager_test.go b/internal/nginx/manager_test.go index 99ecf89..8f43ebe 100644 --- a/internal/nginx/manager_test.go +++ b/internal/nginx/manager_test.go @@ -2419,6 +2419,25 @@ func TestAssignUpstreams(t *testing.T) { } } +func TestAssignUpstreamsKeepsOverridePortsDistinct(t *testing.T) { + servers := []serverData{{Locations: []locationData{ + {Service: "web", RouteService: "web", ContainerPort: 8080}, + {Service: "web", RouteService: "web", ContainerPort: 9090}, + }}} + overrides := map[string][]UpstreamBackend{ + "web:8080": {{Address: "10.0.0.8:8080", Healthy: true}}, + "web:9090": {{Address: "10.0.0.9:9090", Healthy: true}}, + } + + upstreams := assignUpstreams(servers, false, overrides) + if len(upstreams) != 2 || upstreams[0].Name == upstreams[1].Name { + t.Fatalf("upstreams = %#v", upstreams) + } + if upstreams[0].Targets[0].Address != "10.0.0.8:8080" || upstreams[1].Targets[0].Address != "10.0.0.9:9090" { + t.Fatalf("upstreams = %#v", upstreams) + } +} + func TestRenderMultiDomain_BackendOverridesPreserveDeploymentConfig(t *testing.T) { compose := "name: tenant-a\nservices:\n web:\n container_name: tenant-a-web\n" m, deployment := newManagerWithDeployment(t, []models.DomainConfig{ @@ -2427,7 +2446,7 @@ func TestRenderMultiDomain_BackendOverridesPreserveDeploymentConfig(t *testing.T deployment.Metadata.Security = &models.DeploymentSecurityConfig{Enabled: true, BlockedIPs: []string{"192.0.2.10"}} config, err := m.renderMultiDomainConfigWithBackends(deployment, false, map[string][]UpstreamBackend{ - "web": { + "web:8080": { {Address: "10.42.0.8:8080", Healthy: true, Weight: 2}, {Address: "10.42.1.9:8080", Healthy: false, Weight: 1}, }, @@ -2436,13 +2455,18 @@ func TestRenderMultiDomain_BackendOverridesPreserveDeploymentConfig(t *testing.T t.Fatal(err) } for _, expected := range []string{ - "listen 443 ssl", "deny 192.0.2.10", "server 10.42.0.8:8080 weight=2 resolve;", - "server 10.42.1.9:8080 weight=1 down resolve;", "set $upstream flatrun_tenant-a-web_8080;", + "listen 443 ssl", "deny 192.0.2.10", "server 10.42.0.8:8080 weight=2;", + "server 10.42.1.9:8080 weight=1 down;", "set $upstream flatrun_tenant-a-web_8080;", } { if !strings.Contains(config, expected) { t.Fatalf("missing %q in:\n%s", expected, config) } } + for _, unsupported := range []string{"keepalive 16", " resolve;", "zone flatrun_"} { + if strings.Contains(config, unsupported) { + t.Fatalf("unsupported upstream directive %q remains in:\n%s", unsupported, config) + } + } if strings.Contains(config, "server tenant-a-web:8080 resolve;") { t.Fatalf("Compose backend remains in overridden route:\n%s", config) } diff --git a/internal/notify/events_test.go b/internal/notify/events_test.go index f86b6a1..f38667e 100644 --- a/internal/notify/events_test.go +++ b/internal/notify/events_test.go @@ -1,6 +1,8 @@ package notify import ( + "os" + "path/filepath" "strings" "testing" "time" @@ -8,6 +10,17 @@ import ( "github.com/flatrun/agent/internal/events" ) +func TestIncidentsFallsBackWhenStoreCannotOpen(t *testing.T) { + base := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(base, []byte("file"), 0600); err != nil { + t.Fatal(err) + } + service := NewService(base) + if incidents := service.Incidents(); len(incidents) != 0 { + t.Fatalf("incidents = %#v", incidents) + } +} + func TestPublishSendsOneNotificationForCorrelatedFailure(t *testing.T) { service := NewService(t.TempDir()) if err := service.Save(Config{Targets: []Target{{ID: "email", Name: "Email", URL: "smtp://test", Enabled: true}}}); err != nil { diff --git a/internal/notify/notify.go b/internal/notify/notify.go index 304ca8a..f8af11d 100644 --- a/internal/notify/notify.go +++ b/internal/notify/notify.go @@ -148,6 +148,9 @@ func (s *Service) Publish(event events.Event) (events.IngestResult, error) { } func (s *Service) Incidents() []events.Incident { + if s.store == nil || s.storeErr != nil { + return s.events.List() + } incidents, err := s.store.ListIncidents() if err != nil { return s.events.List() diff --git a/internal/orchestrator/k3s.go b/internal/orchestrator/k3s.go index cd580d1..7979584 100644 --- a/internal/orchestrator/k3s.go +++ b/internal/orchestrator/k3s.go @@ -90,7 +90,7 @@ func (p *K3sProvider) Resize(ctx context.Context, id string, resources Resources "name": id, "resources": k3sResources(resources), }}}}}} encoded, _ := json.Marshal(patch) - if _, err := p.run(ctx, nil, "patch", "deployment", id, "--type", "merge", "-p", string(encoded)); err != nil { + if _, err := p.run(ctx, nil, "patch", "deployment", id, "--type", "strategic", "-p", string(encoded)); err != nil { return Status{}, fmt.Errorf("resize K3s workload: %w", err) } return p.Status(ctx, id) diff --git a/internal/orchestrator/k3s_test.go b/internal/orchestrator/k3s_test.go index 93aec92..103a995 100644 --- a/internal/orchestrator/k3s_test.go +++ b/internal/orchestrator/k3s_test.go @@ -86,6 +86,24 @@ func TestK3sStatusReturnsRoutableReadyPods(t *testing.T) { } } +func TestK3sResizeUsesStrategicMerge(t *testing.T) { + runner := &fakeKubectl{responses: [][]byte{ + nil, + []byte(`{"metadata":{"labels":{"flatrun.port":"8080"}},"spec":{"replicas":1},"status":{"availableReplicas":1}}`), + []byte(`{"items":[]}`), + }} + provider := NewK3sProvider("", "apps") + provider.runner = runner + + if _, err := provider.Resize(context.Background(), "shop", Resources{CPURequest: 0.25, CPULimit: 0.5, MemoryRequest: 128 << 20, MemoryLimit: 256 << 20}); err != nil { + t.Fatal(err) + } + wantPrefix := []string{"--namespace", "apps", "patch", "deployment", "shop", "--type", "strategic", "-p"} + if len(runner.calls) == 0 || !reflect.DeepEqual(runner.calls[0].args[:len(wantPrefix)], wantPrefix) { + t.Fatalf("patch args = %#v", runner.calls[0].args) + } +} + func TestK3sMetricsReturnsLimitUtilization(t *testing.T) { runner := &fakeKubectl{responses: [][]byte{ []byte(`{"spec":{"replicas":2,"template":{"spec":{"containers":[{"resources":{"limits":{"cpu":"500m","memory":"256Mi"}}}]}}}}`), diff --git a/internal/orchestrator/swarm.go b/internal/orchestrator/swarm.go index 15dd377..4266665 100644 --- a/internal/orchestrator/swarm.go +++ b/internal/orchestrator/swarm.go @@ -41,7 +41,7 @@ func NewSwarmProvider(client swarmClient) *SwarmProvider { } func NewSwarmProviderFromEnv() (*SwarmProvider, error) { - cli, err := client.New(client.FromEnv, client.WithAPIVersionNegotiation()) + cli, err := client.New(client.FromEnv) if err != nil { return nil, err } diff --git a/internal/routing/adapters.go b/internal/routing/adapters.go index 3396fab..f103c02 100644 --- a/internal/routing/adapters.go +++ b/internal/routing/adapters.go @@ -92,7 +92,7 @@ func (p *routeProvider) Reconcile(ctx context.Context, route Route) error { return err } p.mu.Lock() - p.routes[route.ID] = route + p.routes[route.ID] = cloneRoute(route) p.mu.Unlock() return nil } @@ -100,6 +100,7 @@ func (p *routeProvider) Reconcile(ctx context.Context, route Route) error { func (p *routeProvider) Drain(ctx context.Context, routeID, backendID string) error { p.mu.RLock() route, ok := p.routes[routeID] + route = cloneRoute(route) p.mu.RUnlock() if !ok { return fmt.Errorf("Route %q is not managed", routeID) diff --git a/internal/routing/adapters_test.go b/internal/routing/adapters_test.go index 88e1fdf..d30384c 100644 --- a/internal/routing/adapters_test.go +++ b/internal/routing/adapters_test.go @@ -57,6 +57,22 @@ func TestNginxProviderRendersWeightedUpstreamAndDrainsBackend(t *testing.T) { } } +func TestNginxProviderOwnsReconciledBackendSlice(t *testing.T) { + writer := &memoryConfigWriter{} + provider := NewNginxProvider(writer) + route := testRoute() + if err := provider.Reconcile(context.Background(), route); err != nil { + t.Fatal(err) + } + route.Backends[0].Healthy = false + if err := provider.Drain(context.Background(), "shop", "replica-a"); err != nil { + t.Fatal(err) + } + if strings.Contains(writer.content, "server 10.0.0.12:8080 weight=1 down;") { + t.Fatalf("caller mutation changed the stored route:\n%s", writer.content) + } +} + func TestTraefikProviderExcludesDrainedBackend(t *testing.T) { writer := &memoryConfigWriter{} provider := NewTraefikProvider(writer) diff --git a/internal/routing/k3s_ingress.go b/internal/routing/k3s_ingress.go index 76098e2..aab8eb2 100644 --- a/internal/routing/k3s_ingress.go +++ b/internal/routing/k3s_ingress.go @@ -32,14 +32,18 @@ type K3sIngressProvider struct { routes map[string]Route } -func NewK3sIngressProvider(kubeconfig, namespace string) Provider { +func NewK3sIngressProvider(kubeconfig, namespace string, restored ...Route) Provider { if strings.TrimSpace(namespace) == "" { namespace = "default" } - return &K3sIngressProvider{ + provider := &K3sIngressProvider{ runner: kubectlCommand{}, kubeconfig: strings.TrimSpace(kubeconfig), namespace: strings.TrimSpace(namespace), routes: make(map[string]Route), } + for _, route := range restored { + provider.routes[route.ID] = cloneRoute(route) + } + return provider } func NewK3sIngressProviderWithRunner(kubeconfig, namespace string, runner KubernetesRunner) Provider { @@ -95,7 +99,7 @@ func (p *K3sIngressProvider) Reconcile(ctx context.Context, route Route) error { return fmt.Errorf("apply K3s ingress: %w", err) } p.mu.Lock() - p.routes[route.ID] = route + p.routes[route.ID] = cloneRoute(route) p.mu.Unlock() return nil } diff --git a/internal/routing/managed_nginx.go b/internal/routing/managed_nginx.go index e14599b..9acc2cc 100644 --- a/internal/routing/managed_nginx.go +++ b/internal/routing/managed_nginx.go @@ -3,6 +3,7 @@ package routing import ( "context" "fmt" + "net" "sync" "github.com/flatrun/agent/internal/nginx" @@ -29,8 +30,12 @@ type managedNginxProvider struct { routes map[string]Route } -func NewManagedNginxProvider(manager ManagedNginx, deployments DeploymentSource) Provider { - return &managedNginxProvider{manager: manager, deployments: deployments, routes: make(map[string]Route)} +func NewManagedNginxProvider(manager ManagedNginx, deployments DeploymentSource, restored ...Route) Provider { + routes := make(map[string]Route, len(restored)) + for _, route := range restored { + routes[route.ID] = cloneRoute(route) + } + return &managedNginxProvider{manager: manager, deployments: deployments, routes: routes} } func (p *managedNginxProvider) ID() ProviderID { return ProviderNginx } @@ -57,7 +62,11 @@ func (p *managedNginxProvider) Reconcile(ctx context.Context, route Route) error for _, backend := range route.Backends { backends = append(backends, nginx.UpstreamBackend{Address: backend.Address, Healthy: backend.Healthy, Weight: backend.Weight}) } - content, err := p.manager.RenderVirtualHostWithBackends(deployment, map[string][]nginx.UpstreamBackend{route.Service: backends}) + _, port, err := net.SplitHostPort(route.Backends[0].Address) + if err != nil { + return fmt.Errorf("resolve managed backend port: %w", err) + } + content, err := p.manager.RenderVirtualHostWithBackends(deployment, map[string][]nginx.UpstreamBackend{route.Service + ":" + port: backends}) if err != nil { return fmt.Errorf("render deployment route: %w", err) } @@ -75,11 +84,16 @@ func (p *managedNginxProvider) Reconcile(ctx context.Context, route Route) error return fmt.Errorf("reload Nginx: %w", err) } p.mu.Lock() - p.routes[route.ID] = route + p.routes[route.ID] = cloneRoute(route) p.mu.Unlock() return nil } +func cloneRoute(route Route) Route { + route.Backends = append([]Backend(nil), route.Backends...) + return route +} + func (p *managedNginxProvider) Drain(ctx context.Context, routeID, backendID string) error { route, ok := p.route(routeID) if !ok { diff --git a/internal/routing/managed_nginx_test.go b/internal/routing/managed_nginx_test.go index 77effc0..e717391 100644 --- a/internal/routing/managed_nginx_test.go +++ b/internal/routing/managed_nginx_test.go @@ -47,14 +47,14 @@ func TestManagedNginxProviderReconcilesAndDrainsDeploymentBackends(t *testing.T) if err := provider.Reconcile(context.Background(), route); err != nil { t.Fatal(err) } - if manager.content != "preserved deployment config" || manager.reloads != 1 || len(manager.backends["web"]) != 2 { + if manager.content != "preserved deployment config" || manager.reloads != 1 || len(manager.backends["web:8080"]) != 2 { t.Fatalf("manager = %#v", manager) } if err := provider.Drain(context.Background(), "shop", "two"); err != nil { t.Fatal(err) } - if manager.backends["web"][1].Healthy || manager.reloads != 2 { - t.Fatalf("drained backends = %#v", manager.backends["web"]) + if manager.backends["web:8080"][1].Healthy || manager.reloads != 2 { + t.Fatalf("drained backends = %#v", manager.backends["web:8080"]) } } @@ -76,3 +76,19 @@ func TestManagedNginxProviderRestoresComposeRouteOnRemove(t *testing.T) { t.Fatalf("manager = %#v", manager) } } + +func TestManagedNginxProviderRestoresRouteWithoutReloading(t *testing.T) { + manager := &managedNginxRecorder{} + route := Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http", Backends: []Backend{{ID: "one", Address: "10.42.0.8:8080", Healthy: true}}} + provider := NewManagedNginxProvider(manager, deploymentSourceStub{deployment: &models.Deployment{Name: "shop"}}, route) + + if manager.reloads != 0 { + t.Fatalf("reloads = %d", manager.reloads) + } + if err := provider.Drain(context.Background(), "shop", "one"); err != nil { + t.Fatal(err) + } + if manager.reloads != 1 || manager.backends["web:8080"][0].Healthy { + t.Fatalf("manager = %#v", manager) + } +} From 502206218c3b97f52686aaf1c60df4f9842c4ea7 Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 21:19:53 +0100 Subject: [PATCH 44/46] fix(autoscale): Isolate activation and route state Managed activation now survives a disconnected request while retaining a bounded lifetime. Managed Nginx route copies can no longer mutate shared backend state outside synchronization. --- internal/api/autoscale_handlers.go | 4 +++- internal/api/autoscale_handlers_test.go | 14 +++++++------- internal/routing/managed_nginx.go | 2 +- internal/routing/managed_nginx_test.go | 15 +++++++++++++++ 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/internal/api/autoscale_handlers.go b/internal/api/autoscale_handlers.go index 51e8124..16966a3 100644 --- a/internal/api/autoscale_handlers.go +++ b/internal/api/autoscale_handlers.go @@ -33,7 +33,9 @@ func (s *Server) activateDeploymentAutoscale(c *gin.Context) { c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Autoscaling activation is unavailable"}) return } - activation, err := s.runAutoscaleActivation(c.Request.Context(), c.Param("name")) + ctx, cancel := context.WithTimeout(context.WithoutCancel(c.Request.Context()), 3*time.Minute) + defer cancel() + activation, err := s.runAutoscaleActivation(ctx, c.Param("name")) if err != nil { c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()}) return diff --git a/internal/api/autoscale_handlers_test.go b/internal/api/autoscale_handlers_test.go index e9354aa..7f84eff 100644 --- a/internal/api/autoscale_handlers_test.go +++ b/internal/api/autoscale_handlers_test.go @@ -137,11 +137,11 @@ func TestDeploymentAutoscalePolicyThroughHTTP(t *testing.T) { } } -func TestActivateDeploymentAutoscaleUsesRequestContext(t *testing.T) { - contextCanceled := false +func TestActivateDeploymentAutoscaleSurvivesRequestCancellation(t *testing.T) { + contextCanceled := true server := &Server{runAutoscaleActivation: func(ctx context.Context, _ string) (autoscale.Activation, error) { - contextCanceled = ctx.Err() == context.Canceled - return autoscale.Activation{}, ctx.Err() + contextCanceled = ctx.Err() != nil + return autoscale.Activation{}, nil }} router := gin.New() router.POST("/deployments/:name/autoscale/activate", server.activateDeploymentAutoscale) @@ -152,10 +152,10 @@ func TestActivateDeploymentAutoscaleUsesRequestContext(t *testing.T) { w := httptest.NewRecorder() router.ServeHTTP(w, req) - if !contextCanceled { - t.Fatal("activation did not receive the canceled request context") + if contextCanceled { + t.Fatal("request cancellation stopped activation") } - if w.Code != http.StatusUnprocessableEntity { + if w.Code != http.StatusOK { t.Fatalf("status = %d", w.Code) } } diff --git a/internal/routing/managed_nginx.go b/internal/routing/managed_nginx.go index 9acc2cc..0c5501b 100644 --- a/internal/routing/managed_nginx.go +++ b/internal/routing/managed_nginx.go @@ -143,5 +143,5 @@ func (p *managedNginxProvider) route(id string) (Route, bool) { p.mu.RLock() defer p.mu.RUnlock() route, ok := p.routes[id] - return route, ok + return cloneRoute(route), ok } diff --git a/internal/routing/managed_nginx_test.go b/internal/routing/managed_nginx_test.go index e717391..0bd4124 100644 --- a/internal/routing/managed_nginx_test.go +++ b/internal/routing/managed_nginx_test.go @@ -92,3 +92,18 @@ func TestManagedNginxProviderRestoresRouteWithoutReloading(t *testing.T) { t.Fatalf("manager = %#v", manager) } } + +func TestManagedNginxProviderOwnsReturnedBackendSlice(t *testing.T) { + route := Route{ID: "shop", Backends: []Backend{{ID: "one", Healthy: true}}} + provider := NewManagedNginxProvider(&managedNginxRecorder{}, deploymentSourceStub{}, route).(*managedNginxProvider) + + loaded, ok := provider.route("shop") + if !ok { + t.Fatal("route was not restored") + } + loaded.Backends[0].Healthy = false + stored, _ := provider.route("shop") + if !stored.Backends[0].Healthy { + t.Fatal("caller mutation changed the stored route") + } +} From 79a2622830f8e3835bfe52034909d72a262384dd Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 22:36:56 +0100 Subject: [PATCH 45/46] fix(cluster): Repair existing Fleet access on startup Stored peers receive missing default policies automatically. Existing credentials adopt those policies without requiring reconnection. --- internal/api/cluster_handlers.go | 30 +++++++- internal/api/cluster_handlers_test.go | 47 ++++++++++++ internal/api/server.go | 3 + internal/cluster/db.go | 15 ++++ internal/cluster/db_test.go | 100 ++++++++++++++++++++++++++ 5 files changed, 194 insertions(+), 1 deletion(-) diff --git a/internal/api/cluster_handlers.go b/internal/api/cluster_handlers.go index 307b5aa..1b52eb8 100644 --- a/internal/api/cluster_handlers.go +++ b/internal/api/cluster_handlers.go @@ -755,12 +755,40 @@ func (s *Server) applyClusterPeerPolicy(policy cluster.PeerPolicy) error { continue } permissions, deployments := clusterPolicyAccess(policy) - _, err := s.authManager.UpdateAPIKey(key.ID, key.Name, key.Description, key.Role, permissions, deployments, key.ExpiresAt) + _, err := s.authManager.UpdateAPIKey( + key.ID, + key.Name, + key.Description, + auth.Role(""), + permissions, + deployments, + key.ExpiresAt, + ) return err } return fmt.Errorf("Active peer credential not found") } +func (s *Server) reconcileClusterPeerPolicies() error { + if s.clusterManager == nil || s.authManager == nil { + return nil + } + peers, err := s.clusterManager.DB().ListPeers() + if err != nil { + return err + } + for _, peer := range peers { + policy, err := s.clusterManager.DB().GetPeerPolicy(peer.Name) + if err != nil { + return err + } + if err := s.applyClusterPeerPolicy(*policy); err != nil { + return err + } + } + return nil +} + func (s *Server) clusterRemovePeer(c *gin.Context) { mgr := s.getClusterManager() if mgr == nil { diff --git a/internal/api/cluster_handlers_test.go b/internal/api/cluster_handlers_test.go index 2260b61..c9f9ef1 100644 --- a/internal/api/cluster_handlers_test.go +++ b/internal/api/cluster_handlers_test.go @@ -246,6 +246,53 @@ func TestUpdateClusterPeerPolicyThroughHTTP(t *testing.T) { } } +func TestReconcileClusterPeerPoliciesScopesExistingCredentials(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + if err := env.server.clusterManager.AddPeer("server-b", "https://server-b.example.com", "peer-key"); err != nil { + t.Fatalf("AddPeer failed: %v", err) + } + user, err := env.server.authManager.GetUserByUsername("admin") + if err != nil { + t.Fatalf("get admin: %v", err) + } + _, err = env.server.authManager.CreateAPIKeyFromRaw( + "existing-peer-key", + user.ID, + "cluster-peer-server-b", + "Existing Fleet peer", + auth.RoleAdmin, + nil, + nil, + time.Time{}, + ) + if err != nil { + t.Fatalf("create existing peer credential: %v", err) + } + + if err := env.server.reconcileClusterPeerPolicies(); err != nil { + t.Fatalf("reconcile peer policies: %v", err) + } + keys, err := env.server.authManager.GetAllAPIKeys() + if err != nil { + t.Fatalf("list API keys: %v", err) + } + for _, key := range keys { + if key.Name != "cluster-peer-server-b" { + continue + } + if key.Role != "" { + t.Fatalf("peer role = %q", key.Role) + } + permissions, _ := clusterPolicyAccess(cluster.PeerPolicy{Grants: cluster.DefaultPeerGrants()}) + if !slices.Equal(key.Permissions, permissions) { + t.Fatalf("peer permissions = %#v, want %#v", key.Permissions, permissions) + } + return + } + t.Fatal("existing peer credential not found") +} + func TestClusterPolicyAccessScopesDeployments(t *testing.T) { permissions, deployments := clusterPolicyAccess(cluster.PeerPolicy{Grants: []cluster.Grant{ {Capability: cluster.CapabilityDeploymentsRead, Deployments: []string{"public-site", "docs"}}, diff --git a/internal/api/server.go b/internal/api/server.go index 881b25a..e226165 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -389,6 +389,9 @@ func New(cfg *config.Config, configPath string) *Server { s.runDeploymentAction = s.defaultRunDeploymentAction s.runServiceAction = s.defaultRunServiceAction s.runAutoscaleActivation = s.defaultRunAutoscaleActivation + if err := s.reconcileClusterPeerPolicies(); err != nil { + log.Printf("Warning: Failed to reconcile Fleet peer access: %v", err) + } if s.autoscaleStore != nil { autoscaleContext, cancelAutoscale := context.WithCancel(context.Background()) s.autoscaleCancel = cancelAutoscale diff --git a/internal/cluster/db.go b/internal/cluster/db.go index 9ed4f72..ee562a8 100644 --- a/internal/cluster/db.go +++ b/internal/cluster/db.go @@ -68,6 +68,10 @@ func NewDB(deploymentsPath string) (*DB, error) { conn.Close() return nil, err } + if err := db.repair(); err != nil { + conn.Close() + return nil, err + } return db, nil } @@ -96,6 +100,17 @@ func (db *DB) migrate() error { return err } +func (db *DB) repair() error { + grants, err := json.Marshal(DefaultPeerGrants()) + if err != nil { + return err + } + _, err = db.conn.Exec(` + INSERT OR IGNORE INTO peer_policies (peer_name, grants_json) + SELECT name, ? FROM peers`, grants) + return err +} + func HashToken(token string) string { h := sha256.Sum256([]byte(token)) return hex.EncodeToString(h[:]) diff --git a/internal/cluster/db_test.go b/internal/cluster/db_test.go index 0ed845d..0b2647f 100644 --- a/internal/cluster/db_test.go +++ b/internal/cluster/db_test.go @@ -53,6 +53,106 @@ func TestNewDBRecordsSchemaVersion(t *testing.T) { } } +func TestNewDBMigratesExistingPeersWithDefaultPolicy(t *testing.T) { + tmpDir := t.TempDir() + dbDir := filepath.Join(tmpDir, ".flatrun") + if err := os.MkdirAll(dbDir, 0755); err != nil { + t.Fatalf("create database directory: %v", err) + } + conn, err := sql.Open("sqlite", "file:"+filepath.Join(dbDir, "cluster.db")) + if err != nil { + t.Fatalf("open legacy database: %v", err) + } + _, err = conn.Exec(` + CREATE TABLE peers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT UNIQUE NOT NULL, + url TEXT NOT NULL, + api_key_hash TEXT NOT NULL, + api_key_encrypted TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + last_seen_at DATETIME + ); + CREATE TABLE invites ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + token_hash TEXT UNIQUE NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + created_by INTEGER NOT NULL, + accepted_peer TEXT, + expires_at DATETIME NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP + ); + INSERT INTO peers (name, url, api_key_hash, api_key_encrypted) + VALUES ('prod-2', 'https://prod-2.example.com', 'hash', 'encrypted'); + `) + if err != nil { + _ = conn.Close() + t.Fatalf("seed legacy database: %v", err) + } + if err := conn.Close(); err != nil { + t.Fatalf("close legacy database: %v", err) + } + + db, err := NewDB(tmpDir) + if err != nil { + t.Fatalf("migrate legacy database: %v", err) + } + defer db.Close() + + peer, err := db.GetPeer("prod-2") + if err != nil { + t.Fatalf("read migrated peer: %v", err) + } + if peer.URL != "https://prod-2.example.com" { + t.Fatalf("peer URL = %q", peer.URL) + } + policy, err := db.GetPeerPolicy("prod-2") + if err != nil { + t.Fatalf("read migrated peer policy: %v", err) + } + if len(policy.Grants) != len(DefaultPeerGrants()) { + t.Fatalf("policy grants = %#v", policy.Grants) + } +} + +func TestNewDBRepairsMissingPeerPolicy(t *testing.T) { + tmpDir := t.TempDir() + db, err := NewDB(tmpDir) + if err != nil { + t.Fatalf("open database: %v", err) + } + peer := &Peer{ + Name: "prod-2", + URL: "https://prod-2.example.com", + APIKeyHash: "hash", + APIKeyEncrypted: "encrypted", + Status: "active", + } + if _, err := db.CreatePeer(peer); err != nil { + t.Fatalf("create peer: %v", err) + } + if _, err := db.conn.Exec(`DELETE FROM peer_policies WHERE peer_name = ?`, peer.Name); err != nil { + t.Fatalf("remove peer policy: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close database: %v", err) + } + + db, err = NewDB(tmpDir) + if err != nil { + t.Fatalf("reopen database: %v", err) + } + defer db.Close() + policy, err := db.GetPeerPolicy(peer.Name) + if err != nil { + t.Fatalf("read repaired peer policy: %v", err) + } + if len(policy.Grants) != len(DefaultPeerGrants()) { + t.Fatalf("policy grants = %#v", policy.Grants) + } +} + func TestDBPath(t *testing.T) { tmpDir, err := os.MkdirTemp("", "cluster_test") if err != nil { From bc69954459886c67bd9f88a49e3d0bd3bac7964d Mon Sep 17 00:00:00 2001 From: nfebe Date: Sat, 22 Aug 2026 22:37:05 +0100 Subject: [PATCH 46/46] feat: Prepare 0.4.0-beta.5 The next beta includes Fleet management, managed scaling, and grouped notifications. Upgrade notes cover automatic repair for existing peer connections. --- CHANGELOG.md | 14 ++++++++++++++ VERSION | 2 +- internal/api/openapi.json | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c68e48..b871b5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [0.4.0-beta.5] - 2026-08-22 + +Fifth beta of the Albacore release, making connected servers manageable as one Fleet. + +### Added +- Guided Fleet setup, peer access policies, remote deployment inventories, and runtime provider selection +- Host and deployment capacity decisions with managed horizontal and vertical scaling +- Docker Swarm and k3s orchestration adapters with nginx and Traefik routing adapters +- Grouped incidents and configurable notification targets and delivery rules + +### Fixed +- Existing Fleet peers gain default access policies during startup repair without reconnecting +- Existing peer credentials are restricted to their configured Fleet policy during startup + ## [0.4.0-beta.4] - 2026-08-21 Fourth beta of the Albacore release, expanding operations, API discovery, and operator notifications. diff --git a/VERSION b/VERSION index bfa2c56..c7cc572 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.0-beta.4 +0.4.0-beta.5 diff --git a/internal/api/openapi.json b/internal/api/openapi.json index 81cc260..508edd4 100644 --- a/internal/api/openapi.json +++ b/internal/api/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "FlatRun Agent API", "description": "Generated from the agent's routes and the types its handlers bind and return.", - "version": "0.4.0-beta.4" + "version": "0.4.0-beta.5" }, "paths": { "/api/agent/update": {