diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c68e48..b871b5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [0.4.0-beta.5] - 2026-08-22 + +Fifth beta of the Albacore release, making connected servers manageable as one Fleet. + +### Added +- Guided Fleet setup, peer access policies, remote deployment inventories, and runtime provider selection +- Host and deployment capacity decisions with managed horizontal and vertical scaling +- Docker Swarm and k3s orchestration adapters with nginx and Traefik routing adapters +- Grouped incidents and configurable notification targets and delivery rules + +### Fixed +- Existing Fleet peers gain default access policies during startup repair without reconnecting +- Existing peer credentials are restricted to their configured Fleet policy during startup + ## [0.4.0-beta.4] - 2026-08-21 Fourth beta of the Albacore release, expanding operations, API discovery, and operator notifications. diff --git a/VERSION b/VERSION index bfa2c56..c7cc572 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.0-beta.4 +0.4.0-beta.5 diff --git a/go.mod b/go.mod index 18c3104..ac388ec 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/digitalocean/godo v1.171.0 github.com/distribution/reference v0.6.0 github.com/docker/docker v28.5.2+incompatible + github.com/docker/go-units v0.5.0 github.com/fsnotify/fsnotify v1.10.1 github.com/gin-contrib/cors v1.7.6 github.com/gin-gonic/gin v1.10.1 @@ -24,9 +25,11 @@ require ( github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 github.com/lib/pq v1.10.9 + github.com/moby/moby/api v1.53.0 github.com/moby/moby/client v0.2.2 github.com/modelcontextprotocol/go-sdk v1.6.1 github.com/nicholas-fedor/shoutrrr v0.16.1 + github.com/pressly/goose/v3 v3.26.0 github.com/robfig/cron/v3 v3.0.1 github.com/spf13/cobra v1.10.2 github.com/testcontainers/testcontainers-go v0.41.0 @@ -92,7 +95,6 @@ require ( github.com/docker/compose/v5 v5.1.0 // indirect github.com/docker/docker-credential-helpers v0.9.5 // indirect github.com/docker/go-connections v0.6.0 // indirect - github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/eclipse/paho.golang v0.23.0 // indirect @@ -137,12 +139,12 @@ require ( github.com/mattn/go-isatty v0.0.22 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mattn/go-shellwords v1.0.12 // indirect + github.com/mfridman/interpolate v0.0.2 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/moby/buildkit v0.27.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect github.com/moby/locker v1.0.1 // indirect - github.com/moby/moby/api v1.53.0 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect github.com/moby/sys/capability v0.4.0 // indirect @@ -170,6 +172,7 @@ require ( github.com/secure-systems-lab/go-securesystemslib v0.9.1 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/segmentio/encoding v0.5.4 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect github.com/shibumi/go-pathspec v1.3.0 // indirect github.com/shirou/gopsutil/v4 v4.26.2 // indirect github.com/sigstore/sigstore v1.10.4 // indirect @@ -200,6 +203,7 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0 // indirect go.opentelemetry.io/otel/trace v1.41.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect + go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect go.yaml.in/yaml/v4 v4.0.0-rc.4 // indirect golang.org/x/arch v0.18.0 // indirect diff --git a/go.sum b/go.sum index 4ac740c..a4711ad 100644 --- a/go.sum +++ b/go.sum @@ -343,6 +343,8 @@ github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6T github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= +github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/moby/buildkit v0.27.1 h1:qlIWpnZzqCkrYiGkctM1gBD/YZPOJTjtUdRBlI0oBOU= @@ -420,6 +422,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM= +github.com/pressly/goose/v3 v3.26.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -446,6 +450,8 @@ github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI= @@ -570,6 +576,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= diff --git a/internal/api/autoscale_handlers.go b/internal/api/autoscale_handlers.go new file mode 100644 index 0000000..16966a3 --- /dev/null +++ b/internal/api/autoscale_handlers.go @@ -0,0 +1,355 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/flatrun/agent/internal/autoscale" + "github.com/flatrun/agent/internal/cluster" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" + "github.com/flatrun/agent/pkg/models" + "github.com/gin-gonic/gin" +) + +type autoscalePolicyRequest struct { + Enabled bool `json:"enabled"` + MinReplicas int `json:"min_replicas"` + MaxReplicas int `json:"max_replicas"` + ScaleUpPercent float64 `json:"scale_up_percent"` + ScaleDownPercent float64 `json:"scale_down_percent"` + ScaleUpWindows int `json:"scale_up_windows"` + ScaleDownWindows int `json:"scale_down_windows"` + CooldownSeconds int64 `json:"cooldown_seconds"` + AllowFleetCapacity bool `json:"allow_fleet_capacity"` +} + +func (s *Server) activateDeploymentAutoscale(c *gin.Context) { + if s.runAutoscaleActivation == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Autoscaling activation is unavailable"}) + return + } + ctx, cancel := context.WithTimeout(context.WithoutCancel(c.Request.Context()), 3*time.Minute) + defer cancel() + activation, err := s.runAutoscaleActivation(ctx, c.Param("name")) + if err != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, activation) +} + +func (s *Server) defaultRunAutoscaleActivation(ctx context.Context, name string) (autoscale.Activation, error) { + if s.autoscaleStore == nil { + return autoscale.Activation{}, fmt.Errorf("Autoscaling storage is unavailable") + } + orchestratorID := orchestrator.ProviderID(s.config.Cluster.Orchestrator) + routingID := routing.ProviderID(s.config.Cluster.Routing) + if routingID == "" { + routingID = routing.ProviderNginx + } + if orchestratorID != orchestrator.ProviderSwarm && orchestratorID != orchestrator.ProviderK3s { + return autoscale.Activation{}, fmt.Errorf("Autoscaling activation requires Swarm or K3s") + } + if (orchestratorID == orchestrator.ProviderSwarm && routingID != routing.ProviderNginx) || + (orchestratorID == orchestrator.ProviderK3s && routingID != routing.ProviderTraefik) { + return autoscale.Activation{}, fmt.Errorf("The selected orchestrator and routing providers are incompatible") + } + deployment, err := s.manager.GetDeployment(name) + if err != nil { + return autoscale.Activation{}, fmt.Errorf("Deployment not found") + } + composeContent, _, err := s.manager.GetComposeFile(name) + if err != nil { + return autoscale.Activation{}, fmt.Errorf("Compose configuration is unavailable") + } + policy, err := s.autoscaleStore.Policy(name) + if err != nil { + return autoscale.Activation{}, err + } + workload, err := autoscale.BuildWorkload(deployment, composeContent, policy.MinReplicas, s.config.Infrastructure.DefaultProxyNetwork) + if err != nil { + return autoscale.Activation{}, err + } + domain, err := autoscaleDomain(deployment, workload) + if err != nil { + return autoscale.Activation{}, err + } + var orchestratorProvider orchestrator.Provider + var routeProvider routing.Provider + if orchestratorID == orchestrator.ProviderSwarm { + swarmProvider, err := orchestrator.NewSwarmProviderFromEnv() + if err != nil { + return autoscale.Activation{}, fmt.Errorf("create Swarm provider: %w", err) + } + defer swarmProvider.Close() + workload.Placement, err = s.autoscalePlacement(ctx, swarmProvider, policy, workload.Resources) + if err != nil { + return autoscale.Activation{}, err + } + orchestratorProvider = swarmProvider + routeProvider = routing.NewManagedNginxProvider(s.proxyOrchestrator.NginxManager(), s.manager) + } else { + orchestratorProvider = orchestrator.NewK3sProvider(s.config.Cluster.K3s.Kubeconfig, s.config.Cluster.K3s.Namespace) + routeProvider = routing.NewK3sIngressProvider(s.config.Cluster.K3s.Kubeconfig, s.config.Cluster.K3s.Namespace) + } + stopper := autoscale.ServiceStopperFunc(func(deployment, service string) (string, error) { + return s.manager.StopService(deployment, service) + }) + previousState, err := s.autoscaleStore.State(name) + if err != nil { + return autoscale.Activation{}, err + } + persisted := false + activation, err := autoscale.NewActivator(orchestratorProvider, routeProvider, stopper).ActivateDurably(ctx, name, deployment.Metadata.Scaling.Service, workload, routing.Route{ + ID: name, Service: deployment.Metadata.Scaling.Service, Domain: domain.Domain, Path: domain.PathPrefix, Protocol: "http", + }, func(activation autoscale.Activation) error { + state := previousState + state.Active = true + state.Provider = orchestratorID + state.Service = deployment.Metadata.Scaling.Service + state.Replicas = activation.Workload.Desired + state.Route = activation.Route + state.LastAction = time.Now() + if err := s.autoscaleStore.SetState(name, state); err != nil { + return err + } + persisted = true + return nil + }) + if err != nil { + if persisted { + if restoreErr := s.autoscaleStore.SetState(name, previousState); restoreErr != nil { + return autoscale.Activation{}, fmt.Errorf("%v; restore autoscaling state: %w", err, restoreErr) + } + } + return autoscale.Activation{}, err + } + return activation, nil +} + +func (s *Server) autoscalePlacement(ctx context.Context, provider *orchestrator.SwarmProvider, policy autoscale.Policy, resources orchestrator.Resources) (orchestrator.Placement, error) { + identity, err := provider.EnsureLocalNodeLabel(ctx, "flatrun.capacity.local", "true") + if err != nil { + return orchestrator.Placement{}, err + } + local := orchestrator.Placement{Constraints: []string{"node.hostname==" + identity.Hostname}} + manager := s.getClusterManager() + if !policy.AllowFleetCapacity || manager == nil { + return local, nil + } + if resources.CPULimit == 0 || resources.MemoryLimit == 0 { + return orchestrator.Placement{}, fmt.Errorf("Fleet capacity requires CPU and memory limits in the Compose deployment resources") + } + label := capacityNodeLabel(manager.ServerName()) + if _, err := provider.EnsureLocalNodeLabel(ctx, label, "true"); err != nil { + return orchestrator.Placement{}, err + } + claims := manager.ForEachPeer(ctx, func(ctx context.Context, _ string, client *cluster.Client) ([]byte, error) { + data, status, err := client.Post(ctx, "/api/cluster/capacity/claim", nil) + if err != nil { + return nil, err + } + if status != http.StatusOK { + return nil, fmt.Errorf("capacity claim returned status %d", status) + } + return data, nil + }) + allowed := 0 + incompatible := 0 + maxReplicas := 0 + constraint := "node.labels." + label + "==true" + for _, result := range claims { + if result.Error != "" { + continue + } + var claim clusterCapacityClaimResponse + if err := json.Unmarshal(result.Data, &claim); err != nil || !claim.Enabled || claim.Constraint != constraint { + continue + } + if claim.Node.ClusterID != identity.ClusterID { + incompatible++ + continue + } + if !capacityClaimFits(claim, resources, identity.ClusterID) { + continue + } + allowed++ + if claim.MaxReplicas > 0 && (maxReplicas == 0 || claim.MaxReplicas < maxReplicas) { + maxReplicas = claim.MaxReplicas + } + } + if allowed == 0 { + if incompatible > 0 { + return orchestrator.Placement{}, fmt.Errorf("Permitted Fleet servers must join the same Docker Swarm before they can lend capacity") + } + return local, nil + } + return orchestrator.Placement{Constraints: []string{constraint}, MaxReplicasPerNode: uint64(maxReplicas)}, nil +} + +func capacityClaimFits(claim clusterCapacityClaimResponse, resources orchestrator.Resources, clusterID string) bool { + return clusterID != "" && claim.Node.ClusterID == clusterID && + (claim.MaxCPU == 0 || resources.CPULimit <= claim.MaxCPU) && + (claim.MaxMemory == 0 || resources.MemoryLimit <= claim.MaxMemory) +} + +func (s *Server) fleetCapacityAvailable(ctx context.Context, provider *orchestrator.SwarmProvider, resources orchestrator.Resources) (bool, error) { + manager := s.getClusterManager() + if manager == nil { + return false, nil + } + identity, err := provider.LocalNodeIdentity(ctx) + if err != nil { + return false, err + } + constraint := "node.labels." + capacityNodeLabel(manager.ServerName()) + "==true" + claims := manager.ForEachPeer(ctx, func(ctx context.Context, _ string, client *cluster.Client) ([]byte, error) { + data, status, err := client.Post(ctx, "/api/cluster/capacity/claim", nil) + if err != nil { + return nil, err + } + if status != http.StatusOK { + return nil, fmt.Errorf("capacity claim returned status %d", status) + } + return data, nil + }) + for _, result := range claims { + if result.Error != "" { + continue + } + var claim clusterCapacityClaimResponse + if err := json.Unmarshal(result.Data, &claim); err == nil && claim.Enabled && claim.Constraint == constraint && capacityClaimFits(claim, resources, identity.ClusterID) { + return true, nil + } + } + return false, nil +} + +func autoscaleDomain(deployment *models.Deployment, workload orchestrator.Workload) (models.DomainConfig, error) { + if deployment.Metadata == nil { + return models.DomainConfig{}, fmt.Errorf("Scale-ready service must have an exposed domain") + } + service := deployment.Metadata.Scaling.Service + for _, domain := range deployment.Metadata.GetDomains() { + if domain.Service == service && strings.TrimSpace(domain.Domain) != "" && domain.ContainerPort == workload.Port { + return domain, nil + } + } + return models.DomainConfig{}, fmt.Errorf("Scale-ready service must have an exposed domain") +} + +type autoscalePolicyResponse struct { + autoscalePolicyRequest + State autoscale.State `json:"state"` +} + +func (s *Server) getDeploymentAutoscalePolicy(c *gin.Context) { + if s.autoscaleStore == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Autoscaling storage is unavailable"}) + return + } + name := c.Param("name") + if _, err := s.manager.GetDeployment(name); err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "Deployment not found"}) + return + } + policy, err := s.autoscaleStore.Policy(name) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + state, err := s.autoscaleStore.State(name) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, autoscalePolicyResponse{autoscalePolicyRequest: policyRequest(policy), State: state}) +} + +func (s *Server) getDeploymentAutoscaleCompatibility(c *gin.Context) { + s.writeAutoscaleCompatibility(c, c.Param("name")) +} + +func (s *Server) writeAutoscaleCompatibility(c *gin.Context, name string) { + deployment, err := s.manager.GetDeployment(name) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "Deployment not found"}) + return + } + composeContent, _, err := s.manager.GetComposeFile(name) + if err != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": "Compose configuration is unavailable"}) + return + } + c.JSON(http.StatusOK, autoscale.AssessCompatibility(deployment, composeContent)) +} + +func (s *Server) updateDeploymentAutoscaleWorkload(c *gin.Context) { + name := c.Param("name") + deployment, err := s.manager.GetDeployment(name) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "Deployment not found"}) + return + } + var scaling models.ScalingConfig + if err := c.ShouldBindJSON(&scaling); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + if deployment.Metadata == nil { + deployment.Metadata = &models.ServiceMetadata{Name: name} + } + deployment.Metadata.Scaling = &scaling + if err := s.manager.SaveMetadata(name, deployment.Metadata); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + s.writeAutoscaleCompatibility(c, name) +} + +func (s *Server) updateDeploymentAutoscalePolicy(c *gin.Context) { + if s.autoscaleStore == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Autoscaling storage is unavailable"}) + return + } + name := c.Param("name") + if _, err := s.manager.GetDeployment(name); err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "Deployment not found"}) + return + } + var req autoscalePolicyRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + policy := autoscale.Policy{ + Enabled: req.Enabled, MinReplicas: req.MinReplicas, MaxReplicas: req.MaxReplicas, + ScaleUpPercent: req.ScaleUpPercent, ScaleDownPercent: req.ScaleDownPercent, + ScaleUpWindows: req.ScaleUpWindows, ScaleDownWindows: req.ScaleDownWindows, + Cooldown: time.Duration(req.CooldownSeconds) * time.Second, AllowFleetCapacity: req.AllowFleetCapacity, + } + if err := s.autoscaleStore.SetPolicy(name, policy); err != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()}) + return + } + state, err := s.autoscaleStore.State(name) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, autoscalePolicyResponse{autoscalePolicyRequest: req, State: state}) +} + +func policyRequest(policy autoscale.Policy) autoscalePolicyRequest { + return autoscalePolicyRequest{ + Enabled: policy.Enabled, MinReplicas: policy.MinReplicas, MaxReplicas: policy.MaxReplicas, + ScaleUpPercent: policy.ScaleUpPercent, ScaleDownPercent: policy.ScaleDownPercent, + ScaleUpWindows: policy.ScaleUpWindows, ScaleDownWindows: policy.ScaleDownWindows, + CooldownSeconds: int64(policy.Cooldown / time.Second), AllowFleetCapacity: policy.AllowFleetCapacity, + } +} diff --git a/internal/api/autoscale_handlers_test.go b/internal/api/autoscale_handlers_test.go new file mode 100644 index 0000000..7f84eff --- /dev/null +++ b/internal/api/autoscale_handlers_test.go @@ -0,0 +1,181 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/flatrun/agent/internal/auth" + "github.com/flatrun/agent/internal/autoscale" + "github.com/flatrun/agent/internal/docker" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" + "github.com/flatrun/agent/pkg/config" + "github.com/gin-gonic/gin" +) + +func TestDeploymentAutoscalePolicyThroughHTTP(t *testing.T) { + dir := t.TempDir() + deploymentDir := filepath.Join(dir, "shop") + if err := os.MkdirAll(deploymentDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(deploymentDir, "compose.yml"), []byte("services:\n app:\n image: nginx:alpine\n"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(deploymentDir, "service.yml"), []byte("name: shop\nscaling:\n service: app\n stateless: true\n"), 0644); err != nil { + t.Fatal(err) + } + cfg := &config.Config{DeploymentsPath: dir, Auth: config.AuthConfig{Enabled: true, JWTSecret: "autoscale-test-secret"}} + t.Setenv("FLATRUN_ADMIN_PASSWORD", "testadminpass") + authManager, err := auth.NewManager(dir, &cfg.Auth, true) + if err != nil { + t.Fatal(err) + } + defer authManager.Close() + store, err := autoscale.NewStore(dir) + if err != nil { + t.Fatal(err) + } + defer store.Close() + activated := "" + server := &Server{ + config: cfg, manager: docker.NewManager(dir), authManager: authManager, autoscaleStore: store, + runAutoscaleActivation: func(_ context.Context, name string) (autoscale.Activation, error) { + activated = name + return autoscale.Activation{ + Workload: orchestrator.Status{Workload: name, Desired: 2, Available: 2}, + Route: routing.Route{ID: name, Domain: "shop.example.com", Protocol: "http"}, + }, nil + }, + } + middleware := auth.NewMiddlewareWithManager(&cfg.Auth, authManager) + router := gin.New() + router.POST("/api/auth/login", middleware.Login) + protected := router.Group("/api", middleware.RequireAuth()) + protected.GET("/deployments/:name/autoscale", middleware.RequirePermission(auth.PermDeploymentsRead), middleware.RequireDeploymentAccess(auth.AccessLevelRead), server.getDeploymentAutoscalePolicy) + protected.GET("/deployments/:name/autoscale/compatibility", middleware.RequirePermission(auth.PermDeploymentsRead), middleware.RequireDeploymentAccess(auth.AccessLevelRead), server.getDeploymentAutoscaleCompatibility) + protected.PUT("/deployments/:name/autoscale/workload", middleware.RequirePermission(auth.PermDeploymentsWrite), middleware.RequireDeploymentAccess(auth.AccessLevelWrite), server.updateDeploymentAutoscaleWorkload) + protected.PUT("/deployments/:name/autoscale", middleware.RequirePermission(auth.PermDeploymentsWrite), middleware.RequireDeploymentAccess(auth.AccessLevelWrite), server.updateDeploymentAutoscalePolicy) + protected.POST("/deployments/:name/autoscale/activate", middleware.RequirePermission(auth.PermDeploymentsWrite), middleware.RequireDeploymentAccess(auth.AccessLevelWrite), server.activateDeploymentAutoscale) + token := loginAndGetToken(t, router, "admin", "testadminpass") + + payload := autoscalePolicyRequest{ + Enabled: true, MinReplicas: 2, MaxReplicas: 6, ScaleUpPercent: 75, ScaleDownPercent: 25, + ScaleUpWindows: 3, ScaleDownWindows: 8, CooldownSeconds: 120, AllowFleetCapacity: true, + } + raw, err := json.Marshal(payload) + if err != nil { + t.Fatal(err) + } + req := httptest.NewRequest(http.MethodPut, "/api/deployments/shop/autoscale", bytes.NewReader(raw)) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + + req = httptest.NewRequest(http.MethodGet, "/api/deployments/shop/autoscale", nil) + req.Header.Set("Authorization", "Bearer "+token) + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + var response autoscalePolicyResponse + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.MaxReplicas != 6 || response.CooldownSeconds != 120 || !response.AllowFleetCapacity { + t.Fatalf("unexpected response: %+v", response) + } + + req = httptest.NewRequest(http.MethodGet, "/api/deployments/shop/autoscale/compatibility", nil) + req.Header.Set("Authorization", "Bearer "+token) + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + var compatibility autoscale.Compatibility + if err := json.Unmarshal(w.Body.Bytes(), &compatibility); err != nil { + t.Fatal(err) + } + if !compatibility.Compatible || compatibility.Service != "app" || compatibility.Image != "nginx:alpine" { + t.Fatalf("compatibility = %#v", compatibility) + } + + workload := bytes.NewBufferString(`{"service":"app","stateless":true,"storage":{"mode":"none"}}`) + req = httptest.NewRequest(http.MethodPut, "/api/deployments/shop/autoscale/workload", workload) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + if err := json.Unmarshal(w.Body.Bytes(), &compatibility); err != nil { + t.Fatal(err) + } + if !compatibility.Compatible || len(compatibility.Services) != 1 || compatibility.Services[0] != "app" { + t.Fatalf("compatibility = %#v", compatibility) + } + + req = httptest.NewRequest(http.MethodPost, "/api/deployments/shop/autoscale/activate", nil) + req.Header.Set("Authorization", "Bearer "+token) + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK || activated != "shop" { + t.Fatalf("activation returned %d for %q: %s", w.Code, activated, w.Body.String()) + } +} + +func TestActivateDeploymentAutoscaleSurvivesRequestCancellation(t *testing.T) { + contextCanceled := true + server := &Server{runAutoscaleActivation: func(ctx context.Context, _ string) (autoscale.Activation, error) { + contextCanceled = ctx.Err() != nil + return autoscale.Activation{}, nil + }} + router := gin.New() + router.POST("/deployments/:name/autoscale/activate", server.activateDeploymentAutoscale) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + req := httptest.NewRequest(http.MethodPost, "/deployments/shop/autoscale/activate", nil).WithContext(ctx) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + if contextCanceled { + t.Fatal("request cancellation stopped activation") + } + if w.Code != http.StatusOK { + t.Fatalf("status = %d", w.Code) + } +} + +func TestCapacityClaimFitsWorkloadLimits(t *testing.T) { + resources := orchestrator.Resources{CPULimit: 2, MemoryLimit: 2 << 30} + claim := clusterCapacityClaimResponse{Node: orchestrator.NodeIdentity{ClusterID: "swarm-1"}, MaxCPU: 2, MaxMemory: 2 << 30} + if !capacityClaimFits(claim, resources, "swarm-1") { + t.Fatal("matching capacity grant was rejected") + } + claim.MaxCPU, claim.MaxMemory = 1, 4<<30 + if capacityClaimFits(claim, resources, "swarm-1") { + t.Fatal("CPU limit above the capacity grant was accepted") + } + claim.MaxCPU, claim.MaxMemory = 4, 1<<30 + if capacityClaimFits(claim, resources, "swarm-1") { + t.Fatal("memory limit above the capacity grant was accepted") + } + claim.MaxCPU, claim.MaxMemory = 4, 4<<30 + if capacityClaimFits(claim, resources, "swarm-2") { + t.Fatal("capacity from a different Swarm was accepted") + } +} diff --git a/internal/api/autoscale_runtime.go b/internal/api/autoscale_runtime.go new file mode 100644 index 0000000..72f5078 --- /dev/null +++ b/internal/api/autoscale_runtime.go @@ -0,0 +1,147 @@ +package api + +import ( + "context" + "fmt" + "math" + "strings" + + "github.com/flatrun/agent/internal/autoscale" + "github.com/flatrun/agent/internal/capacity" + "github.com/flatrun/agent/internal/docker" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" + "github.com/flatrun/agent/internal/system" + "github.com/flatrun/agent/pkg/config" +) + +type autoscaleRuntimeFactory struct { + server *Server +} + +type autoscaleReplicaObservation struct { + Stats docker.ContainerStats + Limits docker.ResourceLimits +} + +func (f autoscaleRuntimeFactory) Build(ctx context.Context, deployment string, state autoscale.State) (autoscale.RuntimeSession, error) { + if state.Provider == orchestrator.ProviderK3s { + provider := orchestrator.NewK3sProvider(f.server.config.Cluster.K3s.Kubeconfig, f.server.config.Cluster.K3s.Namespace) + status, err := provider.Status(ctx, deployment) + if err != nil { + return autoscale.RuntimeSession{}, err + } + usage, err := provider.Metrics(ctx, deployment) + if err != nil { + return autoscale.RuntimeSession{}, err + } + routeProvider := routing.NewK3sIngressProvider(f.server.config.Cluster.K3s.Kubeconfig, f.server.config.Cluster.K3s.Namespace, state.Route) + return autoscale.RuntimeSession{ + Input: autoscale.Input{ + Replicas: status.Desired, CPUPercent: usage.CPUPercent, MemoryPercent: usage.MemoryPercent, + Diagnosis: capacity.Diagnosis{Pressure: capacity.PressureNone, Action: capacity.ActionNone, Reason: "K3s workload metrics are within policy"}, + }, + Executor: autoscale.NewExecutor(provider, routeProvider), + }, nil + } + if state.Provider != orchestrator.ProviderSwarm { + return autoscale.RuntimeSession{}, fmt.Errorf("orchestrator %q does not support active reconciliation", state.Provider) + } + provider, err := orchestrator.NewSwarmProviderFromEnv() + if err != nil { + return autoscale.RuntimeSession{}, err + } + fail := func(err error) (autoscale.RuntimeSession, error) { + _ = provider.Close() + return autoscale.RuntimeSession{}, err + } + status, err := provider.Status(ctx, deployment) + if err != nil { + return fail(err) + } + stats, err := docker.GetManagedDeploymentStats(deployment) + if err != nil { + return fail(fmt.Errorf("read managed workload statistics: %w", err)) + } + if status.Available == 0 || len(stats) != status.Available { + return fail(fmt.Errorf("statistics are available for %d of %d running replicas", len(stats), status.Available)) + } + hostStats, err := system.GetSystemStats() + if err != nil { + return fail(fmt.Errorf("read host capacity: %w", err)) + } + observations := make([]autoscaleReplicaObservation, 0, len(stats)) + for _, stat := range stats { + limits, err := docker.GetContainerResources(stat.ContainerID) + if err != nil { + return fail(fmt.Errorf("read resources for replica %s: %w", stat.ContainerID, err)) + } + observations = append(observations, autoscaleReplicaObservation{Stats: stat, Limits: *limits}) + } + input := autoscaleInput(observations, status, hostStats, f.server.config.Capacity) + placement, err := provider.Placement(ctx, deployment) + if err != nil { + return fail(fmt.Errorf("read managed workload placement: %w", err)) + } + if usesFleetPlacement(placement) { + available, err := f.server.fleetCapacityAvailable(ctx, provider, input.CurrentResources) + if err != nil { + return fail(fmt.Errorf("read permitted Fleet capacity: %w", err)) + } + input.FleetOffer = capacity.Offer{Enabled: available} + } + routeProvider := routing.NewManagedNginxProvider(f.server.proxyOrchestrator.NginxManager(), f.server.manager, state.Route) + return autoscale.RuntimeSession{ + Input: input, Executor: autoscale.NewExecutor(provider, routeProvider), Close: provider.Close, + }, nil +} + +func usesFleetPlacement(placement orchestrator.Placement) bool { + for _, constraint := range placement.Constraints { + if strings.HasPrefix(constraint, "node.labels.flatrun.capacity.") { + return true + } + } + return false +} + +func autoscaleInput(observations []autoscaleReplicaObservation, status orchestrator.Status, hostStats *system.SystemStats, configPolicy config.CapacityConfig) autoscale.Input { + var selected capacity.Container + var cpuPercent float64 + var memoryPercent float64 + var score float64 + for _, observation := range observations { + stat := observation.Stats + limits := observation.Limits + container := capacity.Container{ + ID: stat.ContainerID, Name: stat.Name, CPUPercent: stat.CPUPercent, CPULimit: limits.CPUs, + MemoryUsage: stat.MemoryUsage, MemoryLimit: uint64(max(limits.MemoryLimit, 0)), + } + containerScore := math.Max(stat.CPUPercent, stat.MemoryPercent) + if selected.ID == "" || containerScore > score { + selected = container + score = containerScore + } + cpuPercent = math.Max(cpuPercent, stat.CPUPercent) + memoryPercent = math.Max(memoryPercent, stat.MemoryPercent) + } + host := capacity.Host{ + CPUCores: float64(hostStats.CPU.Cores), CPUUsagePercent: hostStats.CPU.UsagePercent, + MemoryTotal: hostStats.Memory.Total, MemoryAvailable: hostStats.Memory.Available, + } + policy := capacity.PolicyFromConfig(configPolicy) + diagnosis := capacity.Diagnose(host, selected, policy) + resources := orchestrator.Resources{CPULimit: selected.CPULimit, MemoryLimit: selected.MemoryLimit} + suggested := resources + if diagnosis.Action == capacity.ActionIncreaseCPU { + suggested.CPULimit = diagnosis.RecommendedLimit + } + if diagnosis.Action == capacity.ActionIncreaseMemory { + suggested.MemoryLimit = uint64(diagnosis.RecommendedLimit) + } + return autoscale.Input{ + Replicas: status.Desired, CPUPercent: cpuPercent, MemoryPercent: memoryPercent, + Diagnosis: diagnosis, RequiresFleet: diagnosis.Action == capacity.ActionAddReplica, + CurrentResources: resources, SuggestedResource: suggested, + } +} diff --git a/internal/api/autoscale_runtime_test.go b/internal/api/autoscale_runtime_test.go new file mode 100644 index 0000000..130ec4a --- /dev/null +++ b/internal/api/autoscale_runtime_test.go @@ -0,0 +1,36 @@ +package api + +import ( + "testing" + + "github.com/flatrun/agent/internal/capacity" + "github.com/flatrun/agent/internal/docker" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/system" + "github.com/flatrun/agent/pkg/config" +) + +func TestAutoscaleInputUsesMostConstrainedReplica(t *testing.T) { + input := autoscaleInput([]autoscaleReplicaObservation{ + {Stats: docker.ContainerStats{ContainerID: "one", CPUPercent: 25, MemoryUsage: 100, MemoryPercent: 10}, Limits: docker.ResourceLimits{CPUs: 1, MemoryLimit: 1000}}, + {Stats: docker.ContainerStats{ContainerID: "two", CPUPercent: 95, MemoryUsage: 900, MemoryPercent: 90}, Limits: docker.ResourceLimits{CPUs: 1, MemoryLimit: 1000}}, + }, orchestrator.Status{Desired: 2}, &system.SystemStats{ + CPU: system.CPUStats{Cores: 8, UsagePercent: 20}, + Memory: system.MemoryStats{Total: 16 << 30, Available: 8 << 30}, + }, config.CapacityConfig{}) + if input.Replicas != 2 || input.CPUPercent != 95 || input.MemoryPercent != 90 { + t.Fatalf("input = %#v", input) + } + if input.Diagnosis.Action != capacity.ActionIncreaseMemory || input.SuggestedResource.MemoryLimit <= input.CurrentResources.MemoryLimit { + t.Fatalf("diagnosis = %#v, input = %#v", input.Diagnosis, input) + } +} + +func TestUsesFleetPlacementOnlyForCapacityLabels(t *testing.T) { + if !usesFleetPlacement(orchestrator.Placement{Constraints: []string{"node.labels.flatrun.capacity.a1b2==true"}}) { + t.Fatal("Fleet capacity placement was not detected") + } + if usesFleetPlacement(orchestrator.Placement{Constraints: []string{"node.hostname==prod-1"}}) { + t.Fatal("local placement was treated as Fleet capacity") + } +} diff --git a/internal/api/cluster_handlers.go b/internal/api/cluster_handlers.go index ae5e9d1..1b52eb8 100644 --- a/internal/api/cluster_handlers.go +++ b/internal/api/cluster_handlers.go @@ -1,50 +1,456 @@ package api import ( + "context" "crypto/rand" + "crypto/sha256" "database/sql" "encoding/base64" + "encoding/hex" "encoding/json" "fmt" "io" "net/http" + "net/url" + "slices" + "strings" "time" "github.com/flatrun/agent/internal/auth" + "github.com/flatrun/agent/internal/capacity" "github.com/flatrun/agent/internal/cluster" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" + "github.com/flatrun/agent/internal/system" + "github.com/flatrun/agent/pkg/config" "github.com/flatrun/agent/pkg/version" "github.com/gin-gonic/gin" ) +type clusterProviderOption struct { + ID string `json:"id"` + Active bool `json:"active"` + Available bool `json:"available"` + Reason string `json:"reason,omitempty"` +} + +type clusterProvidersResponse struct { + Orchestrators []clusterProviderOption `json:"orchestrators"` + Routing []clusterProviderOption `json:"routing"` + K3s config.K3sConfig `json:"k3s"` +} + +type clusterCapacityClaimResponse struct { + Enabled bool `json:"enabled"` + Reason string `json:"reason"` + Node orchestrator.NodeIdentity `json:"node"` + Constraint string `json:"constraint,omitempty"` + MaxCPU float64 `json:"max_cpu,omitempty"` + MaxMemory uint64 `json:"max_memory,omitempty"` + MaxReplicas int `json:"max_replicas,omitempty"` +} + +func (s *Server) clusterCapacityClaim(c *gin.Context) { + peer, err := clusterPeerFromActor(auth.GetActorFromContext(c)) + if err != nil { + c.JSON(http.StatusForbidden, gin.H{"error": err.Error()}) + return + } + mgr := s.getClusterManager() + if mgr == nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) + return + } + policy, err := mgr.DB().GetPeerPolicy(peer) + if err != nil { + c.JSON(http.StatusForbidden, gin.H{"error": "Capacity has not been granted to this peer"}) + return + } + grant, ok := capacityOfferGrant(*policy) + if !ok { + c.JSON(http.StatusOK, clusterCapacityClaimResponse{Reason: "This server has not permitted Fleet workloads from this peer"}) + return + } + provider, err := orchestrator.NewSwarmProviderFromEnv() + if err != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()}) + return + } + defer provider.Close() + label := capacityNodeLabel(peer) + node, err := provider.EnsureLocalNodeLabel(c, label, "true") + if err != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, clusterCapacityClaimResponse{ + Enabled: true, Reason: "This node permits Fleet workloads from this peer", Node: node, + Constraint: "node.labels." + label + "==true", + MaxCPU: grant.MaxCPU, MaxMemory: grant.MaxMemory, MaxReplicas: grant.MaxReplicas, + }) +} + +func clusterPeerFromActor(actor *auth.ActorContext) (string, error) { + if actor == nil || actor.APIKey == nil { + return "", fmt.Errorf("A Fleet peer credential is required") + } + peer, ok := strings.CutPrefix(actor.APIKey.Name, "cluster-peer-") + if !ok || strings.TrimSpace(peer) == "" { + return "", fmt.Errorf("A Fleet peer credential is required") + } + return peer, nil +} + +func capacityOfferGrant(policy cluster.PeerPolicy) (cluster.Grant, bool) { + for _, grant := range policy.Grants { + if grant.Capability == cluster.CapabilityCapacityOffer { + return grant, true + } + } + return cluster.Grant{}, false +} + +func capacityNodeLabel(peer string) string { + sum := sha256.Sum256([]byte(peer)) + return "flatrun.capacity." + hex.EncodeToString(sum[:6]) +} + +func (s *Server) clusterProviders(c *gin.Context) { + orchestratorID := s.config.Cluster.Orchestrator + if orchestratorID == "" { + orchestratorID = string(orchestrator.ProviderStandalone) + } + routingID := s.config.Cluster.Routing + if routingID == "" { + routingID = string(routing.ProviderNginx) + } + + c.JSON(http.StatusOK, clusterProvidersResponse{ + Orchestrators: []clusterProviderOption{ + s.orchestratorOption(c, orchestrator.ProviderStandalone, orchestratorID), + s.orchestratorOption(c, orchestrator.ProviderSwarm, orchestratorID), + s.orchestratorOption(c, orchestrator.ProviderK3s, orchestratorID), + }, + Routing: []clusterProviderOption{ + s.routingOption(c, routing.ProviderNginx, routingID), + s.routingOption(c, routing.ProviderTraefik, routingID), + }, + K3s: s.config.Cluster.K3s, + }) +} + +func (s *Server) orchestratorOption(c *gin.Context, id orchestrator.ProviderID, active string) clusterProviderOption { + option := clusterProviderOption{ID: string(id), Active: active == string(id)} + if err := s.checkOrchestrator(c, id); err != nil { + option.Reason = err.Error() + return option + } + option.Available = true + return option +} + +func (s *Server) routingOption(c *gin.Context, id routing.ProviderID, active string) clusterProviderOption { + option := clusterProviderOption{ID: string(id), Active: active == string(id)} + if err := s.checkRouting(c, id); err != nil { + option.Reason = err.Error() + return option + } + option.Available = true + return option +} + +func (s *Server) checkOrchestrator(ctx context.Context, id orchestrator.ProviderID) error { + if s.probeOrchestrator != nil { + return s.probeOrchestrator(ctx, id) + } + switch id { + case orchestrator.ProviderStandalone: + return nil + case orchestrator.ProviderSwarm: + provider, err := orchestrator.NewSwarmProviderFromEnv() + if err != nil { + return err + } + defer provider.Close() + return provider.Ready(ctx) + case orchestrator.ProviderK3s: + provider := orchestrator.NewK3sProvider(s.config.Cluster.K3s.Kubeconfig, s.config.Cluster.K3s.Namespace) + return provider.Ready(ctx) + default: + return fmt.Errorf("orchestrator %q is not supported", id) + } +} + +func (s *Server) checkRouting(ctx context.Context, id routing.ProviderID) error { + if s.probeRouting != nil { + return s.probeRouting(ctx, id) + } + switch id { + case routing.ProviderNginx: + if !s.config.Nginx.Enabled { + return fmt.Errorf("Nginx is not enabled") + } + return nil + case routing.ProviderTraefik: + return orchestrator.NewK3sProvider(s.config.Cluster.K3s.Kubeconfig, s.config.Cluster.K3s.Namespace).Ready(ctx) + default: + return fmt.Errorf("routing provider %q is not supported", id) + } +} + +type updateClusterProvidersRequest struct { + Orchestrator string `json:"orchestrator" binding:"required"` + Routing string `json:"routing" binding:"required"` + K3s config.K3sConfig `json:"k3s"` +} + +func (s *Server) updateClusterProviders(c *gin.Context) { + var req updateClusterProvidersRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + orchestratorID := orchestrator.ProviderID(strings.TrimSpace(req.Orchestrator)) + routingID := routing.ProviderID(strings.TrimSpace(req.Routing)) + if (orchestratorID == orchestrator.ProviderK3s && routingID != routing.ProviderTraefik) || + (orchestratorID == orchestrator.ProviderSwarm && routingID != routing.ProviderNginx) || + (orchestratorID == orchestrator.ProviderStandalone && routingID != routing.ProviderNginx) { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": "The selected orchestrator and routing providers are incompatible"}) + return + } + var orchestratorErr error + if orchestratorID == orchestrator.ProviderK3s && s.probeOrchestrator == nil { + orchestratorErr = orchestrator.NewK3sProvider(req.K3s.Kubeconfig, req.K3s.Namespace).Ready(c) + } else { + orchestratorErr = s.checkOrchestrator(c, orchestratorID) + } + if orchestratorErr != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": orchestratorErr.Error()}) + return + } + var routingErr error + if routingID == routing.ProviderTraefik && s.probeRouting == nil { + routingErr = orchestrator.NewK3sProvider(req.K3s.Kubeconfig, req.K3s.Namespace).Ready(c) + } else { + routingErr = s.checkRouting(c, routingID) + } + if routingErr != nil { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": routingErr.Error()}) + return + } + + previous := s.config.Cluster + s.config.Cluster.Orchestrator = string(orchestratorID) + s.config.Cluster.Routing = string(routingID) + s.config.Cluster.K3s = req.K3s + if s.configPath != "" { + if err := config.Save(s.config, s.configPath); err != nil { + s.config.Cluster = previous + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to save provider configuration: %v", err)}) + return + } + } + c.JSON(http.StatusOK, gin.H{ + "orchestrator": s.config.Cluster.Orchestrator, + "routing": s.config.Cluster.Routing, + "k3s": s.config.Cluster.K3s, + }) +} + func (s *Server) clusterStatus(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusOK, gin.H{ "enabled": false, }) return } - peers := s.clusterManager.ListPeers() + peers := mgr.ListPeers() c.JSON(http.StatusOK, gin.H{ "enabled": true, - "server_name": s.clusterManager.ServerName(), + "server_name": mgr.ServerName(), "peer_count": len(peers), "version": version.Get(), }) } +type clusterSetupRequest struct { + ServerName string `json:"server_name" binding:"required"` + AdvertiseURL string `json:"advertise_url" binding:"required"` +} + +func (s *Server) clusterSetup(c *gin.Context) { + var req clusterSetupRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + req.ServerName = strings.TrimSpace(req.ServerName) + req.AdvertiseURL = strings.TrimRight(strings.TrimSpace(req.AdvertiseURL), "/") + if req.ServerName == "" || strings.ContainsAny(req.ServerName, "/\\") { + c.JSON(http.StatusBadRequest, gin.H{"error": "Server name must not be empty or contain slashes"}) + return + } + parsedURL, err := url.ParseRequestURI(req.AdvertiseURL) + if err != nil || parsedURL.Host == "" || (parsedURL.Scheme != "http" && parsedURL.Scheme != "https") { + c.JSON(http.StatusBadRequest, gin.H{"error": "Advertise URL must be a valid HTTP or HTTPS URL"}) + return + } + + s.clusterMu.Lock() + defer s.clusterMu.Unlock() + if s.clusterManager != nil { + c.JSON(http.StatusConflict, gin.H{"error": "Cluster is already enabled"}) + return + } + + clusterDB, err := cluster.NewDB(s.config.DeploymentsPath) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to initialize cluster database: %v", err)}) + return + } + previous := s.config.Cluster + s.config.Cluster.Enabled = true + s.config.Cluster.ServerName = req.ServerName + s.config.Cluster.AdvertiseURL = req.AdvertiseURL + if s.config.Cluster.HealthInterval == "" { + s.config.Cluster.HealthInterval = "30s" + } + if s.config.Cluster.RequestTimeout == "" { + s.config.Cluster.RequestTimeout = "10s" + } + if s.configPath != "" { + if err := config.Save(s.config, s.configPath); err != nil { + s.config.Cluster = previous + _ = clusterDB.Close() + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to save cluster configuration: %v", err)}) + return + } + } + + healthInterval, _ := time.ParseDuration(s.config.Cluster.HealthInterval) + requestTimeout, _ := time.ParseDuration(s.config.Cluster.RequestTimeout) + if healthInterval <= 0 { + healthInterval = 30 * time.Second + } + if requestTimeout <= 0 { + requestTimeout = 10 * time.Second + } + mgr := cluster.NewManager(clusterDB, req.ServerName, healthInterval, requestTimeout, s.config.Auth.JWTSecret) + mgr.SetEventPublisher(s.notify) + if err := mgr.Start(context.Background()); err != nil { + _ = clusterDB.Close() + s.config.Cluster = previous + if s.configPath != "" { + _ = config.Save(s.config, s.configPath) + } + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to start cluster: %v", err)}) + return + } + s.clusterManager = mgr + + c.JSON(http.StatusOK, gin.H{ + "enabled": true, + "server_name": req.ServerName, + "advertise_url": req.AdvertiseURL, + "peer_count": 0, + "version": version.Get(), + }) +} + +func (s *Server) getClusterManager() *cluster.Manager { + s.clusterMu.RLock() + defer s.clusterMu.RUnlock() + return s.clusterManager +} + func (s *Server) clusterListPeers(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } - peers := s.clusterManager.ListPeers() + peers := mgr.ListPeers() c.JSON(http.StatusOK, gin.H{"peers": peers}) } +func (s *Server) clusterPeerPolicy(c *gin.Context) { + mgr := s.getClusterManager() + if mgr == nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) + return + } + policy, err := mgr.DB().GetPeerPolicy(c.Param("name")) + if err != nil { + if err == sql.ErrNoRows { + c.JSON(http.StatusNotFound, gin.H{"error": "Peer not found"}) + return + } + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, policy) +} + +func (s *Server) updateClusterPeerPolicy(c *gin.Context) { + mgr := s.getClusterManager() + if mgr == nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) + return + } + policy := cluster.PeerPolicy{Peer: c.Param("name")} + if err := c.ShouldBindJSON(&policy); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + policy.Peer = c.Param("name") + seen := make(map[cluster.Capability]bool, len(policy.Grants)) + for _, grant := range policy.Grants { + if !cluster.ValidCapability(grant.Capability) { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Unknown capability %q", grant.Capability)}) + return + } + if seen[grant.Capability] { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Duplicate capability %q", grant.Capability)}) + return + } + if grant.MaxCPU < 0 || grant.MaxReplicas < 0 { + c.JSON(http.StatusBadRequest, gin.H{"error": "Resource limits cannot be negative"}) + return + } + seen[grant.Capability] = true + } + previous, err := mgr.DB().GetPeerPolicy(policy.Peer) + if err != nil { + if err == sql.ErrNoRows { + c.JSON(http.StatusNotFound, gin.H{"error": "Peer not found"}) + return + } + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if err := mgr.DB().SetPeerPolicy(policy); err != nil { + if err == sql.ErrNoRows { + c.JSON(http.StatusNotFound, gin.H{"error": "Peer not found"}) + return + } + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + if err := s.applyClusterPeerPolicy(policy); err != nil { + _ = mgr.DB().SetPeerPolicy(*previous) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to apply peer policy: " + err.Error()}) + return + } + c.JSON(http.StatusOK, policy) +} + func (s *Server) clusterInvite(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } @@ -70,7 +476,7 @@ func (s *Server) clusterInvite(c *gin.Context) { ExpiresAt: time.Now().Add(1 * time.Hour), } - if _, err := s.clusterManager.DB().CreateInvite(invite); err != nil { + if _, err := mgr.DB().CreateInvite(invite); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create invite"}) return } @@ -82,7 +488,8 @@ func (s *Server) clusterInvite(c *gin.Context) { } func (s *Server) clusterAccept(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } @@ -119,7 +526,7 @@ func (s *Server) clusterAccept(c *gin.Context) { InviteToken: req.InviteToken, URL: callbackURL, APIKey: ourAPIKeyForThem, - Name: s.clusterManager.ServerName(), + Name: mgr.ServerName(), } body, err := json.Marshal(exchangeReq) @@ -145,13 +552,17 @@ func (s *Server) clusterAccept(c *gin.Context) { return } - if err := s.clusterManager.AddPeer(exchangeResp.Name, req.PeerURL, exchangeResp.APIKey); err != nil { + if err := mgr.AddPeer(exchangeResp.Name, req.PeerURL, exchangeResp.APIKey); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to store peer: %v", err)}) return } if s.authManager != nil { - s.createClusterAPIKey(ourAPIKeyForThem, exchangeResp.Name) + if err := s.createClusterAPIKey(ourAPIKeyForThem, exchangeResp.Name); err != nil { + _ = mgr.RemovePeer(exchangeResp.Name) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create peer credential"}) + return + } } c.JSON(http.StatusOK, gin.H{ @@ -174,7 +585,8 @@ type exchangeResponse struct { } func (s *Server) clusterExchange(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } @@ -186,7 +598,7 @@ func (s *Server) clusterExchange(c *gin.Context) { } tokenHash := cluster.HashToken(req.InviteToken) - invite, err := s.clusterManager.DB().GetInviteByHash(tokenHash) + invite, err := mgr.DB().GetInviteByHash(tokenHash) if err != nil { if err == sql.ErrNoRows { c.JSON(http.StatusNotFound, gin.H{"error": "Invalid or expired invite token"}) @@ -206,7 +618,7 @@ func (s *Server) clusterExchange(c *gin.Context) { return } - if err := s.clusterManager.DB().ConsumeInvite(tokenHash, req.Name); err != nil { + if err := mgr.DB().ConsumeInvite(tokenHash, req.Name); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to consume invite"}) return } @@ -218,54 +630,201 @@ func (s *Server) clusterExchange(c *gin.Context) { } ourAPIKeyForThem := base64.URLEncoding.EncodeToString(apiKeyBytes) - if err := s.clusterManager.AddPeer(req.Name, req.URL, req.APIKey); err != nil { + if err := mgr.AddPeer(req.Name, req.URL, req.APIKey); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to store peer: %v", err)}) return } if s.authManager != nil { - s.createClusterAPIKey(ourAPIKeyForThem, req.Name) + if err := s.createClusterAPIKey(ourAPIKeyForThem, req.Name); err != nil { + _ = mgr.RemovePeer(req.Name) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create peer credential"}) + return + } } c.JSON(http.StatusOK, exchangeResponse{ APIKey: ourAPIKeyForThem, - Name: s.clusterManager.ServerName(), + Name: mgr.ServerName(), }) } -func (s *Server) createClusterAPIKey(rawKey, peerName string) { +func (s *Server) createClusterAPIKey(rawKey, peerName string) error { if s.authManager == nil { - return + return fmt.Errorf("Authentication manager is not available") } - _, _ = s.authManager.CreateAPIKeyFromRaw( + userID, err := s.clusterServiceUserID() + if err != nil { + return err + } + permissions, deployments := clusterPolicyAccess(cluster.PeerPolicy{Peer: peerName, Grants: cluster.DefaultPeerGrants()}) + _, err = s.authManager.CreateAPIKeyFromRaw( rawKey, - 1, + userID, fmt.Sprintf("cluster-peer-%s", peerName), fmt.Sprintf("Auto-generated API key for cluster peer %s", peerName), - auth.RoleAdmin, - nil, - nil, + auth.Role(""), + permissions, + deployments, time.Time{}, ) + return err +} + +func (s *Server) clusterServiceUserID() (int64, error) { + const username = "__flatrun_cluster" + user, err := s.authManager.GetUserByUsername(username) + if err == nil { + if user.Role != auth.RoleService { + return 0, fmt.Errorf("Reserved Fleet identity has an invalid role") + } + return user.ID, nil + } + passwordBytes := make([]byte, 32) + if _, err := rand.Read(passwordBytes); err != nil { + return 0, err + } + user, err = s.authManager.CreateUser(username, "", base64.RawURLEncoding.EncodeToString(passwordBytes), auth.RoleService, nil) + if err != nil { + return 0, err + } + return user.ID, nil +} + +func clusterPolicyAccess(policy cluster.PeerPolicy) ([]string, auth.DeploymentAccess) { + permissions := make(map[string]bool) + deployments := make(auth.DeploymentAccess) + unrestrictedDeployments := false + for _, grant := range policy.Grants { + switch grant.Capability { + case cluster.CapabilityFleetRead: + permissions[auth.PermClusterRead.String()] = true + case cluster.CapabilityDeploymentsRead: + permissions[auth.PermDeploymentsRead.String()] = true + permissions[auth.PermContainersRead.String()] = true + unrestrictedDeployments = mergeClusterDeploymentAccess(deployments, grant.Deployments, auth.AccessLevelRead, unrestrictedDeployments) + case cluster.CapabilityDeploymentsRun: + permissions[auth.PermDeploymentsRead.String()] = true + permissions[auth.PermDeploymentsWrite.String()] = true + permissions[auth.PermContainersRead.String()] = true + permissions[auth.PermContainersWrite.String()] = true + unrestrictedDeployments = mergeClusterDeploymentAccess(deployments, grant.Deployments, auth.AccessLevelWrite, unrestrictedDeployments) + case cluster.CapabilityCapacityRead: + permissions[auth.PermSystemRead.String()] = true + case cluster.CapabilityCapacityOffer: + permissions[auth.PermClusterCapacityClaim.String()] = true + case cluster.CapabilityRoutingManage: + permissions[auth.PermInfrastructureRead.String()] = true + permissions[auth.PermInfrastructureWrite.String()] = true + } + } + result := make([]string, 0, len(permissions)) + for permission := range permissions { + result = append(result, permission) + } + slices.Sort(result) + if unrestrictedDeployments { + deployments = nil + } + return result, deployments +} + +func mergeClusterDeploymentAccess(access auth.DeploymentAccess, names []string, level string, unrestricted bool) bool { + if unrestricted || len(names) == 0 { + return true + } + for _, name := range names { + if current, ok := access[name]; !ok || current == auth.AccessLevelRead && level == auth.AccessLevelWrite { + access[name] = level + } + } + return false +} + +func (s *Server) applyClusterPeerPolicy(policy cluster.PeerPolicy) error { + if s.authManager == nil { + return fmt.Errorf("Authentication manager is not available") + } + keys, err := s.authManager.GetAllAPIKeys() + if err != nil { + return err + } + name := fmt.Sprintf("cluster-peer-%s", policy.Peer) + for _, key := range keys { + if key.Name != name || !key.IsActive { + continue + } + permissions, deployments := clusterPolicyAccess(policy) + _, err := s.authManager.UpdateAPIKey( + key.ID, + key.Name, + key.Description, + auth.Role(""), + permissions, + deployments, + key.ExpiresAt, + ) + return err + } + return fmt.Errorf("Active peer credential not found") +} + +func (s *Server) reconcileClusterPeerPolicies() error { + if s.clusterManager == nil || s.authManager == nil { + return nil + } + peers, err := s.clusterManager.DB().ListPeers() + if err != nil { + return err + } + for _, peer := range peers { + policy, err := s.clusterManager.DB().GetPeerPolicy(peer.Name) + if err != nil { + return err + } + if err := s.applyClusterPeerPolicy(*policy); err != nil { + return err + } + } + return nil } func (s *Server) clusterRemovePeer(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } name := c.Param("name") - if err := s.clusterManager.RemovePeer(name); err != nil { + if err := mgr.RemovePeer(name); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } + s.revokeClusterAPIKey(name) c.JSON(http.StatusOK, gin.H{"status": "removed", "peer": name}) } +func (s *Server) revokeClusterAPIKey(peerName string) { + if s.authManager == nil { + return + } + keys, err := s.authManager.GetAllAPIKeys() + if err != nil { + return + } + name := fmt.Sprintf("cluster-peer-%s", peerName) + for _, key := range keys { + if key.Name == name { + _ = s.authManager.DeactivateAPIKey(key.ID) + } + } +} + func (s *Server) clusterProxy(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } @@ -273,7 +832,7 @@ func (s *Server) clusterProxy(c *gin.Context) { name := c.Param("name") path := c.Param("path") - client, err := s.clusterManager.GetPeer(name) + client, err := mgr.GetPeer(name) if err != nil { c.JSON(http.StatusNotFound, gin.H{"error": err.Error()}) return @@ -299,7 +858,8 @@ func (s *Server) clusterProxy(c *gin.Context) { } func (s *Server) clusterAggregateDeployments(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } @@ -310,18 +870,19 @@ func (s *Server) clusterAggregateDeployments(c *gin.Context) { return } - localData, err := json.Marshal(deployments) + localData, err := json.Marshal(NewList(deployments, "deployments").Also("path", s.manager.BasePath())) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to marshal local deployments"}) return } - result := cluster.AggregateFromPeers(c.Request.Context(), localData, s.clusterManager, "/api/deployments") + result := cluster.AggregateFromPeers(c.Request.Context(), localData, mgr, "/api/deployments") c.JSON(http.StatusOK, result) } func (s *Server) clusterAggregateStats(c *gin.Context) { - if s.clusterManager == nil { + mgr := s.getClusterManager() + if mgr == nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) return } @@ -337,6 +898,39 @@ func (s *Server) clusterAggregateStats(c *gin.Context) { return } - result := cluster.AggregateFromPeers(c.Request.Context(), localData, s.clusterManager, "/api/health") + result := cluster.AggregateFromPeers(c.Request.Context(), localData, mgr, "/api/health") + c.JSON(http.StatusOK, result) +} + +func (s *Server) clusterAggregateCapacity(c *gin.Context) { + mgr := s.getClusterManager() + if mgr == nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Cluster is not enabled"}) + return + } + + hostStats, err := system.GetSystemStats() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + host := capacity.Host{ + CPUCores: float64(hostStats.CPU.Cores), + CPUUsagePercent: hostStats.CPU.UsagePercent, + MemoryTotal: hostStats.Memory.Total, + MemoryAvailable: hostStats.Memory.Available, + } + policy := capacity.PolicyFromConfig(s.config.Capacity) + localData, err := json.Marshal(gin.H{ + "host": host, + "policy": policy, + "offer": capacity.FleetOffer(host, policy, s.config.Capacity.OfferToFleet), + }) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to marshal local capacity"}) + return + } + + result := cluster.AggregateFromPeers(c.Request.Context(), localData, mgr, "/api/capacity") c.JSON(http.StatusOK, result) } diff --git a/internal/api/cluster_handlers_test.go b/internal/api/cluster_handlers_test.go index 6194760..c9f9ef1 100644 --- a/internal/api/cluster_handlers_test.go +++ b/internal/api/cluster_handlers_test.go @@ -4,18 +4,40 @@ import ( "bytes" "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "os" + "slices" "testing" "time" "github.com/flatrun/agent/internal/auth" + "github.com/flatrun/agent/internal/capacity" "github.com/flatrun/agent/internal/cluster" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" "github.com/flatrun/agent/pkg/config" "github.com/gin-gonic/gin" ) +func TestCapacityClaimUsesPeerSpecificGrant(t *testing.T) { + peer, err := clusterPeerFromActor(&auth.ActorContext{APIKey: &auth.APIKey{Name: "cluster-peer-prod1"}}) + if err != nil || peer != "prod1" { + t.Fatalf("peer = %q, error = %v", peer, err) + } + grant, ok := capacityOfferGrant(cluster.PeerPolicy{Grants: []cluster.Grant{ + {Capability: cluster.CapabilityCapacityRead}, + {Capability: cluster.CapabilityCapacityOffer, MaxCPU: 2, MaxMemory: 4 << 30, MaxReplicas: 3}, + }}) + if !ok || grant.MaxReplicas != 3 || grant.MaxCPU != 2 { + t.Fatalf("grant = %#v, found = %v", grant, ok) + } + if capacityNodeLabel("prod1") == capacityNodeLabel("prod2") { + t.Fatal("peer labels must be isolated") + } +} + type testClusterEnv struct { server *Server router *gin.Engine @@ -77,6 +99,7 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool server := &Server{ config: cfg, + configPath: tmpDir + "/config.yml", authManager: authManager, clusterManager: clusterManager, } @@ -92,23 +115,37 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool protected := api.Group("") protected.Use(authMiddleware.RequireAuth()) { + protected.GET("/capacity", authMiddleware.RequirePermission(auth.PermSystemRead), server.getCapacityStatus) + protected.GET("/test/deployments", authMiddleware.RequirePermission(auth.PermDeploymentsRead), func(c *gin.Context) { + c.Status(http.StatusNoContent) + }) + protected.GET("/test/users", authMiddleware.RequirePermission(auth.PermUsersWrite), func(c *gin.Context) { + c.Status(http.StatusNoContent) + }) + protected.POST("/cluster/capacity/claim", authMiddleware.RequirePermission(auth.PermClusterCapacityClaim), server.clusterCapacityClaim) clusterGroup := protected.Group("/cluster") clusterGroup.Use(authMiddleware.RequirePermission(auth.PermClusterRead)) { clusterGroup.GET("/status", server.clusterStatus) + clusterGroup.GET("/providers", server.clusterProviders) + clusterGroup.PUT("/providers", authMiddleware.RequirePermission(auth.PermClusterWrite), server.updateClusterProviders) + clusterGroup.POST("/setup", authMiddleware.RequirePermission(auth.PermClusterWrite), server.clusterSetup) clusterGroup.GET("/peers", server.clusterListPeers) + clusterGroup.GET("/peers/:name/policy", server.clusterPeerPolicy) + clusterGroup.PUT("/peers/:name/policy", authMiddleware.RequirePermission(auth.PermClusterWrite), server.updateClusterPeerPolicy) clusterGroup.POST("/invite", authMiddleware.RequirePermission(auth.PermClusterWrite), server.clusterInvite) clusterGroup.POST("/accept", authMiddleware.RequirePermission(auth.PermClusterWrite), server.clusterAccept) clusterGroup.DELETE("/peers/:name", authMiddleware.RequirePermission(auth.PermClusterWrite), server.clusterRemovePeer) clusterGroup.Any("/peers/:name/proxy/*path", authMiddleware.RequirePermission(auth.PermClusterWrite), server.clusterProxy) clusterGroup.GET("/deployments", server.clusterAggregateDeployments) clusterGroup.GET("/stats", server.clusterAggregateStats) + clusterGroup.GET("/capacity", server.clusterAggregateCapacity) } } cleanup := func() { - if clusterManager != nil { - clusterManager.Stop() + if manager := server.getClusterManager(); manager != nil { + manager.Stop() } authManager.Close() os.RemoveAll(tmpDir) @@ -122,6 +159,316 @@ func setupClusterTestServer(t *testing.T, serverName string, clusterEnabled bool } } +func TestClusterCapacityIncludesLocalOfferPolicy(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + + token := clusterLogin(t, env.router) + req := httptest.NewRequest(http.MethodGet, "/api/cluster/capacity", nil) + req.Header.Set("Authorization", "Bearer "+token) + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + var response struct { + Servers map[string]struct { + Online bool `json:"online"` + Data struct { + Offer capacity.Offer `json:"offer"` + } `json:"data"` + } `json:"servers"` + } + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatalf("decode response: %v", err) + } + local, ok := response.Servers["server-a"] + if !ok || !local.Online { + t.Fatalf("local server = %#v", local) + } + if local.Data.Offer.Enabled { + t.Fatal("fleet capacity should require explicit permission") + } +} + +func TestUpdateClusterPeerPolicyThroughHTTP(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + if err := env.server.clusterManager.AddPeer("server-b", "https://server-b.example.com", "peer-key"); err != nil { + t.Fatalf("AddPeer failed: %v", err) + } + if err := env.server.createClusterAPIKey("server-b-inbound-key", "server-b"); err != nil { + t.Fatalf("createClusterAPIKey failed: %v", err) + } + + token := clusterLogin(t, env.router) + body := []byte(`{"grants":[{"capability":"capacity.offer","max_cpu":2,"max_memory":2147483648,"max_replicas":2}]}`) + req := httptest.NewRequest(http.MethodPut, "/api/cluster/peers/server-b/policy", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + + getReq := httptest.NewRequest(http.MethodGet, "/api/cluster/peers/server-b/policy", nil) + getReq.Header.Set("Authorization", "Bearer "+token) + getWriter := httptest.NewRecorder() + env.router.ServeHTTP(getWriter, getReq) + if getWriter.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", getWriter.Code, getWriter.Body.String()) + } + var policy cluster.PeerPolicy + if err := json.Unmarshal(getWriter.Body.Bytes(), &policy); err != nil { + t.Fatalf("decode policy: %v", err) + } + if len(policy.Grants) != 1 || policy.Grants[0].MaxCPU != 2 || policy.Grants[0].MaxReplicas != 2 { + t.Fatalf("policy = %#v", policy) + } + keys, err := env.server.authManager.GetAllAPIKeys() + if err != nil { + t.Fatalf("list API keys: %v", err) + } + for _, key := range keys { + if key.Name == "cluster-peer-server-b" && !slices.Equal(key.Permissions, []string{auth.PermClusterCapacityClaim.String()}) { + t.Fatalf("capacity offer permissions = %#v", key.Permissions) + } + } + claimReq := httptest.NewRequest(http.MethodPost, "/api/cluster/capacity/claim", nil) + claimReq.Header.Set("Authorization", "Bearer server-b-inbound-key") + claimWriter := httptest.NewRecorder() + env.router.ServeHTTP(claimWriter, claimReq) + if claimWriter.Code == http.StatusForbidden || claimWriter.Code == http.StatusUnauthorized { + t.Fatalf("capacity offer credential was rejected by middleware: %d: %s", claimWriter.Code, claimWriter.Body.String()) + } +} + +func TestReconcileClusterPeerPoliciesScopesExistingCredentials(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + if err := env.server.clusterManager.AddPeer("server-b", "https://server-b.example.com", "peer-key"); err != nil { + t.Fatalf("AddPeer failed: %v", err) + } + user, err := env.server.authManager.GetUserByUsername("admin") + if err != nil { + t.Fatalf("get admin: %v", err) + } + _, err = env.server.authManager.CreateAPIKeyFromRaw( + "existing-peer-key", + user.ID, + "cluster-peer-server-b", + "Existing Fleet peer", + auth.RoleAdmin, + nil, + nil, + time.Time{}, + ) + if err != nil { + t.Fatalf("create existing peer credential: %v", err) + } + + if err := env.server.reconcileClusterPeerPolicies(); err != nil { + t.Fatalf("reconcile peer policies: %v", err) + } + keys, err := env.server.authManager.GetAllAPIKeys() + if err != nil { + t.Fatalf("list API keys: %v", err) + } + for _, key := range keys { + if key.Name != "cluster-peer-server-b" { + continue + } + if key.Role != "" { + t.Fatalf("peer role = %q", key.Role) + } + permissions, _ := clusterPolicyAccess(cluster.PeerPolicy{Grants: cluster.DefaultPeerGrants()}) + if !slices.Equal(key.Permissions, permissions) { + t.Fatalf("peer permissions = %#v, want %#v", key.Permissions, permissions) + } + return + } + t.Fatal("existing peer credential not found") +} + +func TestClusterPolicyAccessScopesDeployments(t *testing.T) { + permissions, deployments := clusterPolicyAccess(cluster.PeerPolicy{Grants: []cluster.Grant{ + {Capability: cluster.CapabilityDeploymentsRead, Deployments: []string{"public-site", "docs"}}, + {Capability: cluster.CapabilityDeploymentsRun, Deployments: []string{"public-site"}}, + {Capability: cluster.CapabilityCapacityRead}, + }}) + + if deployments["public-site"] != auth.AccessLevelWrite || deployments["docs"] != auth.AccessLevelRead { + t.Fatalf("deployment access = %#v", deployments) + } + wanted := map[string]bool{ + auth.PermDeploymentsRead.String(): true, + auth.PermDeploymentsWrite.String(): true, + auth.PermContainersRead.String(): true, + auth.PermContainersWrite.String(): true, + auth.PermSystemRead.String(): true, + } + for _, permission := range permissions { + delete(wanted, permission) + } + if len(wanted) != 0 { + t.Fatalf("missing permissions = %#v", wanted) + } +} + +func TestClusterSetupEnablesClusterWithoutRestart(t *testing.T) { + env := setupClusterTestServer(t, "", false) + defer env.cleanup() + + token := clusterLogin(t, env.router) + body, _ := json.Marshal(map[string]string{ + "server_name": "server-a", + "advertise_url": "https://server-a.example.com/", + }) + req := httptest.NewRequest(http.MethodPost, "/api/cluster/setup", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("Expected 200, got %d: %s", w.Code, w.Body.String()) + } + + statusReq := httptest.NewRequest(http.MethodGet, "/api/cluster/status", nil) + statusReq.Header.Set("Authorization", "Bearer "+token) + statusWriter := httptest.NewRecorder() + env.router.ServeHTTP(statusWriter, statusReq) + if statusWriter.Code != http.StatusOK { + t.Fatalf("Expected status 200, got %d: %s", statusWriter.Code, statusWriter.Body.String()) + } + + var status map[string]interface{} + if err := json.Unmarshal(statusWriter.Body.Bytes(), &status); err != nil { + t.Fatal(err) + } + if status["enabled"] != true || status["server_name"] != "server-a" { + t.Fatalf("Unexpected status: %s", statusWriter.Body.String()) + } + if env.server.config.Cluster.AdvertiseURL != "https://server-a.example.com" { + t.Fatalf("AdvertiseURL = %q", env.server.config.Cluster.AdvertiseURL) + } +} + +func TestClusterSetupRejectsInvalidAdvertiseURL(t *testing.T) { + env := setupClusterTestServer(t, "", false) + defer env.cleanup() + + token := clusterLogin(t, env.router) + body, _ := json.Marshal(map[string]string{"server_name": "server-a", "advertise_url": "server-a"}) + req := httptest.NewRequest(http.MethodPost, "/api/cluster/setup", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("Expected 400, got %d: %s", w.Code, w.Body.String()) + } + if env.server.getClusterManager() != nil { + t.Fatal("Cluster manager started after invalid setup") + } +} + +func TestClusterAPIKeyUsesExplicitPermissions(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + + if err := env.server.createClusterAPIKey("peer-key-for-test", "server-b"); err != nil { + t.Fatalf("createClusterAPIKey failed: %v", err) + } + keys, err := env.server.authManager.GetAllAPIKeys() + if err != nil { + t.Fatal(err) + } + var peerKey *auth.APIKey + for i := range keys { + if keys[i].Name == "cluster-peer-server-b" { + peerKey = &keys[i] + break + } + } + if peerKey == nil { + t.Fatal("Cluster API key was not created") + } + if peerKey.Role == auth.RoleAdmin { + t.Fatal("Cluster API key has administrator role") + } + owner, err := env.server.authManager.GetUser(peerKey.UserID) + if err != nil { + t.Fatalf("get key owner: %v", err) + } + if owner.Role != auth.RoleService { + t.Fatalf("key owner role = %q", owner.Role) + } + actor, err := env.server.authManager.BuildActorContext(owner, peerKey) + if err != nil { + t.Fatalf("build actor: %v", err) + } + if actor.HasPermission(auth.PermUsersWrite) || !actor.HasPermission(auth.PermDeploymentsWrite) { + t.Fatalf("effective actor = %#v", actor) + } + permissions := make(map[string]bool, len(peerKey.Permissions)) + for _, permission := range peerKey.Permissions { + permissions[permission] = true + } + if !permissions[auth.PermDeploymentsRead.String()] || !permissions[auth.PermDeploymentsWrite.String()] { + t.Fatalf("permissions = %#v", peerKey.Permissions) + } + if permissions[auth.PermUsersWrite.String()] || permissions[auth.PermConfigWrite.String()] { + t.Fatalf("permissions include administrative access: %#v", peerKey.Permissions) + } +} + +func TestClusterAPIKeyEnforcesPermissionsThroughHTTP(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + const rawKey = "peer-http-key-for-test" + if err := env.server.createClusterAPIKey(rawKey, "server-b"); err != nil { + t.Fatalf("createClusterAPIKey failed: %v", err) + } + + request := func(path string) int { + req := httptest.NewRequest(http.MethodGet, path, nil) + req.Header.Set("Authorization", "Bearer "+rawKey) + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + return w.Code + } + if status := request("/api/test/deployments"); status != http.StatusNoContent { + t.Fatalf("deployment read status = %d", status) + } + if status := request("/api/test/users"); status != http.StatusForbidden { + t.Fatalf("user write status = %d", status) + } +} + +func TestCapacityClaimRejectsUnpermittedPeerThroughHTTP(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + const rawKey = "peer-capacity-key-for-test" + if err := env.server.getClusterManager().AddPeer("server-b", "http://server-b.invalid", "remote-key"); err != nil { + t.Fatal(err) + } + if err := env.server.createClusterAPIKey(rawKey, "server-b"); err != nil { + t.Fatal(err) + } + req := httptest.NewRequest(http.MethodPost, "/api/cluster/capacity/claim", nil) + req.Header.Set("Authorization", "Bearer "+rawKey) + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + if w.Code != http.StatusForbidden { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } +} + func clusterLogin(t *testing.T, router *gin.Engine) string { t.Helper() body, _ := json.Marshal(map[string]string{ @@ -499,6 +846,119 @@ func TestClusterUnauthorizedAccess(t *testing.T) { } } +func TestClusterProvidersReportActiveAndAvailableAdapters(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + env.server.probeOrchestrator = func(_ context.Context, id orchestrator.ProviderID) error { + if id == orchestrator.ProviderK3s { + return fmt.Errorf("k3s adapter is not configured") + } + return nil + } + env.server.probeRouting = func(_ context.Context, id routing.ProviderID) error { + if id == routing.ProviderTraefik { + return fmt.Errorf("Traefik adapter is not configured") + } + return nil + } + + req := httptest.NewRequest(http.MethodGet, "/api/cluster/providers", nil) + req.Header.Set("Authorization", "Bearer "+clusterLogin(t, env.router)) + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + var response clusterProvidersResponse + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if !response.Orchestrators[0].Active || !response.Orchestrators[0].Available { + t.Fatalf("unexpected standalone provider: %+v", response.Orchestrators[0]) + } + if response.Orchestrators[2].Available || response.Orchestrators[2].Reason == "" { + t.Fatalf("unexpected k3s provider: %+v", response.Orchestrators[2]) + } + if !response.Routing[0].Active || !response.Routing[0].Available { + t.Fatalf("unexpected Nginx provider: %+v", response.Routing[0]) + } +} + +func TestUpdateClusterProvidersPersistsAvailableSelection(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + env.server.probeOrchestrator = func(_ context.Context, _ orchestrator.ProviderID) error { return nil } + env.server.probeRouting = func(_ context.Context, _ routing.ProviderID) error { return nil } + + body := bytes.NewBufferString(`{"orchestrator":"swarm","routing":"nginx"}`) + req := httptest.NewRequest(http.MethodPut, "/api/cluster/providers", body) + req.Header.Set("Authorization", "Bearer "+clusterLogin(t, env.router)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + + saved, err := config.Load(env.server.configPath) + if err != nil { + t.Fatal(err) + } + if saved.Cluster.Orchestrator != "swarm" || saved.Cluster.Routing != "nginx" { + t.Fatalf("unexpected saved providers: %+v", saved.Cluster) + } +} + +func TestUpdateClusterProvidersPersistsK3sConnection(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + env.server.probeOrchestrator = func(_ context.Context, _ orchestrator.ProviderID) error { return nil } + env.server.probeRouting = func(_ context.Context, _ routing.ProviderID) error { return nil } + + body := bytes.NewBufferString(`{"orchestrator":"k3s","routing":"traefik","k3s":{"kubeconfig":"/etc/rancher/k3s/k3s.yaml","namespace":"flatrun"}}`) + req := httptest.NewRequest(http.MethodPut, "/api/cluster/providers", body) + req.Header.Set("Authorization", "Bearer "+clusterLogin(t, env.router)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String()) + } + + saved, err := config.Load(env.server.configPath) + if err != nil { + t.Fatal(err) + } + if saved.Cluster.Routing != "traefik" || saved.Cluster.K3s.Kubeconfig != "/etc/rancher/k3s/k3s.yaml" || saved.Cluster.K3s.Namespace != "flatrun" { + t.Fatalf("unexpected K3s connection: %+v", saved.Cluster.K3s) + } +} + +func TestUpdateClusterProvidersRejectsUnavailableSelection(t *testing.T) { + env := setupClusterTestServer(t, "server-a", true) + defer env.cleanup() + env.server.probeOrchestrator = func(_ context.Context, id orchestrator.ProviderID) error { + if id == orchestrator.ProviderK3s { + return fmt.Errorf("k3s adapter is not configured") + } + return nil + } + env.server.probeRouting = func(_ context.Context, _ routing.ProviderID) error { return nil } + + body := bytes.NewBufferString(`{"orchestrator":"k3s","routing":"nginx"}`) + req := httptest.NewRequest(http.MethodPut, "/api/cluster/providers", body) + req.Header.Set("Authorization", "Bearer "+clusterLogin(t, env.router)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + env.router.ServeHTTP(w, req) + if w.Code != http.StatusUnprocessableEntity { + t.Fatalf("expected 422, got %d: %s", w.Code, w.Body.String()) + } + if env.server.config.Cluster.Orchestrator != "" { + t.Fatalf("unexpected orchestrator change: %q", env.server.config.Cluster.Orchestrator) + } +} + func TestClusterFullPeeringE2E(t *testing.T) { // This test simulates the full peering flow between two servers // using httptest servers as live HTTP endpoints. diff --git a/internal/api/events_handlers.go b/internal/api/events_handlers.go new file mode 100644 index 0000000..d2028ce --- /dev/null +++ b/internal/api/events_handlers.go @@ -0,0 +1,58 @@ +package api + +import ( + "net/http" + "time" + + "github.com/flatrun/agent/internal/events" + "github.com/flatrun/agent/internal/notify" + "github.com/gin-gonic/gin" +) + +func (s *Server) listNotificationRules(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{"rules": s.notify.Load().Rules}) +} + +func (s *Server) updateNotificationRules(c *gin.Context) { + var req struct { + Rules []notify.Rule `json:"rules"` + } + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request"}) + return + } + if err := s.notify.UpdateRules(req.Rules); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, gin.H{"rules": s.notify.Load().Rules}) +} + +func (s *Server) listNotificationIncidents(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{"incidents": s.notify.Incidents()}) +} + +func (s *Server) emitEvent(c *gin.Context) { + if s.pluginToken == "" || c.GetHeader("X-Plugin-Token") != s.pluginToken { + c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"}) + return + } + var event events.Event + if err := c.ShouldBindJSON(&event); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request"}) + return + } + if event.Source == "" || event.Type == "" || event.Title == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "source, type, and title are required"}) + return + } + if event.OccurredAt.IsZero() { + event.OccurredAt = time.Now().UTC() + } + result, err := s.notify.Publish(event) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error(), "result": result}) + return + } + c.JSON(http.StatusAccepted, result) +} diff --git a/internal/api/notifications_test.go b/internal/api/notifications_test.go index 60b927c..9f45026 100644 --- a/internal/api/notifications_test.go +++ b/internal/api/notifications_test.go @@ -23,11 +23,61 @@ func setupNotifyTest(t *testing.T) (*Server, *gin.Engine) { } r := gin.New() r.GET("/notifications/targets", s.getNotificationTargets) + r.GET("/notifications/incidents", s.listNotificationIncidents) + r.GET("/notifications/rules", s.listNotificationRules) + r.PUT("/notifications/rules", s.updateNotificationRules) r.PUT("/notifications/targets", s.updateNotificationTargets) + r.POST("/notifications/test", s.testNotification) r.POST("/internal/notify/emit", s.emitNotification) + r.POST("/internal/events", s.emitEvent) return s, r } +func TestNotificationRulesRoundTripThroughHTTP(t *testing.T) { + _, r := setupNotifyTest(t) + payload := `{"rules":[{"id":"critical-fleet","name":"Critical fleet incidents","enabled":true,"topics":["fleet"],"severities":["critical"],"target_ids":["email"]}]}` + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPut, "/notifications/rules", bytes.NewBufferString(payload)) + req.Header.Set("Content-Type", "application/json") + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) + } + + w = httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/notifications/rules", nil)) + if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"id":"critical-fleet"`) { + t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) + } +} + +func TestEmitEventCorrelatesIncidentThroughHTTP(t *testing.T) { + _, r := setupNotifyTest(t) + emit := func(payload string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/internal/events", bytes.NewBufferString(payload)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Plugin-Token", "plugin-token") + r.ServeHTTP(w, req) + return w + } + + first := emit(`{"source":"fleet","type":"node.unavailable","severity":"critical","title":"prod2 unavailable","scope":{"node":"prod2"}}`) + if first.Code != http.StatusAccepted { + t.Fatalf("status = %d, body = %s", first.Code, first.Body.String()) + } + second := emit(`{"source":"capacity","type":"deployment.unavailable","severity":"critical","title":"app unavailable","scope":{"node":"prod2","deployment":"app"},"correlation_key":"node:prod2"}`) + if second.Code != http.StatusAccepted { + t.Fatalf("status = %d, body = %s", second.Code, second.Body.String()) + } + + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/notifications/incidents", nil)) + if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"event_count":2`) { + t.Fatalf("incidents status = %d, body = %s", w.Code, w.Body.String()) + } +} + func TestGetNotificationTargetsMasksSecret(t *testing.T) { s, r := setupNotifyTest(t) if err := s.notify.Save(notify.Config{Targets: []notify.Target{ @@ -49,6 +99,29 @@ func TestGetNotificationTargetsMasksSecret(t *testing.T) { if !strings.Contains(body, notify.MaskedURL) { t.Errorf("response should mask the target URL: %s", body) } + if !strings.Contains(body, `"kind":"email"`) { + t.Errorf("response should identify the safe target kind: %s", body) + } +} + +func TestNotificationTargetByIDThroughHTTP(t *testing.T) { + s, r := setupNotifyTest(t) + if err := s.notify.Save(notify.Config{Targets: []notify.Target{ + {ID: "ops", Name: "Operations", URL: "generic+https://example.com/hook", Enabled: false}, + }}); err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/notifications/test", strings.NewReader(`{"target_id":"ops"}`)) + req.Header.Set("Content-Type", "application/json") + r.ServeHTTP(w, req) + + if w.Code != http.StatusBadGateway { + t.Fatalf("status = %d, body = %s", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "target is disabled") { + t.Fatalf("saved target was not selected: %s", w.Body.String()) + } } func TestUpdateNotificationTargetsPreservesMaskedSecret(t *testing.T) { diff --git a/internal/api/openapi.json b/internal/api/openapi.json index 7f9182e..508edd4 100644 --- a/internal/api/openapi.json +++ b/internal/api/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "FlatRun Agent API", "description": "Generated from the agent's routes and the types its handlers bind and return.", - "version": "0.4.0-beta.4" + "version": "0.4.0-beta.5" }, "paths": { "/api/agent/update": { @@ -1257,6 +1257,20 @@ "x-permission": "backups:write" } }, + "/api/capacity": { + "get": { + "operationId": "get-capacity", + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "capacity" + ], + "x-permission": "system:read" + } + }, "/api/certificates": { "get": { "operationId": "get-certificates", @@ -1511,6 +1525,47 @@ "x-permission": "cluster:write" } }, + "/api/cluster/capacity": { + "get": { + "operationId": "get-cluster-capacity", + "tags": [ + "cluster" + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/cluster.AggregatedResponse" + } + } + } + } + } + } + }, + "/api/cluster/capacity/claim": { + "post": { + "operationId": "post-cluster-capacity-claim", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/api.clusterCapacityClaimResponse" + } + } + }, + "description": "Success" + } + }, + "tags": [ + "cluster" + ], + "x-permission": "cluster:capacity:claim" + } + }, "/api/cluster/deployments": { "get": { "operationId": "get-cluster-deployments", @@ -1612,6 +1667,141 @@ "x-permission": "cluster:write" } }, + "/api/cluster/peers/{name}/policy": { + "get": { + "operationId": "get-cluster-peers-by-name-policy", + "tags": [ + "cluster" + ], + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/cluster.PeerPolicy" + } + } + } + } + } + }, + "put": { + "operationId": "put-cluster-peers-by-name-policy", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/cluster.PeerPolicy" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/cluster.PeerPolicy" + } + } + }, + "description": "Success" + } + }, + "tags": [ + "cluster" + ], + "x-permission": "cluster:write" + } + }, + "/api/cluster/providers": { + "get": { + "operationId": "get-cluster-providers", + "tags": [ + "cluster" + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/api.clusterProvidersResponse" + } + } + } + } + } + }, + "put": { + "operationId": "put-cluster-providers", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/api.updateClusterProvidersRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "cluster" + ], + "x-permission": "cluster:write" + } + }, + "/api/cluster/setup": { + "post": { + "operationId": "post-cluster-setup", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/api.clusterSetupRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "cluster" + ], + "x-permission": "cluster:write" + } + }, "/api/cluster/stats": { "get": { "operationId": "get-cluster-stats", @@ -1799,6 +1989,30 @@ "x-permission": "containers:delete" } }, + "/api/containers/{id}/capacity": { + "get": { + "operationId": "get-containers-by-id-capacity", + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "containers" + ], + "x-permission": "containers:read" + } + }, "/api/containers/{id}/exec": { "get": { "operationId": "get-containers-by-id-exec", @@ -3642,9 +3856,9 @@ "x-permission": "deployments:read" } }, - "/api/deployments/{name}/backup-config": { + "/api/deployments/{name}/autoscale": { "get": { - "operationId": "get-deployments-by-name-backup-config", + "operationId": "get-deployments-by-name-autoscale", "parameters": [ { "in": "path", @@ -3657,16 +3871,23 @@ ], "responses": { "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/api.autoscalePolicyResponse" + } + } + }, "description": "Success" } }, "tags": [ "deployments" ], - "x-permission": "backups:read" + "x-permission": "deployments:read" }, "put": { - "operationId": "put-deployments-by-name-backup-config", + "operationId": "put-deployments-by-name-autoscale", "parameters": [ { "in": "path", @@ -3681,7 +3902,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/models.BackupSpec" + "$ref": "#/components/schemas/api.autoscalePolicyRequest" } } }, @@ -3689,18 +3910,25 @@ }, "responses": { "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/api.autoscalePolicyResponse" + } + } + }, "description": "Success" } }, "tags": [ "deployments" ], - "x-permission": "backups:write" + "x-permission": "deployments:write" } }, - "/api/deployments/{name}/backups": { - "get": { - "operationId": "get-deployments-by-name-backups", + "/api/deployments/{name}/autoscale/activate": { + "post": { + "operationId": "post-deployments-by-name-autoscale-activate", "parameters": [ { "in": "path", @@ -3709,13 +3937,6 @@ "schema": { "type": "string" } - }, - { - "in": "query", - "name": "limit", - "schema": { - "type": "string" - } } ], "responses": { @@ -3723,7 +3944,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/api.ListOfBackup" + "$ref": "#/components/schemas/autoscale.Activation" } } }, @@ -3733,10 +3954,12 @@ "tags": [ "deployments" ], - "x-permission": "backups:read" - }, - "post": { - "operationId": "post-deployments-by-name-backups", + "x-permission": "deployments:write" + } + }, + "/api/deployments/{name}/autoscale/compatibility": { + "get": { + "operationId": "get-deployments-by-name-autoscale-compatibility", "parameters": [ { "in": "path", @@ -3755,12 +3978,12 @@ "tags": [ "deployments" ], - "x-permission": "backups:write" + "x-permission": "deployments:read" } }, - "/api/deployments/{name}/certificates/renew": { - "post": { - "operationId": "post-deployments-by-name-certificates-renew", + "/api/deployments/{name}/autoscale/workload": { + "put": { + "operationId": "put-deployments-by-name-autoscale-workload", "parameters": [ { "in": "path", @@ -3771,6 +3994,16 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/models.ScalingConfig" + } + } + }, + "required": true + }, "responses": { "200": { "description": "Success" @@ -3779,12 +4012,12 @@ "tags": [ "deployments" ], - "x-permission": "certificates:write" + "x-permission": "deployments:write" } }, - "/api/deployments/{name}/compose": { + "/api/deployments/{name}/backup-config": { "get": { - "operationId": "get-deployments-by-name-compose", + "operationId": "get-deployments-by-name-backup-config", "parameters": [ { "in": "path", @@ -3803,12 +4036,152 @@ "tags": [ "deployments" ], - "x-permission": "deployments:read" - } - }, - "/api/deployments/{name}/compose/mount": { - "post": { - "operationId": "post-deployments-by-name-compose-mount", + "x-permission": "backups:read" + }, + "put": { + "operationId": "put-deployments-by-name-backup-config", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/models.BackupSpec" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "deployments" + ], + "x-permission": "backups:write" + } + }, + "/api/deployments/{name}/backups": { + "get": { + "operationId": "get-deployments-by-name-backups", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "limit", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/api.ListOfBackup" + } + } + }, + "description": "Success" + } + }, + "tags": [ + "deployments" + ], + "x-permission": "backups:read" + }, + "post": { + "operationId": "post-deployments-by-name-backups", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "deployments" + ], + "x-permission": "backups:write" + } + }, + "/api/deployments/{name}/certificates/renew": { + "post": { + "operationId": "post-deployments-by-name-certificates-renew", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "deployments" + ], + "x-permission": "certificates:write" + } + }, + "/api/deployments/{name}/compose": { + "get": { + "operationId": "get-deployments-by-name-compose", + "parameters": [ + { + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "deployments" + ], + "x-permission": "deployments:read" + } + }, + "/api/deployments/{name}/compose/mount": { + "post": { + "operationId": "post-deployments-by-name-compose-mount", "parameters": [ { "in": "path", @@ -6485,6 +6858,67 @@ "x-permission": "networks:write" } }, + "/api/notifications/incidents": { + "get": { + "operationId": "get-notifications-incidents", + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "notifications" + ], + "x-permission": "settings:read" + } + }, + "/api/notifications/rules": { + "get": { + "operationId": "get-notifications-rules", + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "notifications" + ], + "x-permission": "settings:read" + }, + "put": { + "operationId": "put-notifications-rules", + "requestBody": { + "content": { + "application/json": { + "schema": { + "properties": { + "rules": { + "items": { + "$ref": "#/components/schemas/notify.Rule" + }, + "type": "array" + } + }, + "type": "object", + "x-property-order": [ + "rules" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Success" + } + }, + "tags": [ + "notifications" + ], + "x-permission": "settings:write" + } + }, "/api/notifications/targets": { "get": { "operationId": "get-notifications-targets", @@ -6543,16 +6977,21 @@ "application/json": { "schema": { "properties": { + "target_id": { + "type": "string" + }, "url": { "type": "string" } }, "type": "object", "x-columns": [ - "url" + "url", + "target_id" ], "x-property-order": [ - "url" + "url", + "target_id" ] } } @@ -11006,63 +11445,295 @@ "tail" ] }, - "api.deploymentSource": { + "api.autoscalePolicyRequest": { "type": "object", "properties": { - "branch": { - "type": "string" + "allow_fleet_capacity": { + "type": "boolean" }, - "credential_id": { - "type": "string" + "cooldown_seconds": { + "type": "integer" }, - "ref": { - "type": "string" + "enabled": { + "type": "boolean" }, - "subpath": { - "type": "string" + "max_replicas": { + "type": "integer" }, - "token": { - "type": "string" + "min_replicas": { + "type": "integer" }, - "type": { - "type": "string" + "scale_down_percent": { + "type": "number" }, - "username": { - "type": "string" + "scale_down_windows": { + "type": "integer" + }, + "scale_up_percent": { + "type": "number" + }, + "scale_up_windows": { + "type": "integer" } }, "x-property-order": [ - "type", - "ref", - "branch", - "subpath", - "credential_id", - "username", - "token" + "enabled", + "min_replicas", + "max_replicas", + "scale_up_percent", + "scale_down_percent", + "scale_up_windows", + "scale_down_windows", + "cooldown_seconds", + "allow_fleet_capacity" ], "x-columns": [ - "type", - "ref", - "branch", - "subpath", - "credential_id", - "username", - "token" + "enabled", + "min_replicas", + "max_replicas", + "scale_up_percent", + "scale_down_percent", + "scale_up_windows", + "scale_down_windows", + "cooldown_seconds", + "allow_fleet_capacity" ] }, - "api.exchangeRequest": { + "api.autoscalePolicyResponse": { "type": "object", "properties": { - "api_key": { - "type": "string" + "allow_fleet_capacity": { + "type": "boolean" }, - "invite_token": { - "type": "string" + "cooldown_seconds": { + "type": "integer" }, - "name": { - "type": "string" + "enabled": { + "type": "boolean" }, - "url": { + "max_replicas": { + "type": "integer" + }, + "min_replicas": { + "type": "integer" + }, + "scale_down_percent": { + "type": "number" + }, + "scale_down_windows": { + "type": "integer" + }, + "scale_up_percent": { + "type": "number" + }, + "scale_up_windows": { + "type": "integer" + }, + "state": { + "$ref": "#/components/schemas/autoscale.State" + } + }, + "x-property-order": [ + "enabled", + "min_replicas", + "max_replicas", + "scale_up_percent", + "scale_down_percent", + "scale_up_windows", + "scale_down_windows", + "cooldown_seconds", + "allow_fleet_capacity", + "state" + ], + "x-columns": [ + "enabled", + "min_replicas", + "max_replicas", + "scale_up_percent", + "scale_down_percent", + "scale_up_windows", + "scale_down_windows", + "cooldown_seconds", + "allow_fleet_capacity" + ] + }, + "api.clusterCapacityClaimResponse": { + "type": "object", + "properties": { + "constraint": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "max_cpu": { + "type": "number" + }, + "max_memory": { + "type": "integer" + }, + "max_replicas": { + "type": "integer" + }, + "node": { + "$ref": "#/components/schemas/orchestrator.NodeIdentity" + }, + "reason": { + "type": "string" + } + }, + "x-property-order": [ + "enabled", + "reason", + "node", + "constraint", + "max_cpu", + "max_memory", + "max_replicas" + ], + "x-columns": [ + "enabled", + "reason", + "constraint", + "max_cpu", + "max_memory", + "max_replicas" + ] + }, + "api.clusterProviderOption": { + "type": "object", + "properties": { + "active": { + "type": "boolean" + }, + "available": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "x-property-order": [ + "id", + "active", + "available", + "reason" + ], + "x-columns": [ + "id", + "active", + "available", + "reason" + ] + }, + "api.clusterProvidersResponse": { + "type": "object", + "properties": { + "k3s": { + "$ref": "#/components/schemas/config.K3sConfig" + }, + "orchestrators": { + "type": "array", + "items": { + "$ref": "#/components/schemas/api.clusterProviderOption" + } + }, + "routing": { + "type": "array", + "items": { + "$ref": "#/components/schemas/api.clusterProviderOption" + } + } + }, + "x-property-order": [ + "orchestrators", + "routing", + "k3s" + ] + }, + "api.clusterSetupRequest": { + "type": "object", + "properties": { + "advertise_url": { + "type": "string" + }, + "server_name": { + "type": "string" + } + }, + "x-property-order": [ + "server_name", + "advertise_url" + ], + "x-columns": [ + "server_name", + "advertise_url" + ], + "required": [ + "server_name", + "advertise_url" + ] + }, + "api.deploymentSource": { + "type": "object", + "properties": { + "branch": { + "type": "string" + }, + "credential_id": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "subpath": { + "type": "string" + }, + "token": { + "type": "string" + }, + "type": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "x-property-order": [ + "type", + "ref", + "branch", + "subpath", + "credential_id", + "username", + "token" + ], + "x-columns": [ + "type", + "ref", + "branch", + "subpath", + "credential_id", + "username", + "token" + ] + }, + "api.exchangeRequest": { + "type": "object", + "properties": { + "api_key": { + "type": "string" + }, + "invite_token": { + "type": "string" + }, + "name": { + "type": "string" + }, + "url": { "type": "string" } }, @@ -11241,6 +11912,33 @@ "deployment" ] }, + "api.updateClusterProvidersRequest": { + "type": "object", + "properties": { + "k3s": { + "$ref": "#/components/schemas/config.K3sConfig" + }, + "orchestrator": { + "type": "string" + }, + "routing": { + "type": "string" + } + }, + "x-property-order": [ + "orchestrator", + "routing", + "k3s" + ], + "x-columns": [ + "orchestrator", + "routing" + ], + "required": [ + "orchestrator", + "routing" + ] + }, "audit.ActorStats": { "type": "object", "properties": { @@ -11468,6 +12166,70 @@ "count" ] }, + "autoscale.Activation": { + "type": "object", + "properties": { + "route": { + "$ref": "#/components/schemas/routing.Route" + }, + "workload": { + "$ref": "#/components/schemas/orchestrator.Status" + } + }, + "x-property-order": [ + "workload", + "route" + ] + }, + "autoscale.State": { + "type": "object", + "properties": { + "active": { + "type": "boolean" + }, + "high_windows": { + "type": "integer" + }, + "last_action": { + "type": "string", + "format": "date-time" + }, + "low_windows": { + "type": "integer" + }, + "provider": { + "type": "string" + }, + "replicas": { + "type": "integer" + }, + "route": { + "$ref": "#/components/schemas/routing.Route" + }, + "service": { + "type": "string" + } + }, + "x-property-order": [ + "high_windows", + "low_windows", + "last_action", + "active", + "provider", + "service", + "replicas", + "route" + ], + "x-columns": [ + "high_windows", + "low_windows", + "last_action", + "active", + "provider", + "service", + "replicas" + ] + }, "backup.Backup": { "type": "object", "properties": { @@ -11611,6 +12373,63 @@ "servers" ] }, + "cluster.Grant": { + "type": "object", + "properties": { + "capability": { + "type": "string" + }, + "deployments": { + "type": "array", + "items": { + "type": "string" + } + }, + "max_cpu": { + "type": "number" + }, + "max_memory": { + "type": "integer" + }, + "max_replicas": { + "type": "integer" + } + }, + "x-property-order": [ + "capability", + "deployments", + "max_cpu", + "max_memory", + "max_replicas" + ], + "x-columns": [ + "capability", + "max_cpu", + "max_memory", + "max_replicas" + ] + }, + "cluster.PeerPolicy": { + "type": "object", + "properties": { + "grants": { + "type": "array", + "items": { + "$ref": "#/components/schemas/cluster.Grant" + } + }, + "peer": { + "type": "string" + } + }, + "x-property-order": [ + "peer", + "grants" + ], + "x-columns": [ + "peer" + ] + }, "cluster.ServerResult": { "type": "object", "properties": { @@ -11705,6 +12524,25 @@ "enabled" ] }, + "config.K3sConfig": { + "type": "object", + "properties": { + "kubeconfig": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "x-property-order": [ + "kubeconfig", + "namespace" + ], + "x-columns": [ + "kubeconfig", + "namespace" + ] + }, "dashboards.Dashboard": { "type": "object", "properties": { @@ -13029,6 +13867,48 @@ "auto_cert" ] }, + "models.ScalingConfig": { + "type": "object", + "properties": { + "service": { + "type": "string" + }, + "stateless": { + "type": "boolean" + }, + "storage": { + "$ref": "#/components/schemas/models.ScalingStorageConfig" + } + }, + "x-property-order": [ + "service", + "stateless", + "storage" + ], + "x-columns": [ + "service", + "stateless" + ] + }, + "models.ScalingStorageConfig": { + "type": "object", + "properties": { + "class": { + "type": "string" + }, + "mode": { + "type": "string" + } + }, + "x-property-order": [ + "mode", + "class" + ], + "x-columns": [ + "mode", + "class" + ] + }, "models.Service": { "type": "object", "properties": { @@ -13142,6 +14022,9 @@ "require_plan": { "type": "boolean" }, + "scaling": { + "$ref": "#/components/schemas/models.ScalingConfig" + }, "security": { "$ref": "#/components/schemas/models.DeploymentSecurityConfig" }, @@ -13170,6 +14053,7 @@ "quick_actions", "security", "backup", + "scaling", "protected_mode", "require_plan", "credential_id", @@ -13189,6 +14073,12 @@ "notify.Config": { "type": "object", "properties": { + "rules": { + "type": "array", + "items": { + "$ref": "#/components/schemas/notify.Rule" + } + }, "targets": { "type": "array", "items": { @@ -13197,12 +14087,92 @@ } }, "x-property-order": [ - "targets" + "targets", + "rules" + ] + }, + "notify.Rule": { + "type": "object", + "properties": { + "deployments": { + "type": "array", + "items": { + "type": "string" + } + }, + "enabled": { + "type": "boolean" + }, + "event_types": { + "type": "array", + "items": { + "type": "string" + } + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "nodes": { + "type": "array", + "items": { + "type": "string" + } + }, + "notifications": { + "type": "array", + "items": { + "type": "string" + } + }, + "severities": { + "type": "array", + "items": { + "type": "string" + } + }, + "target_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "topics": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "x-property-order": [ + "id", + "name", + "enabled", + "topics", + "event_types", + "severities", + "nodes", + "deployments", + "notifications", + "target_ids" + ], + "x-columns": [ + "id", + "name", + "enabled" ] }, "notify.Target": { "type": "object", "properties": { + "deployments": { + "type": "array", + "items": { + "type": "string" + } + }, "enabled": { "type": "boolean" }, @@ -13212,6 +14182,24 @@ "name": { "type": "string" }, + "nodes": { + "type": "array", + "items": { + "type": "string" + } + }, + "severities": { + "type": "array", + "items": { + "type": "string" + } + }, + "topics": { + "type": "array", + "items": { + "type": "string" + } + }, "url": { "type": "string" } @@ -13220,7 +14208,11 @@ "id", "name", "url", - "enabled" + "enabled", + "topics", + "severities", + "nodes", + "deployments" ], "x-columns": [ "id", @@ -13229,6 +14221,165 @@ "enabled" ] }, + "orchestrator.Instance": { + "type": "object", + "properties": { + "address": { + "type": "string" + }, + "healthy": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "node": { + "type": "string" + }, + "ready": { + "type": "boolean" + } + }, + "x-property-order": [ + "id", + "node", + "address", + "healthy", + "ready" + ], + "x-columns": [ + "id", + "node", + "address", + "healthy", + "ready" + ] + }, + "orchestrator.NodeIdentity": { + "type": "object", + "properties": { + "cluster_id": { + "type": "string" + }, + "hostname": { + "type": "string" + }, + "id": { + "type": "string" + } + }, + "x-property-order": [ + "id", + "hostname", + "cluster_id" + ], + "x-columns": [ + "id", + "hostname", + "cluster_id" + ] + }, + "orchestrator.Status": { + "type": "object", + "properties": { + "available": { + "type": "integer" + }, + "desired": { + "type": "integer" + }, + "instances": { + "type": "array", + "items": { + "$ref": "#/components/schemas/orchestrator.Instance" + } + }, + "workload": { + "type": "string" + } + }, + "x-property-order": [ + "workload", + "desired", + "available", + "instances" + ], + "x-columns": [ + "workload", + "desired", + "available" + ] + }, + "routing.Backend": { + "type": "object", + "properties": { + "address": { + "type": "string" + }, + "healthy": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "weight": { + "type": "integer" + } + }, + "x-property-order": [ + "id", + "address", + "healthy", + "weight" + ], + "x-columns": [ + "id", + "address", + "healthy", + "weight" + ] + }, + "routing.Route": { + "type": "object", + "properties": { + "backends": { + "type": "array", + "items": { + "$ref": "#/components/schemas/routing.Backend" + } + }, + "domain": { + "type": "string" + }, + "id": { + "type": "string" + }, + "path": { + "type": "string" + }, + "protocol": { + "type": "string" + }, + "service": { + "type": "string" + } + }, + "x-property-order": [ + "id", + "service", + "domain", + "path", + "protocol", + "backends" + ], + "x-columns": [ + "id", + "service", + "domain", + "path", + "protocol" + ] + }, "scheduler.AgentTaskConfig": { "type": "object", "properties": { diff --git a/internal/api/resource_handlers.go b/internal/api/resource_handlers.go index 70d6f63..5885a9b 100644 --- a/internal/api/resource_handlers.go +++ b/internal/api/resource_handlers.go @@ -4,10 +4,32 @@ import ( "net/http" "github.com/flatrun/agent/internal/auth" + "github.com/flatrun/agent/internal/capacity" "github.com/flatrun/agent/internal/docker" + "github.com/flatrun/agent/internal/system" "github.com/gin-gonic/gin" ) +func (s *Server) getCapacityStatus(c *gin.Context) { + hostStats, err := system.GetSystemStats() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + host := capacity.Host{ + CPUCores: float64(hostStats.CPU.Cores), + CPUUsagePercent: hostStats.CPU.UsagePercent, + MemoryTotal: hostStats.Memory.Total, + MemoryAvailable: hostStats.Memory.Available, + } + policy := capacity.PolicyFromConfig(s.config.Capacity) + c.JSON(http.StatusOK, gin.H{ + "host": host, + "policy": policy, + "offer": capacity.FleetOffer(host, policy, s.config.Capacity.OfferToFleet), + }) +} + func (s *Server) getContainerResources(c *gin.Context) { id := c.Param("id") if !s.requireContainerAccess(c, id, auth.AccessLevelRead) { @@ -27,6 +49,51 @@ func (s *Server) getContainerResources(c *gin.Context) { }) } +func (s *Server) diagnoseContainerCapacity(c *gin.Context) { + id := c.Param("id") + if !s.requireContainerAccess(c, id, auth.AccessLevelRead) { + return + } + + stats, err := docker.GetContainerStats(id) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) + return + } + limits, err := docker.GetContainerResources(id) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()}) + return + } + hostStats, err := system.GetSystemStats() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + host := capacity.Host{ + CPUCores: float64(hostStats.CPU.Cores), + CPUUsagePercent: hostStats.CPU.UsagePercent, + MemoryTotal: hostStats.Memory.Total, + MemoryAvailable: hostStats.Memory.Available, + } + container := capacity.Container{ + ID: stats.ContainerID, + Name: stats.Name, + CPUPercent: stats.CPUPercent, + CPULimit: limits.CPUs, + MemoryUsage: stats.MemoryUsage, + MemoryLimit: uint64(max(limits.MemoryLimit, 0)), + } + policy := capacity.PolicyFromConfig(s.config.Capacity) + c.JSON(http.StatusOK, gin.H{ + "host": host, + "container": container, + "policy": policy, + "diagnosis": capacity.Diagnose(host, container, policy), + }) +} + func (s *Server) updateContainerResources(c *gin.Context) { id := c.Param("id") diff --git a/internal/api/server.go b/internal/api/server.go index 554e4f7..e226165 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -28,6 +28,7 @@ import ( "github.com/flatrun/agent/internal/ai" "github.com/flatrun/agent/internal/audit" "github.com/flatrun/agent/internal/auth" + "github.com/flatrun/agent/internal/autoscale" "github.com/flatrun/agent/internal/backup" "github.com/flatrun/agent/internal/certs" "github.com/flatrun/agent/internal/cluster" @@ -40,9 +41,11 @@ import ( "github.com/flatrun/agent/internal/infra" "github.com/flatrun/agent/internal/networks" "github.com/flatrun/agent/internal/notify" + "github.com/flatrun/agent/internal/orchestrator" "github.com/flatrun/agent/internal/plan" "github.com/flatrun/agent/internal/pluginhost" "github.com/flatrun/agent/internal/proxy" + "github.com/flatrun/agent/internal/routing" "github.com/flatrun/agent/internal/scheduler" "github.com/flatrun/agent/internal/security" "github.com/flatrun/agent/internal/setup" @@ -97,8 +100,12 @@ type Server struct { schedulerManager *scheduler.Manager auditManager *audit.Manager auditMiddleware *audit.Middleware + autoscaleStore *autoscale.Store powerDNSManager *dns.PowerDNSManager + clusterMu sync.RWMutex clusterManager *cluster.Manager + probeOrchestrator func(context.Context, orchestrator.ProviderID) error + probeRouting func(context.Context, routing.ProviderID) error setupManager *setup.Manager setupHandlers *setup.Handlers certRenewer *ssl.Renewer @@ -109,6 +116,9 @@ type Server struct { aiAgents *ai.AgentStore mcpHandler http.Handler + runAutoscaleActivation func(context.Context, string) (autoscale.Activation, error) + autoscaleCancel context.CancelFunc + jobs *jobRegistry // runDeploymentAction runs a deployment action and streams each output // line to emit. Overridable in tests so they need not shell out to docker. @@ -177,6 +187,10 @@ func New(cfg *config.Config, configPath string) *Server { manager := docker.NewManager(cfg.DeploymentsPath) manager.SetCleanupTimeout(cfg.Cleanup.Timeout) + autoscaleStore, err := autoscale.NewStore(cfg.DeploymentsPath) + if err != nil { + log.Printf("Warning: Failed to initialize autoscaling store: %v", err) + } // Deploys read template copies from disk. Seed the embedded infra and // welcome content, then pull the app catalog from its external source into @@ -278,7 +292,7 @@ func New(cfg *config.Config, configPath string) *Server { } var trafficManager *traffic.Manager - trafficManager, err := traffic.NewManager(cfg.DeploymentsPath, 7) + trafficManager, err = traffic.NewManager(cfg.DeploymentsPath, 7) if err != nil { log.Printf("Warning: Failed to initialize traffic manager: %v", err) } @@ -325,13 +339,13 @@ func New(cfg *config.Config, configPath string) *Server { requestTimeout = 10 * time.Second } clusterManager = cluster.NewManager(clusterDB, cfg.Cluster.ServerName, healthInterval, requestTimeout, cfg.Auth.JWTSecret) + clusterManager.SetEventPublisher(notifyService) if startErr := clusterManager.Start(context.Background()); startErr != nil { log.Printf("Warning: Failed to start cluster manager: %v", startErr) clusterManager = nil } } } - s := &Server{ config: cfg, configPath: configPath, @@ -362,6 +376,7 @@ func New(cfg *config.Config, configPath string) *Server { backupManager: backupManager, auditManager: auditManager, auditMiddleware: auditMiddleware, + autoscaleStore: autoscaleStore, powerDNSManager: powerDNSManager, clusterManager: clusterManager, setupManager: setupManager, @@ -373,6 +388,20 @@ func New(cfg *config.Config, configPath string) *Server { } s.runDeploymentAction = s.defaultRunDeploymentAction s.runServiceAction = s.defaultRunServiceAction + s.runAutoscaleActivation = s.defaultRunAutoscaleActivation + if err := s.reconcileClusterPeerPolicies(); err != nil { + log.Printf("Warning: Failed to reconcile Fleet peer access: %v", err) + } + if s.autoscaleStore != nil { + autoscaleContext, cancelAutoscale := context.WithCancel(context.Background()) + s.autoscaleCancel = cancelAutoscale + node := cfg.Cluster.ServerName + if node == "" { + node, _ = os.Hostname() + } + supervisor := autoscale.NewSupervisor(s.autoscaleStore, autoscaleRuntimeFactory{server: s}, s.notify, node, 30*time.Second) + go supervisor.Run(autoscaleContext) + } // Built unconditionally: it is stateless and starts nothing, so requests are // gated on the live config flag instead, letting mcp.enabled toggle at runtime. @@ -523,6 +552,9 @@ func (s *Server) setupRoutes() { protected.GET("/agent/update", s.authMiddleware.RequirePermission(auth.PermSettingsRead), s.getAgentUpdate) protected.POST("/agent/update", s.authMiddleware.RequirePermission(auth.PermSettingsWrite), s.triggerAgentUpdate) protected.GET("/notifications/targets", s.authMiddleware.RequirePermission(auth.PermSettingsRead), s.getNotificationTargets) + protected.GET("/notifications/incidents", s.authMiddleware.RequirePermission(auth.PermSettingsRead), s.listNotificationIncidents) + protected.GET("/notifications/rules", s.authMiddleware.RequirePermission(auth.PermSettingsRead), s.listNotificationRules) + protected.PUT("/notifications/rules", s.authMiddleware.RequirePermission(auth.PermSettingsWrite), s.updateNotificationRules) protected.PUT("/notifications/targets", s.authMiddleware.RequirePermission(auth.PermSettingsWrite), s.updateNotificationTargets) protected.POST("/notifications/test", s.authMiddleware.RequirePermission(auth.PermSettingsWrite), s.testNotification) protected.GET("/config", s.authMiddleware.RequirePermission(auth.PermConfigRead), s.listConfig) @@ -605,9 +637,16 @@ func (s *Server) setupRoutes() { protected.GET("/containers/stats", s.authMiddleware.RequirePermission(auth.PermContainersRead), s.getAllContainerStats) protected.POST("/containers/:id/exec", s.authMiddleware.RequirePermission(auth.PermContainersWrite), s.containerExecHTTP) protected.GET("/containers/:id/resources", s.authMiddleware.RequirePermission(auth.PermContainersRead), s.getContainerResources) + protected.GET("/containers/:id/capacity", s.authMiddleware.RequirePermission(auth.PermContainersRead), s.diagnoseContainerCapacity) + protected.GET("/capacity", s.authMiddleware.RequirePermission(auth.PermSystemRead), s.getCapacityStatus) protected.PUT("/containers/:id/resources", s.authMiddleware.RequirePermission(auth.PermContainersWrite), s.updateContainerResources) protected.GET("/deployments/:name/stats", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentContainerStats) protected.GET("/deployments/:name/resources", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentResources) + protected.GET("/deployments/:name/autoscale", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentAutoscalePolicy) + protected.GET("/deployments/:name/autoscale/compatibility", s.authMiddleware.RequirePermission(auth.PermDeploymentsRead), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelRead), s.getDeploymentAutoscaleCompatibility) + protected.PUT("/deployments/:name/autoscale/workload", s.authMiddleware.RequirePermission(auth.PermDeploymentsWrite), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelWrite), s.updateDeploymentAutoscaleWorkload) + protected.PUT("/deployments/:name/autoscale", s.authMiddleware.RequirePermission(auth.PermDeploymentsWrite), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelWrite), s.updateDeploymentAutoscalePolicy) + protected.POST("/deployments/:name/autoscale/activate", s.authMiddleware.RequirePermission(auth.PermDeploymentsWrite), s.authMiddleware.RequireDeploymentAccess(auth.AccessLevelWrite), s.activateDeploymentAutoscale) // Image endpoints protected.GET("/images", s.authMiddleware.RequirePermission(auth.PermImagesRead), s.listImages) @@ -868,17 +907,24 @@ func (s *Server) setupRoutes() { _ = s.firewall.RegisterRoutes(protected) // Cluster endpoints + protected.POST("/cluster/capacity/claim", s.authMiddleware.RequirePermission(auth.PermClusterCapacityClaim), s.clusterCapacityClaim) clusterGroup := protected.Group("/cluster") clusterGroup.Use(s.authMiddleware.RequirePermission(auth.PermClusterRead)) { clusterGroup.GET("/status", s.clusterStatus) + clusterGroup.POST("/setup", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.clusterSetup) clusterGroup.GET("/peers", s.clusterListPeers) + clusterGroup.GET("/peers/:name/policy", s.clusterPeerPolicy) + clusterGroup.PUT("/peers/:name/policy", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.updateClusterPeerPolicy) clusterGroup.POST("/invite", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.clusterInvite) clusterGroup.POST("/accept", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.clusterAccept) clusterGroup.DELETE("/peers/:name", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.clusterRemovePeer) clusterGroup.Any("/peers/:name/proxy/*path", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.clusterProxy) clusterGroup.GET("/deployments", s.clusterAggregateDeployments) clusterGroup.GET("/stats", s.clusterAggregateStats) + clusterGroup.GET("/capacity", s.clusterAggregateCapacity) + clusterGroup.GET("/providers", s.clusterProviders) + clusterGroup.PUT("/providers", s.authMiddleware.RequirePermission(auth.PermClusterWrite), s.updateClusterProviders) } } @@ -887,6 +933,7 @@ func (s *Server) setupRoutes() { // Plugin-emitted notifications (authenticated by the per-run plugin token). api.POST("/internal/notify/emit", s.emitNotification) + api.POST("/internal/events", s.emitEvent) // Log lines and triage for built-in apps, on the same plugin token. Both keep one // implementation in the agent rather than a second one inside every app. api.GET("/internal/logs/stream", s.streamInternalLogs) @@ -944,6 +991,9 @@ func (s *Server) Start() error { } func (s *Server) Stop() error { + if s.autoscaleCancel != nil { + s.autoscaleCancel() + } if s.pluginHost != nil { s.pluginHost.Stop() } @@ -953,6 +1003,9 @@ func (s *Server) Stop() error { if s.clusterManager != nil { s.clusterManager.Stop() } + if s.autoscaleStore != nil { + _ = s.autoscaleStore.Close() + } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() return s.server.Shutdown(ctx) @@ -1995,6 +2048,9 @@ func mergeMetadata(existing, incoming *models.ServiceMetadata, sentFields map[st if _, ok := sentFields["backup"]; ok { merged.Backup = incoming.Backup } + if _, ok := sentFields["scaling"]; ok { + merged.Scaling = incoming.Scaling + } if _, ok := sentFields["protected_mode"]; ok { merged.ProtectedMode = incoming.ProtectedMode } @@ -3410,13 +3466,20 @@ func (s *Server) updateNotificationTargets(c *gin.Context) { func (s *Server) testNotification(c *gin.Context) { var req struct { - URL string `json:"url"` + URL string `json:"url"` + TargetID string `json:"target_id"` } if err := c.ShouldBindJSON(&req); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request"}) return } - if err := s.notify.Test(req.URL); err != nil { + var err error + if req.TargetID != "" { + err = s.notify.TestTarget(req.TargetID) + } else { + err = s.notify.Test(req.URL) + } + if err != nil { c.JSON(http.StatusBadGateway, gin.H{"error": "delivery failed: " + err.Error()}) return } diff --git a/internal/api/server_info_handlers.go b/internal/api/server_info_handlers.go index 9663b05..7ad1fa2 100644 --- a/internal/api/server_info_handlers.go +++ b/internal/api/server_info_handlers.go @@ -2,6 +2,7 @@ package api import ( "net/http" + "strings" "github.com/flatrun/agent/internal/system" "github.com/gin-gonic/gin" @@ -15,12 +16,29 @@ func (s *Server) getServerInfo(c *gin.Context) { }) return } + info.AgentURL = s.agentURL(c) c.JSON(http.StatusOK, gin.H{ "server": info, }) } +func (s *Server) agentURL(c *gin.Context) string { + if configured := strings.TrimRight(strings.TrimSpace(s.config.Cluster.AdvertiseURL), "/"); configured != "" { + return configured + } + + scheme := "http" + if c.Request.TLS != nil { + scheme = "https" + } + host := c.Request.Host + if host == "" { + return "" + } + return scheme + "://" + host +} + func (s *Server) getNetworkHealth(c *gin.Context) { health, err := system.CheckNetworkHealth(c.Request.Context()) if err != nil { diff --git a/internal/api/server_info_handlers_test.go b/internal/api/server_info_handlers_test.go index 41860b2..7fa5b74 100644 --- a/internal/api/server_info_handlers_test.go +++ b/internal/api/server_info_handlers_test.go @@ -101,6 +101,32 @@ func TestGetServerInfo(t *testing.T) { if _, ok := serverMap["interfaces"]; !ok { t.Error("server should have interfaces") } + if got := serverMap["agent_url"]; got != "http://example.com" { + t.Errorf("agent_url = %q, want %q", got, "http://example.com") + } +} + +func TestGetServerInfoIgnoresUntrustedForwardedAgentURL(t *testing.T) { + router, token, cleanup := setupServerInfoTestServer(t) + defer cleanup() + + req := httptest.NewRequest(http.MethodGet, "/api/server/info", nil) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("X-Forwarded-Proto", "https") + req.Header.Set("X-Forwarded-Host", "agent.example.com") + + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + var resp struct { + Server system.ServerInfo `json:"server"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("Failed to parse response: %v", err) + } + if resp.Server.AgentURL != "http://example.com" { + t.Errorf("agent_url = %q, want %q", resp.Server.AgentURL, "http://example.com") + } } func TestGetServerInfoRequiresAuth(t *testing.T) { diff --git a/internal/auth/models.go b/internal/auth/models.go index 7ae4be4..ee83119 100644 --- a/internal/auth/models.go +++ b/internal/auth/models.go @@ -13,6 +13,7 @@ const ( RoleAdmin Role = "admin" RoleOperator Role = "operator" RoleViewer Role = "viewer" + RoleService Role = "service" ) const ( @@ -27,7 +28,7 @@ func ValidAccessLevel(level string) bool { func (r Role) IsValid() bool { switch r { - case RoleAdmin, RoleOperator, RoleViewer: + case RoleAdmin, RoleOperator, RoleViewer, RoleService: return true } return false diff --git a/internal/auth/permissions.go b/internal/auth/permissions.go index 83e6b9e..1a3c31b 100644 --- a/internal/auth/permissions.go +++ b/internal/auth/permissions.go @@ -78,8 +78,9 @@ const ( PermTrafficRead Permission = "traffic:read" PermTrafficWrite Permission = "traffic:write" - PermClusterRead Permission = "cluster:read" - PermClusterWrite Permission = "cluster:write" + PermClusterRead Permission = "cluster:read" + PermClusterWrite Permission = "cluster:write" + PermClusterCapacityClaim Permission = "cluster:capacity:claim" ) var adminPermissions = []Permission{ @@ -104,7 +105,7 @@ var adminPermissions = []Permission{ PermRegistriesRead, PermRegistriesWrite, PermRegistriesDelete, PermTemplatesRead, PermTemplatesWrite, PermTrafficRead, PermTrafficWrite, - PermClusterRead, PermClusterWrite, + PermClusterRead, PermClusterWrite, PermClusterCapacityClaim, } var operatorPermissions = []Permission{ diff --git a/internal/autoscale/activator.go b/internal/autoscale/activator.go new file mode 100644 index 0000000..ccb8bf3 --- /dev/null +++ b/internal/autoscale/activator.go @@ -0,0 +1,105 @@ +package autoscale + +import ( + "context" + "fmt" + "time" + + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" +) + +type ServiceStopper interface { + StopService(string, string) (string, error) +} + +type ServiceStopperFunc func(string, string) (string, error) + +func (f ServiceStopperFunc) StopService(deployment, service string) (string, error) { + return f(deployment, service) +} + +type Activator struct { + orchestrator orchestrator.Provider + routing routing.Provider + stopper ServiceStopper + pollInterval time.Duration + readyTimeout time.Duration +} + +type Activation struct { + Workload orchestrator.Status `json:"workload"` + Route routing.Route `json:"route"` +} + +func NewActivator(orchestratorProvider orchestrator.Provider, routingProvider routing.Provider, stopper ServiceStopper) *Activator { + return &Activator{orchestrator: orchestratorProvider, routing: routingProvider, stopper: stopper, pollInterval: 2 * time.Second, readyTimeout: 2 * time.Minute} +} + +func (a *Activator) Activate(ctx context.Context, deployment, service string, workload orchestrator.Workload, route routing.Route) (Activation, error) { + return a.ActivateDurably(ctx, deployment, service, workload, route, nil) +} + +func (a *Activator) ActivateDurably(ctx context.Context, deployment, service string, workload orchestrator.Workload, route routing.Route, persist func(Activation) error) (Activation, error) { + status, err := a.orchestrator.Apply(ctx, workload) + if err != nil { + return Activation{}, fmt.Errorf("create managed workload: %w", err) + } + rollback := func() { + _ = a.routing.Remove(context.Background(), route.ID) + _ = a.orchestrator.Remove(context.Background(), workload.ID) + } + status, err = a.waitReady(ctx, workload.ID, status) + if err != nil { + rollback() + return Activation{}, err + } + route, err = routeWithReadyInstances(route, status) + if err != nil { + rollback() + return Activation{}, err + } + if err := a.routing.Reconcile(ctx, route); err != nil { + rollback() + return Activation{}, fmt.Errorf("publish managed route: %w", err) + } + activation := Activation{Workload: status, Route: route} + if persist != nil { + if err := persist(activation); err != nil { + rollback() + return Activation{}, fmt.Errorf("save managed workload state: %w", err) + } + } + if _, err := a.stopper.StopService(deployment, service); err != nil { + rollback() + return Activation{}, fmt.Errorf("stop Compose service after cutover: %w", err) + } + return activation, nil +} + +func (a *Activator) waitReady(ctx context.Context, workloadID string, status orchestrator.Status) (orchestrator.Status, error) { + if status.Desired > 0 && status.Available >= status.Desired { + return status, nil + } + timer := time.NewTimer(a.readyTimeout) + defer timer.Stop() + ticker := time.NewTicker(a.pollInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return status, ctx.Err() + case <-timer.C: + return status, fmt.Errorf("managed workload did not become ready within %s", a.readyTimeout) + case <-ticker.C: + var err error + status, err = a.orchestrator.Status(ctx, workloadID) + if err != nil { + return status, fmt.Errorf("check managed workload readiness: %w", err) + } + if status.Desired > 0 && status.Available >= status.Desired { + return status, nil + } + } + } +} diff --git a/internal/autoscale/activator_test.go b/internal/autoscale/activator_test.go new file mode 100644 index 0000000..11ef8bd --- /dev/null +++ b/internal/autoscale/activator_test.go @@ -0,0 +1,73 @@ +package autoscale + +import ( + "context" + "errors" + "testing" + + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" +) + +type activationStopper struct { + stopped bool + err error +} + +func (s *activationStopper) StopService(_, _ string) (string, error) { + s.stopped = true + return "", s.err +} + +func TestActivatorCutsOverOnlyAfterManagedReplicasAreReady(t *testing.T) { + provider := &fakeOrchestrator{status: orchestrator.Status{Workload: "shop", Desired: 2, Available: 2, Instances: []orchestrator.Instance{ + {ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}, + {ID: "two", Address: "10.0.0.2:8080", Healthy: true, Ready: true}, + }}} + router := &fakeRouter{} + stopper := &activationStopper{} + activation, err := NewActivator(provider, router, stopper).Activate(context.Background(), "shop", "web", orchestrator.Workload{ID: "shop", Image: "shop:1", Replicas: 2}, routing.Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http"}) + if err != nil { + t.Fatal(err) + } + if !stopper.stopped || len(router.reconciled.Backends) != 2 || activation.Workload.Available != 2 { + t.Fatalf("activation = %#v, route = %#v", activation, router.reconciled) + } +} + +func TestActivatorRollsBackWhenComposeCannotStop(t *testing.T) { + provider := &fakeOrchestrator{status: orchestrator.Status{Workload: "shop", Desired: 1, Available: 1, Instances: []orchestrator.Instance{{ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}}}} + router := &fakeRouter{} + stopper := &activationStopper{err: errors.New("compose failed")} + _, err := NewActivator(provider, router, stopper).Activate(context.Background(), "shop", "web", orchestrator.Workload{ID: "shop", Image: "shop:1", Replicas: 1}, routing.Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http"}) + if err == nil { + t.Fatal("activation succeeded") + } + if !stopper.stopped { + t.Fatal("Compose stop was not attempted") + } + if provider.removed != "shop" || router.removed != "shop" { + t.Fatalf("rollback removed workload %q and route %q", provider.removed, router.removed) + } +} + +func TestActivatorPersistsBeforeStoppingCompose(t *testing.T) { + provider := &fakeOrchestrator{status: orchestrator.Status{Workload: "shop", Desired: 1, Available: 1, Instances: []orchestrator.Instance{{ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}}}} + router := &fakeRouter{} + stopper := &activationStopper{} + persisted := false + _, err := NewActivator(provider, router, stopper).ActivateDurably( + context.Background(), "shop", "web", orchestrator.Workload{ID: "shop", Image: "shop:1", Replicas: 1}, + routing.Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http"}, + func(activation Activation) error { + persisted = activation.Workload.Available == 1 && len(activation.Route.Backends) == 1 + return errors.New("database unavailable") + }, + ) + if err == nil || !persisted { + t.Fatalf("error = %v, persisted = %t", err, persisted) + } + if stopper.stopped || provider.removed != "shop" || router.removed != "shop" { + t.Fatalf("stopped = %t, workload = %q, route = %q", stopper.stopped, provider.removed, router.removed) + } +} diff --git a/internal/autoscale/compatibility.go b/internal/autoscale/compatibility.go new file mode 100644 index 0000000..4357d8e --- /dev/null +++ b/internal/autoscale/compatibility.go @@ -0,0 +1,242 @@ +package autoscale + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/docker/go-units" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/pkg/models" + "gopkg.in/yaml.v3" +) + +type Compatibility struct { + Compatible bool `json:"compatible"` + Service string `json:"service,omitempty"` + Image string `json:"image,omitempty"` + Blockers []string `json:"blockers"` + Warnings []string `json:"warnings"` + Services []string `json:"services"` + Workload *models.ScalingConfig `json:"workload,omitempty"` +} + +type composeCompatibilityFile struct { + Services map[string]composeCompatibilityService `yaml:"services"` +} + +type composeCompatibilityService struct { + Image string `yaml:"image"` + Build any `yaml:"build"` + Volumes []any `yaml:"volumes"` + Configs []any `yaml:"configs"` + Secrets []any `yaml:"secrets"` + Devices []any `yaml:"devices"` + Privileged bool `yaml:"privileged"` + NetworkMode string `yaml:"network_mode"` + DependsOn map[string]any `yaml:"depends_on"` + EnvFile any `yaml:"env_file"` + Environment any `yaml:"environment"` + Entrypoint any `yaml:"entrypoint"` + Command any `yaml:"command"` + WorkingDir string `yaml:"working_dir"` + Deploy composeDeploy `yaml:"deploy"` +} + +type composeDeploy struct { + Resources composeResources `yaml:"resources"` +} + +type composeResources struct { + Limits composeResourceValues `yaml:"limits"` + Reservations composeResourceValues `yaml:"reservations"` +} + +type composeResourceValues struct { + CPUs any `yaml:"cpus"` + Memory any `yaml:"memory"` +} + +func AssessCompatibility(deployment *models.Deployment, composeContent string) Compatibility { + result := Compatibility{Blockers: []string{}, Warnings: []string{}} + if deployment == nil || deployment.Metadata == nil || deployment.Metadata.Scaling == nil { + result.Blockers = append(result.Blockers, "Add a scaling declaration to service.yml") + return result + } + scaling := deployment.Metadata.Scaling + result.Workload = scaling + result.Service = strings.TrimSpace(scaling.Service) + if result.Service == "" { + result.Blockers = append(result.Blockers, "Choose the Compose service that may scale") + } + if !scaling.Stateless { + result.Blockers = append(result.Blockers, "Only workloads declared stateless can scale across servers") + } + mode := strings.TrimSpace(scaling.Storage.Mode) + if mode == "" { + mode = "none" + } + if mode != "none" && mode != "shared" { + result.Blockers = append(result.Blockers, "Storage mode must be none or shared") + } + if mode == "shared" && strings.TrimSpace(scaling.Storage.Class) == "" { + result.Blockers = append(result.Blockers, "Shared storage requires a storage class") + } + if mode == "shared" { + result.Blockers = append(result.Blockers, "Shared storage needs an installed Fleet storage adapter") + } + + var compose composeCompatibilityFile + if err := yaml.Unmarshal([]byte(composeContent), &compose); err != nil { + result.Blockers = append(result.Blockers, fmt.Sprintf("Compose configuration cannot be read: %v", err)) + return result + } + for name := range compose.Services { + result.Services = append(result.Services, name) + } + sort.Strings(result.Services) + service, exists := compose.Services[result.Service] + if !exists && result.Service != "" { + result.Blockers = append(result.Blockers, fmt.Sprintf("Compose service %q does not exist", result.Service)) + return result + } + result.Image = strings.TrimSpace(service.Image) + if result.Image == "" { + result.Blockers = append(result.Blockers, "The scale-ready service needs a portable image") + } + if len(service.Volumes) > 0 && mode != "shared" { + result.Blockers = append(result.Blockers, "The service uses volumes but shared storage is not declared") + } + if len(service.Configs) > 0 || len(service.Secrets) > 0 { + result.Blockers = append(result.Blockers, "Compose configs and secrets need a Fleet distribution policy") + } + if service.EnvFile != nil { + result.Blockers = append(result.Blockers, "Environment files must be converted to inline deployment environment values") + } + if len(service.Devices) > 0 || service.Privileged || service.NetworkMode != "" { + result.Blockers = append(result.Blockers, "Host-specific container access cannot move between Fleet servers") + } + if service.Build != nil && result.Image != "" { + result.Warnings = append(result.Warnings, "Publish the declared image before Fleet places replicas on another server") + } + if len(service.DependsOn) > 0 { + result.Warnings = append(result.Warnings, "Dependencies must be reachable from every server allowed to run this workload") + } + result.Compatible = len(result.Blockers) == 0 + return result +} + +func BuildWorkload(deployment *models.Deployment, composeContent string, replicas int, proxyNetwork string) (orchestrator.Workload, error) { + compatibility := AssessCompatibility(deployment, composeContent) + if !compatibility.Compatible { + return orchestrator.Workload{}, fmt.Errorf("workload is not scale-ready: %s", strings.Join(compatibility.Blockers, "; ")) + } + var compose composeCompatibilityFile + if err := yaml.Unmarshal([]byte(composeContent), &compose); err != nil { + return orchestrator.Workload{}, err + } + service := compose.Services[compatibility.Service] + workload := orchestrator.Workload{ + ID: deployment.Name, Image: service.Image, Replicas: replicas, + Environment: map[string]string{}, Entrypoint: stringList(service.Entrypoint), Command: stringList(service.Command), + WorkingDir: service.WorkingDir, Labels: map[string]string{"flatrun.deployment": deployment.Name}, + } + parsedResources, err := workloadResources(service.Deploy.Resources) + if err != nil { + return orchestrator.Workload{}, err + } + workload.Resources = parsedResources + if strings.TrimSpace(proxyNetwork) != "" { + workload.Networks = []string{proxyNetwork} + } + workload.Environment = environmentMap(service.Environment) + for _, domain := range deployment.Metadata.GetDomains() { + if domain.Service == compatibility.Service { + workload.Port = domain.ContainerPort + break + } + } + workload.Health.Path = deployment.Metadata.HealthCheck.Path + return workload, nil +} + +func workloadResources(resources composeResources) (orchestrator.Resources, error) { + cpuLimit, err := cpuValue(resources.Limits.CPUs) + if err != nil { + return orchestrator.Resources{}, fmt.Errorf("invalid CPU limit: %w", err) + } + cpuRequest, err := cpuValue(resources.Reservations.CPUs) + if err != nil { + return orchestrator.Resources{}, fmt.Errorf("invalid CPU reservation: %w", err) + } + memoryLimit, err := memoryValue(resources.Limits.Memory) + if err != nil { + return orchestrator.Resources{}, fmt.Errorf("invalid memory limit: %w", err) + } + memoryRequest, err := memoryValue(resources.Reservations.Memory) + if err != nil { + return orchestrator.Resources{}, fmt.Errorf("invalid memory reservation: %w", err) + } + return orchestrator.Resources{CPURequest: cpuRequest, CPULimit: cpuLimit, MemoryRequest: memoryRequest, MemoryLimit: memoryLimit}, nil +} + +func cpuValue(value any) (float64, error) { + if value == nil { + return 0, nil + } + parsed, err := strconv.ParseFloat(strings.TrimSpace(fmt.Sprint(value)), 64) + if err != nil || parsed < 0 { + return 0, fmt.Errorf("%q", value) + } + return parsed, nil +} + +func memoryValue(value any) (uint64, error) { + if value == nil { + return 0, nil + } + parsed, err := units.RAMInBytes(strings.TrimSpace(fmt.Sprint(value))) + if err != nil || parsed < 0 { + return 0, fmt.Errorf("%q", value) + } + return uint64(parsed), nil +} + +func environmentMap(value any) map[string]string { + result := map[string]string{} + switch typed := value.(type) { + case map[string]any: + for key, item := range typed { + if item != nil { + result[key] = fmt.Sprint(item) + } + } + case []any: + for _, item := range typed { + parts := strings.SplitN(fmt.Sprint(item), "=", 2) + if len(parts) == 2 { + result[parts[0]] = parts[1] + } + } + } + return result +} + +func stringList(value any) []string { + switch typed := value.(type) { + case string: + if strings.TrimSpace(typed) == "" { + return nil + } + return []string{"/bin/sh", "-c", typed} + case []any: + result := make([]string, 0, len(typed)) + for _, item := range typed { + result = append(result, fmt.Sprint(item)) + } + return result + default: + return nil + } +} diff --git a/internal/autoscale/compatibility_test.go b/internal/autoscale/compatibility_test.go new file mode 100644 index 0000000..4d1fa36 --- /dev/null +++ b/internal/autoscale/compatibility_test.go @@ -0,0 +1,63 @@ +package autoscale + +import ( + "testing" + + "github.com/flatrun/agent/pkg/models" +) + +func TestAssessCompatibilityAcceptsDeclaredStatelessImage(t *testing.T) { + deployment := &models.Deployment{Metadata: &models.ServiceMetadata{Scaling: &models.ScalingConfig{Service: "web", Stateless: true}}} + result := AssessCompatibility(deployment, "services:\n web:\n image: registry.example.com/shop:1\n") + if !result.Compatible || result.Service != "web" || result.Image != "registry.example.com/shop:1" { + t.Fatalf("result = %#v", result) + } +} + +func TestAssessCompatibilityExplainsUnsafeComposeFeatures(t *testing.T) { + deployment := &models.Deployment{Metadata: &models.ServiceMetadata{Scaling: &models.ScalingConfig{Service: "web", Stateless: true}}} + result := AssessCompatibility(deployment, "services:\n web:\n image: shop:1\n privileged: true\n volumes:\n - ./data:/data\n") + if result.Compatible || len(result.Blockers) != 2 { + t.Fatalf("result = %#v", result) + } +} + +func TestBuildWorkloadCarriesPortableRuntimeInputs(t *testing.T) { + deployment := &models.Deployment{Name: "shop", Metadata: &models.ServiceMetadata{ + Scaling: &models.ScalingConfig{Service: "web", Stateless: true}, + Domains: []models.DomainConfig{{Service: "web", ContainerPort: 8080, Domain: "shop.example.com"}}, + HealthCheck: models.HealthCheckConfig{Path: "/ready"}, + }} + workload, err := BuildWorkload(deployment, `services: + web: + image: registry.example.com/shop:1 + environment: + APP_ENV: production + entrypoint: ["/app/entrypoint"] + command: ["serve", "--port", "8080"] + working_dir: /app + deploy: + resources: + limits: + cpus: "1.5" + memory: 512M + reservations: + cpus: "0.5" + memory: 256M +`, 2, "proxy") + if err != nil { + t.Fatal(err) + } + if workload.Replicas != 2 || workload.Port != 8080 || workload.Environment["APP_ENV"] != "production" || workload.WorkingDir != "/app" || workload.Health.Path != "/ready" { + t.Fatalf("workload = %#v", workload) + } + if len(workload.Entrypoint) != 1 || len(workload.Command) != 3 { + t.Fatalf("workload commands = %#v / %#v", workload.Entrypoint, workload.Command) + } + if len(workload.Networks) != 1 || workload.Networks[0] != "proxy" { + t.Fatalf("workload networks = %#v", workload.Networks) + } + if workload.Resources.CPULimit != 1.5 || workload.Resources.CPURequest != 0.5 || workload.Resources.MemoryLimit != 512<<20 || workload.Resources.MemoryRequest != 256<<20 { + t.Fatalf("workload resources = %#v", workload.Resources) + } +} diff --git a/internal/autoscale/controller.go b/internal/autoscale/controller.go new file mode 100644 index 0000000..56c2044 --- /dev/null +++ b/internal/autoscale/controller.go @@ -0,0 +1,146 @@ +package autoscale + +import ( + "fmt" + "math" + "time" + + "github.com/flatrun/agent/internal/capacity" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" +) + +type Action string + +const ( + ActionNone Action = "none" + ActionIncreaseCPU Action = "increase_cpu" + ActionIncreaseMemory Action = "increase_memory" + ActionAddReplica Action = "add_replica" + ActionRemoveReplica Action = "remove_replica" + ActionNotify Action = "notify" +) + +type Policy struct { + Enabled bool `json:"enabled"` + MinReplicas int `json:"min_replicas"` + MaxReplicas int `json:"max_replicas"` + ScaleUpPercent float64 `json:"scale_up_percent"` + ScaleDownPercent float64 `json:"scale_down_percent"` + ScaleUpWindows int `json:"scale_up_windows"` + ScaleDownWindows int `json:"scale_down_windows"` + Cooldown time.Duration `json:"cooldown"` + AllowFleetCapacity bool `json:"allow_fleet_capacity"` +} + +type State struct { + HighWindows int `json:"high_windows"` + LowWindows int `json:"low_windows"` + LastAction time.Time `json:"last_action,omitempty"` + Active bool `json:"active"` + Provider orchestrator.ProviderID `json:"provider,omitempty"` + Service string `json:"service,omitempty"` + Replicas int `json:"replicas,omitempty"` + Route routing.Route `json:"route,omitempty"` +} + +type Input struct { + Now time.Time + Replicas int + CPUPercent float64 + MemoryPercent float64 + Diagnosis capacity.Diagnosis + FleetOffer capacity.Offer + RequiresFleet bool + CurrentResources orchestrator.Resources + SuggestedResource orchestrator.Resources +} + +type Decision struct { + Action Action `json:"action"` + Replicas int `json:"replicas,omitempty"` + Resources orchestrator.Resources `json:"resources,omitempty"` + Reason string `json:"reason"` +} + +func DefaultPolicy() Policy { + return Policy{ + Enabled: true, MinReplicas: 1, MaxReplicas: 3, + ScaleUpPercent: 80, ScaleDownPercent: 30, + ScaleUpWindows: 3, ScaleDownWindows: 10, + Cooldown: 5 * time.Minute, + } +} + +func Reconcile(policy Policy, state State, input Input) (State, Decision) { + if err := ValidatePolicy(policy); err != nil { + return state, Decision{Action: ActionNotify, Reason: err.Error()} + } + if !policy.Enabled { + state.HighWindows = 0 + state.LowWindows = 0 + return state, Decision{Action: ActionNone, Reason: "Autoscaling is disabled"} + } + if input.Now.IsZero() { + input.Now = time.Now() + } + if input.Replicas < policy.MinReplicas { + return acted(state, input.Now), Decision{Action: ActionAddReplica, Replicas: policy.MinReplicas, Reason: "Replica count is below policy minimum"} + } + if !state.LastAction.IsZero() && input.Now.Sub(state.LastAction) < policy.Cooldown { + return state, Decision{Action: ActionNone, Reason: "Autoscaling is in cooldown"} + } + + switch input.Diagnosis.Action { + case capacity.ActionIncreaseCPU: + return acted(state, input.Now), Decision{Action: ActionIncreaseCPU, Resources: input.SuggestedResource, Reason: input.Diagnosis.Reason} + case capacity.ActionIncreaseMemory: + return acted(state, input.Now), Decision{Action: ActionIncreaseMemory, Resources: input.SuggestedResource, Reason: input.Diagnosis.Reason} + } + + utilization := math.Max(input.CPUPercent, input.MemoryPercent) + if utilization >= policy.ScaleUpPercent { + state.HighWindows++ + state.LowWindows = 0 + } else if utilization <= policy.ScaleDownPercent { + state.LowWindows++ + state.HighWindows = 0 + } else { + state.HighWindows = 0 + state.LowWindows = 0 + } + + if state.HighWindows >= policy.ScaleUpWindows { + if input.Replicas >= policy.MaxReplicas { + return state, Decision{Action: ActionNotify, Reason: "Workload has reached its replica limit"} + } + if input.RequiresFleet && (!policy.AllowFleetCapacity || !input.FleetOffer.Enabled) { + return state, Decision{Action: ActionNotify, Reason: "No permitted fleet capacity is available"} + } + return acted(state, input.Now), Decision{Action: ActionAddReplica, Replicas: input.Replicas + 1, Reason: "Resource pressure remained above the scale-up threshold"} + } + if state.LowWindows >= policy.ScaleDownWindows && input.Replicas > policy.MinReplicas { + return acted(state, input.Now), Decision{Action: ActionRemoveReplica, Replicas: input.Replicas - 1, Reason: "Resource use remained below the scale-down threshold"} + } + return state, Decision{Action: ActionNone, Reason: "No scaling threshold has been sustained"} +} + +func ValidatePolicy(policy Policy) error { + if policy.MinReplicas < 1 || policy.MaxReplicas < policy.MinReplicas { + return fmt.Errorf("Replica limits are invalid") + } + if policy.ScaleUpPercent <= policy.ScaleDownPercent || policy.ScaleUpPercent > 100 || policy.ScaleDownPercent < 0 { + return fmt.Errorf("Scaling thresholds are invalid") + } + if policy.ScaleUpWindows < 1 || policy.ScaleDownWindows < 1 || policy.Cooldown < 0 { + return fmt.Errorf("Scaling timing is invalid") + } + return nil +} + +func acted(state State, now time.Time) State { + state.HighWindows = 0 + state.LowWindows = 0 + state.LastAction = now + return state +} diff --git a/internal/autoscale/controller_test.go b/internal/autoscale/controller_test.go new file mode 100644 index 0000000..45ce745 --- /dev/null +++ b/internal/autoscale/controller_test.go @@ -0,0 +1,89 @@ +package autoscale + +import ( + "testing" + "time" + + "github.com/flatrun/agent/internal/capacity" + "github.com/flatrun/agent/internal/orchestrator" +) + +func TestReconcileIncreasesLocalAllocationBeforeReplicas(t *testing.T) { + policy := DefaultPolicy() + now := time.Now() + state, decision := Reconcile(policy, State{HighWindows: policy.ScaleUpWindows - 1}, Input{ + Now: now, Replicas: 1, CPUPercent: 95, + Diagnosis: capacity.Diagnosis{Action: capacity.ActionIncreaseCPU, Reason: "Host has headroom"}, + SuggestedResource: orchestrator.Resources{CPULimit: 2}, + }) + + if decision.Action != ActionIncreaseCPU || decision.Resources.CPULimit != 2 { + t.Fatalf("decision = %#v", decision) + } + if !state.LastAction.Equal(now) || state.HighWindows != 0 { + t.Fatalf("state = %#v", state) + } +} + +func TestReconcileRequiresSustainedPressureToAddReplica(t *testing.T) { + policy := DefaultPolicy() + now := time.Now() + state := State{} + for window := 1; window <= policy.ScaleUpWindows; window++ { + var decision Decision + state, decision = Reconcile(policy, state, Input{Now: now, Replicas: 1, CPUPercent: 90}) + if window < policy.ScaleUpWindows && decision.Action != ActionNone { + t.Fatalf("window %d decision = %#v", window, decision) + } + if window == policy.ScaleUpWindows && (decision.Action != ActionAddReplica || decision.Replicas != 2) { + t.Fatalf("window %d decision = %#v", window, decision) + } + } +} + +func TestReconcileRequiresPermittedFleetCapacity(t *testing.T) { + policy := DefaultPolicy() + policy.AllowFleetCapacity = true + state := State{HighWindows: policy.ScaleUpWindows - 1} + _, decision := Reconcile(policy, state, Input{ + Now: time.Now(), Replicas: 1, CPUPercent: 95, RequiresFleet: true, + FleetOffer: capacity.Offer{Enabled: false}, + }) + if decision.Action != ActionNotify || decision.Reason != "No permitted fleet capacity is available" { + t.Fatalf("decision = %#v", decision) + } +} + +func TestReconcileRemovesReplicaAfterSustainedRecovery(t *testing.T) { + policy := DefaultPolicy() + state := State{LowWindows: policy.ScaleDownWindows - 1} + _, decision := Reconcile(policy, state, Input{Now: time.Now(), Replicas: 3, CPUPercent: 10, MemoryPercent: 20}) + if decision.Action != ActionRemoveReplica || decision.Replicas != 2 { + t.Fatalf("decision = %#v", decision) + } +} + +func TestReconcileHonorsCooldown(t *testing.T) { + policy := DefaultPolicy() + now := time.Now() + state := State{HighWindows: policy.ScaleUpWindows - 1, LastAction: now.Add(-time.Minute)} + unchanged, decision := Reconcile(policy, state, Input{Now: now, Replicas: 1, CPUPercent: 99}) + if decision.Action != ActionNone || unchanged.HighWindows != state.HighWindows { + t.Fatalf("state = %#v, decision = %#v", unchanged, decision) + } +} + +func TestReconcileDisabledPreservesManagedWorkloadState(t *testing.T) { + policy := DefaultPolicy() + policy.Enabled = false + state, decision := Reconcile(policy, State{ + HighWindows: 2, LowWindows: 3, Active: true, + Provider: orchestrator.ProviderSwarm, Service: "web", Replicas: 2, + }, Input{}) + if decision.Action != ActionNone || state.HighWindows != 0 || state.LowWindows != 0 { + t.Fatalf("state = %#v, decision = %#v", state, decision) + } + if !state.Active || state.Provider != orchestrator.ProviderSwarm || state.Service != "web" || state.Replicas != 2 { + t.Fatalf("runtime state was cleared: %#v", state) + } +} diff --git a/internal/autoscale/executor.go b/internal/autoscale/executor.go new file mode 100644 index 0000000..a64f199 --- /dev/null +++ b/internal/autoscale/executor.go @@ -0,0 +1,120 @@ +package autoscale + +import ( + "context" + "fmt" + "slices" + + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" +) + +type Executor struct { + orchestrator orchestrator.Provider + routing routing.Provider +} + +type Execution struct { + Decision Decision `json:"decision"` + Status orchestrator.Status `json:"status"` + Route routing.Route `json:"route,omitempty"` + Pending bool `json:"pending"` +} + +func NewExecutor(orchestrator orchestrator.Provider, routing routing.Provider) *Executor { + return &Executor{orchestrator: orchestrator, routing: routing} +} + +func (e *Executor) Execute(ctx context.Context, workloadID string, route routing.Route, decision Decision) (Execution, error) { + result := Execution{Decision: decision} + switch decision.Action { + case ActionNone: + status, err := e.orchestrator.Status(ctx, workloadID) + result.Status = status + if err != nil || status.Available < status.Desired { + result.Pending = err == nil + return result, err + } + updated, err := routeWithReadyInstances(route, status) + if err != nil { + return result, err + } + if slices.Equal(updated.Backends, route.Backends) { + return result, nil + } + if err := e.routing.Reconcile(ctx, updated); err != nil { + return result, fmt.Errorf("publish ready route: %w", err) + } + result.Route = updated + return result, nil + case ActionNotify: + status, err := e.orchestrator.Status(ctx, workloadID) + result.Status = status + return result, err + case ActionIncreaseCPU, ActionIncreaseMemory: + status, err := e.orchestrator.Resize(ctx, workloadID, decision.Resources) + result.Status = status + return result, err + case ActionAddReplica: + status, err := e.orchestrator.Scale(ctx, workloadID, decision.Replicas) + if err != nil { + return result, err + } + result.Status = status + if status.Available < status.Desired { + result.Pending = true + return result, nil + } + updated, err := routeWithReadyInstances(route, status) + if err != nil { + return result, err + } + if err := e.routing.Reconcile(ctx, updated); err != nil { + return result, fmt.Errorf("publish scaled route: %w", err) + } + result.Route = updated + return result, nil + case ActionRemoveReplica: + status, err := e.orchestrator.Scale(ctx, workloadID, decision.Replicas) + result.Status = status + if err == nil && status.Available != status.Desired { + result.Pending = true + return result, nil + } + if err == nil && e.routing.ID() == routing.ProviderTraefik { + result.Route = route + return result, nil + } + if err == nil { + updated, routeErr := routeWithReadyInstances(route, status) + if routeErr != nil { + return result, routeErr + } + if routeErr = e.routing.Reconcile(ctx, updated); routeErr != nil { + return result, fmt.Errorf("publish scaled route: %w", routeErr) + } + result.Route = updated + } + return result, err + default: + return result, fmt.Errorf("Unknown autoscaling action %q", decision.Action) + } +} + +func routeWithReadyInstances(route routing.Route, status orchestrator.Status) (routing.Route, error) { + backends := make([]routing.Backend, 0, len(status.Instances)) + for _, instance := range status.Instances { + if !instance.Ready || !instance.Healthy { + continue + } + if instance.Address == "" { + return route, fmt.Errorf("Ready instance %q has no routable address", instance.ID) + } + backends = append(backends, routing.Backend{ID: instance.ID, Address: instance.Address, Healthy: true, Weight: 1}) + } + if len(backends) != status.Desired { + return route, fmt.Errorf("Only %d of %d replicas are routable", len(backends), status.Desired) + } + route.Backends = backends + return route, nil +} diff --git a/internal/autoscale/executor_test.go b/internal/autoscale/executor_test.go new file mode 100644 index 0000000..bc4002e --- /dev/null +++ b/internal/autoscale/executor_test.go @@ -0,0 +1,148 @@ +package autoscale + +import ( + "context" + "testing" + + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" +) + +type fakeOrchestrator struct { + status orchestrator.Status + scaledTo int + resizedWith orchestrator.Resources + removed string +} + +func (f *fakeOrchestrator) ID() orchestrator.ProviderID { return orchestrator.ProviderSwarm } +func (f *fakeOrchestrator) Validate(context.Context, orchestrator.Workload) error { return nil } +func (f *fakeOrchestrator) Apply(context.Context, orchestrator.Workload) (orchestrator.Status, error) { + return f.status, nil +} +func (f *fakeOrchestrator) Resize(_ context.Context, _ string, resources orchestrator.Resources) (orchestrator.Status, error) { + f.resizedWith = resources + return f.status, nil +} +func (f *fakeOrchestrator) Scale(_ context.Context, _ string, replicas int) (orchestrator.Status, error) { + f.scaledTo = replicas + f.status.Desired = replicas + if len(f.status.Instances) > replicas { + f.status.Instances = f.status.Instances[:replicas] + f.status.Available = replicas + } + return f.status, nil +} +func (f *fakeOrchestrator) Status(context.Context, string) (orchestrator.Status, error) { + return f.status, nil +} +func (f *fakeOrchestrator) Remove(_ context.Context, id string) error { + f.removed = id + return nil +} + +type fakeRouter struct { + reconciled routing.Route + drained string + removed string +} + +func (f *fakeRouter) ID() routing.ProviderID { return routing.ProviderNginx } +func (f *fakeRouter) Validate(context.Context, routing.Route) error { return nil } +func (f *fakeRouter) Reconcile(_ context.Context, route routing.Route) error { + f.reconciled = route + return nil +} +func (f *fakeRouter) Drain(_ context.Context, _, backendID string) error { + f.drained = backendID + return nil +} +func (f *fakeRouter) Remove(_ context.Context, id string) error { + f.removed = id + return nil +} + +func TestExecutorPublishesOnlyReadyScaledReplicas(t *testing.T) { + orchestratorProvider := &fakeOrchestrator{status: orchestrator.Status{ + Workload: "shop", Desired: 1, Available: 2, + Instances: []orchestrator.Instance{ + {ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}, + {ID: "two", Address: "10.0.0.2:8080", Healthy: true, Ready: true}, + }, + }} + router := &fakeRouter{} + execution, err := NewExecutor(orchestratorProvider, router).Execute( + context.Background(), + "shop", + routing.Route{ID: "shop", Domain: "shop.example.com", Protocol: "http"}, + Decision{Action: ActionAddReplica, Replicas: 2}, + ) + if err != nil { + t.Fatalf("Execute failed: %v", err) + } + if execution.Pending || orchestratorProvider.scaledTo != 2 || len(router.reconciled.Backends) != 2 { + t.Fatalf("execution = %#v, route = %#v", execution, router.reconciled) + } +} + +func TestExecutorWaitsForNewReplicaBeforeRouting(t *testing.T) { + orchestratorProvider := &fakeOrchestrator{status: orchestrator.Status{ + Workload: "shop", Desired: 1, Available: 1, + Instances: []orchestrator.Instance{{ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}}, + }} + router := &fakeRouter{} + execution, err := NewExecutor(orchestratorProvider, router).Execute( + context.Background(), "shop", routing.Route{ID: "shop"}, Decision{Action: ActionAddReplica, Replicas: 2}, + ) + if err != nil { + t.Fatalf("Execute failed: %v", err) + } + if !execution.Pending || len(router.reconciled.Backends) != 0 { + t.Fatalf("execution = %#v, route = %#v", execution, router.reconciled) + } +} + +func TestExecutorPublishesReplicaThatBecameReady(t *testing.T) { + orchestratorProvider := &fakeOrchestrator{status: orchestrator.Status{ + Workload: "shop", Desired: 2, Available: 2, + Instances: []orchestrator.Instance{ + {ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}, + {ID: "two", Address: "10.0.0.2:8080", Healthy: true, Ready: true}, + }, + }} + router := &fakeRouter{} + execution, err := NewExecutor(orchestratorProvider, router).Execute( + context.Background(), "shop", + routing.Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http", Backends: []routing.Backend{{ID: "one", Address: "10.0.0.1:8080", Healthy: true, Weight: 1}}}, + Decision{Action: ActionNone}, + ) + if err != nil { + t.Fatalf("Execute failed: %v", err) + } + if execution.Pending || len(execution.Route.Backends) != 2 || len(router.reconciled.Backends) != 2 { + t.Fatalf("execution = %#v, route = %#v", execution, router.reconciled) + } +} + +func TestExecutorRoutesTheReplicaSetChosenByTheOrchestrator(t *testing.T) { + orchestratorProvider := &fakeOrchestrator{status: orchestrator.Status{ + Workload: "shop", Desired: 2, Available: 2, + Instances: []orchestrator.Instance{ + {ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}, + {ID: "two", Address: "10.0.0.2:8080", Healthy: true, Ready: true}, + }, + }} + router := &fakeRouter{} + _, err := NewExecutor(orchestratorProvider, router).Execute( + context.Background(), + "shop", + routing.Route{ID: "shop", Backends: []routing.Backend{{ID: "one"}, {ID: "two"}}}, + Decision{Action: ActionRemoveReplica, Replicas: 1}, + ) + if err != nil { + t.Fatalf("Execute failed: %v", err) + } + if router.drained != "" || orchestratorProvider.scaledTo != 1 || len(router.reconciled.Backends) != 1 { + t.Fatalf("drained = %q, scaled = %d", router.drained, orchestratorProvider.scaledTo) + } +} diff --git a/internal/autoscale/migrations/00001_initial.sql b/internal/autoscale/migrations/00001_initial.sql new file mode 100644 index 0000000..fc72550 --- /dev/null +++ b/internal/autoscale/migrations/00001_initial.sql @@ -0,0 +1,16 @@ +-- +goose Up +CREATE TABLE autoscale_policies ( + deployment TEXT PRIMARY KEY, + policy_json TEXT NOT NULL, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE autoscale_states ( + deployment TEXT PRIMARY KEY, + state_json TEXT NOT NULL, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +-- +goose Down +DROP TABLE autoscale_states; +DROP TABLE autoscale_policies; diff --git a/internal/autoscale/runner.go b/internal/autoscale/runner.go new file mode 100644 index 0000000..11eb57f --- /dev/null +++ b/internal/autoscale/runner.go @@ -0,0 +1,94 @@ +package autoscale + +import ( + "context" + "fmt" + "time" + + "github.com/flatrun/agent/internal/events" + "github.com/flatrun/agent/internal/routing" +) + +type PolicyStore interface { + Policy(string) (Policy, error) + State(string) (State, error) + SetState(string, State) error +} + +type ActionExecutor interface { + Execute(context.Context, string, routing.Route, Decision) (Execution, error) +} + +type EventPublisher interface { + Publish(events.Event) (events.IngestResult, error) +} + +type Runner struct { + store PolicyStore + executor ActionExecutor + publisher EventPublisher + node string +} + +type ReconcileResult struct { + State State `json:"state"` + Decision Decision `json:"decision"` + Execution *Execution `json:"execution,omitempty"` +} + +func NewRunner(store PolicyStore, executor ActionExecutor, publisher EventPublisher, node string) *Runner { + return &Runner{store: store, executor: executor, publisher: publisher, node: node} +} + +func (r *Runner) Reconcile(ctx context.Context, deployment string, input Input, route routing.Route) (ReconcileResult, error) { + policy, err := r.store.Policy(deployment) + if err != nil { + return ReconcileResult{}, fmt.Errorf("load autoscaling policy: %w", err) + } + state, err := r.store.State(deployment) + if err != nil { + return ReconcileResult{}, fmt.Errorf("load autoscaling state: %w", err) + } + nextState, decision := Reconcile(policy, state, input) + if err := r.store.SetState(deployment, nextState); err != nil { + return ReconcileResult{}, fmt.Errorf("save autoscaling state: %w", err) + } + result := ReconcileResult{State: nextState, Decision: decision} + if decision.Action == ActionNone && !nextState.Active { + return result, nil + } + if decision.Action == ActionNotify { + r.publishFailure(deployment, decision.Reason, events.SeverityWarning) + return result, nil + } + execution, err := r.executor.Execute(ctx, deployment, route, decision) + result.Execution = &execution + if err != nil { + r.publishFailure(deployment, err.Error(), events.SeverityCritical) + return result, fmt.Errorf("execute autoscaling decision: %w", err) + } + nextState.Replicas = execution.Status.Desired + if execution.Route.ID != "" { + nextState.Route = execution.Route + } + if err := r.store.SetState(deployment, nextState); err != nil { + return result, fmt.Errorf("save autoscaling execution: %w", err) + } + result.State = nextState + if execution.Pending { + return result, nil + } + return result, nil +} + +func (r *Runner) publishFailure(deployment, message string, severity events.Severity) { + if r.publisher == nil { + return + } + _, _ = r.publisher.Publish(events.Event{ + Source: "capacity", Type: "autoscale.blocked", Severity: severity, + Title: "Autoscaling needs attention", Message: message, + Scope: events.Scope{Node: r.node, Deployment: deployment}, + CorrelationKey: "autoscale:" + r.node + ":" + deployment, OccurredAt: time.Now(), + }) +} diff --git a/internal/autoscale/runner_test.go b/internal/autoscale/runner_test.go new file mode 100644 index 0000000..8ed69a8 --- /dev/null +++ b/internal/autoscale/runner_test.go @@ -0,0 +1,119 @@ +package autoscale + +import ( + "context" + "testing" + "time" + + "github.com/flatrun/agent/internal/capacity" + "github.com/flatrun/agent/internal/events" + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" +) + +type runnerStore struct { + policy Policy + state State +} + +func (s *runnerStore) Policy(string) (Policy, error) { return s.policy, nil } +func (s *runnerStore) State(string) (State, error) { return s.state, nil } +func (s *runnerStore) SetState(_ string, state State) error { s.state = state; return nil } + +type runnerExecutor struct { + decision Decision +} + +func (e *runnerExecutor) Execute(_ context.Context, _ string, _ routing.Route, decision Decision) (Execution, error) { + e.decision = decision + return Execution{Decision: decision, Status: orchestrator.Status{Desired: decision.Replicas}}, nil +} + +type runnerPublisher struct { + events []events.Event +} + +func (p *runnerPublisher) Publish(event events.Event) (events.IngestResult, error) { + p.events = append(p.events, event) + return events.IngestResult{}, nil +} + +func TestRunnerPersistsObservationAndExecutesDecision(t *testing.T) { + policy := DefaultPolicy() + policy.ScaleUpWindows = 1 + store := &runnerStore{policy: policy} + executor := &runnerExecutor{} + runner := NewRunner(store, executor, nil, "prod-1") + result, err := runner.Reconcile(context.Background(), "shop", Input{ + Now: time.Now(), Replicas: 1, CPUPercent: 90, + Diagnosis: capacity.Diagnosis{Action: capacity.ActionAddReplica}, + }, routing.Route{ID: "shop"}) + if err != nil { + t.Fatal(err) + } + if result.Decision.Action != ActionAddReplica || executor.decision.Replicas != 2 || store.state.HighWindows != 0 { + t.Fatalf("unexpected reconciliation: %+v", result) + } +} + +func TestRunnerPublishesCorrelatedBlockedEvent(t *testing.T) { + policy := DefaultPolicy() + policy.MaxReplicas = 1 + policy.ScaleUpWindows = 1 + store := &runnerStore{policy: policy} + publisher := &runnerPublisher{} + runner := NewRunner(store, &runnerExecutor{}, publisher, "prod-1") + _, err := runner.Reconcile(context.Background(), "shop", Input{Now: time.Now(), Replicas: 1, CPUPercent: 95}, routing.Route{ID: "shop"}) + if err != nil { + t.Fatal(err) + } + if len(publisher.events) != 1 || publisher.events[0].CorrelationKey != "autoscale:prod-1:shop" { + t.Fatalf("unexpected events: %+v", publisher.events) + } +} + +func TestRunnerPersistsSuccessfulExecution(t *testing.T) { + policy := DefaultPolicy() + policy.ScaleUpWindows = 1 + store := &runnerStore{policy: policy, state: State{Active: true, Replicas: 1}} + provider := &fakeOrchestrator{status: orchestrator.Status{ + Workload: "shop", Desired: 1, Available: 2, + Instances: []orchestrator.Instance{ + {ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}, + {ID: "two", Address: "10.0.0.2:8080", Healthy: true, Ready: true}, + }, + }} + router := &fakeRouter{} + runner := NewRunner(store, NewExecutor(provider, router), nil, "prod-1") + result, err := runner.Reconcile(context.Background(), "shop", Input{Now: time.Now(), Replicas: 1, CPUPercent: 95}, routing.Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http"}) + if err != nil { + t.Fatal(err) + } + if result.State.Replicas != 2 || len(result.State.Route.Backends) != 2 { + t.Fatalf("state = %#v", result.State) + } +} + +func TestRunnerPublishesPendingReplicaWhenItBecomesReady(t *testing.T) { + policy := DefaultPolicy() + policy.ScaleUpWindows = 1 + now := time.Now() + store := &runnerStore{policy: policy, state: State{Active: true, Replicas: 1}} + provider := &fakeOrchestrator{status: orchestrator.Status{ + Workload: "shop", Desired: 1, Available: 1, + Instances: []orchestrator.Instance{{ID: "one", Address: "10.0.0.1:8080", Healthy: true, Ready: true}}, + }} + router := &fakeRouter{} + runner := NewRunner(store, NewExecutor(provider, router), nil, "prod-1") + route := routing.Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http", Backends: []routing.Backend{{ID: "one", Address: "10.0.0.1:8080", Healthy: true, Weight: 1}}} + result, err := runner.Reconcile(context.Background(), "shop", Input{Now: now, Replicas: 1, CPUPercent: 95}, route) + if err != nil || result.Execution == nil || !result.Execution.Pending || store.state.Replicas != 2 { + t.Fatalf("result = %#v, state = %#v, error = %v", result, store.state, err) + } + provider.status.Available = 2 + provider.status.Instances = append(provider.status.Instances, orchestrator.Instance{ID: "two", Address: "10.0.0.2:8080", Healthy: true, Ready: true}) + result, err = runner.Reconcile(context.Background(), "shop", Input{Now: now.Add(time.Second), Replicas: 2}, route) + if err != nil || result.Execution == nil || result.Execution.Pending || len(store.state.Route.Backends) != 2 { + t.Fatalf("result = %#v, state = %#v, error = %v", result, store.state, err) + } +} diff --git a/internal/autoscale/store.go b/internal/autoscale/store.go new file mode 100644 index 0000000..6417abb --- /dev/null +++ b/internal/autoscale/store.go @@ -0,0 +1,148 @@ +package autoscale + +import ( + "context" + "database/sql" + "embed" + "encoding/json" + "io/fs" + "os" + "path/filepath" + "sync" + + "github.com/pressly/goose/v3" + _ "modernc.org/sqlite" +) + +//go:embed migrations/*.sql +var migrationFiles embed.FS + +type Store struct { + conn *sql.DB + mu sync.RWMutex +} + +func NewStore(deploymentsPath string) (*Store, error) { + dir := filepath.Join(deploymentsPath, ".flatrun") + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, err + } + conn, err := sql.Open("sqlite", "file:"+filepath.Join(dir, "autoscale.db")+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + if err != nil { + return nil, err + } + store := &Store{conn: conn} + if err := store.migrate(); err != nil { + conn.Close() + return nil, err + } + return store, nil +} + +func (s *Store) migrate() error { + migrations, err := fs.Sub(migrationFiles, "migrations") + if err != nil { + return err + } + provider, err := goose.NewProvider(goose.DialectSQLite3, s.conn, migrations, goose.WithTableName("autoscale_schema_version")) + if err != nil { + return err + } + _, err = provider.Up(context.Background()) + return err +} + +func (s *Store) Close() error { + return s.conn.Close() +} + +func (s *Store) Policy(deployment string) (Policy, error) { + s.mu.RLock() + defer s.mu.RUnlock() + var raw string + err := s.conn.QueryRow(`SELECT policy_json FROM autoscale_policies WHERE deployment = ?`, deployment).Scan(&raw) + if err == sql.ErrNoRows { + return DefaultPolicy(), nil + } + if err != nil { + return Policy{}, err + } + var policy Policy + if err := json.Unmarshal([]byte(raw), &policy); err != nil { + return Policy{}, err + } + return policy, nil +} + +func (s *Store) SetPolicy(deployment string, policy Policy) error { + if err := ValidatePolicy(policy); err != nil { + return err + } + raw, err := json.Marshal(policy) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + _, err = s.conn.Exec(` + INSERT INTO autoscale_policies (deployment, policy_json, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(deployment) DO UPDATE SET policy_json = excluded.policy_json, updated_at = CURRENT_TIMESTAMP`, deployment, raw) + return err +} + +func (s *Store) State(deployment string) (State, error) { + s.mu.RLock() + defer s.mu.RUnlock() + var raw string + err := s.conn.QueryRow(`SELECT state_json FROM autoscale_states WHERE deployment = ?`, deployment).Scan(&raw) + if err == sql.ErrNoRows { + return State{}, nil + } + if err != nil { + return State{}, err + } + var state State + if err := json.Unmarshal([]byte(raw), &state); err != nil { + return State{}, err + } + return state, nil +} + +func (s *Store) SetState(deployment string, state State) error { + raw, err := json.Marshal(state) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + _, err = s.conn.Exec(` + INSERT INTO autoscale_states (deployment, state_json, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(deployment) DO UPDATE SET state_json = excluded.state_json, updated_at = CURRENT_TIMESTAMP`, deployment, raw) + return err +} + +func (s *Store) ActiveStates() (map[string]State, error) { + s.mu.RLock() + defer s.mu.RUnlock() + rows, err := s.conn.Query(`SELECT deployment, state_json FROM autoscale_states`) + if err != nil { + return nil, err + } + defer rows.Close() + states := make(map[string]State) + for rows.Next() { + var deployment string + var raw string + if err := rows.Scan(&deployment, &raw); err != nil { + return nil, err + } + var state State + if err := json.Unmarshal([]byte(raw), &state); err != nil { + return nil, err + } + if state.Active { + states[deployment] = state + } + } + return states, rows.Err() +} diff --git a/internal/autoscale/store_test.go b/internal/autoscale/store_test.go new file mode 100644 index 0000000..fd09bcb --- /dev/null +++ b/internal/autoscale/store_test.go @@ -0,0 +1,74 @@ +package autoscale + +import ( + "testing" + "time" + + "github.com/flatrun/agent/internal/orchestrator" + "github.com/flatrun/agent/internal/routing" +) + +func TestStorePersistsPolicyAndState(t *testing.T) { + dir := t.TempDir() + store, err := NewStore(dir) + if err != nil { + t.Fatal(err) + } + policy := DefaultPolicy() + policy.MaxReplicas = 7 + policy.AllowFleetCapacity = true + state := State{ + HighWindows: 2, LastAction: time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC), Active: true, + Provider: orchestrator.ProviderSwarm, Service: "web", Replicas: 2, + Route: routing.Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http"}, + } + if err := store.SetPolicy("shop", policy); err != nil { + t.Fatal(err) + } + if err := store.SetState("shop", state); err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + + store, err = NewStore(dir) + if err != nil { + t.Fatal(err) + } + defer store.Close() + savedPolicy, err := store.Policy("shop") + if err != nil { + t.Fatal(err) + } + savedState, err := store.State("shop") + if err != nil { + t.Fatal(err) + } + if savedPolicy.MaxReplicas != 7 || !savedPolicy.AllowFleetCapacity { + t.Fatalf("unexpected policy: %+v", savedPolicy) + } + if savedState.HighWindows != 2 || !savedState.LastAction.Equal(state.LastAction) { + t.Fatalf("unexpected state: %+v", savedState) + } + active, err := store.ActiveStates() + if err != nil { + t.Fatal(err) + } + if active["shop"].Route.Domain != "shop.example.com" || active["shop"].Provider != orchestrator.ProviderSwarm { + t.Fatalf("unexpected active states: %+v", active) + } +} + +func TestStoreRejectsInvalidPolicy(t *testing.T) { + store, err := NewStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer store.Close() + policy := DefaultPolicy() + policy.MaxReplicas = 0 + if err := store.SetPolicy("shop", policy); err == nil { + t.Fatal("expected invalid policy error") + } +} diff --git a/internal/autoscale/supervisor.go b/internal/autoscale/supervisor.go new file mode 100644 index 0000000..8590723 --- /dev/null +++ b/internal/autoscale/supervisor.go @@ -0,0 +1,94 @@ +package autoscale + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/flatrun/agent/internal/events" +) + +type ActiveStore interface { + PolicyStore + ActiveStates() (map[string]State, error) +} + +type RuntimeSession struct { + Input Input + Executor ActionExecutor + Close func() error +} + +type RuntimeFactory interface { + Build(context.Context, string, State) (RuntimeSession, error) +} + +type Supervisor struct { + store ActiveStore + factory RuntimeFactory + publisher EventPublisher + node string + interval time.Duration + mu sync.Mutex +} + +func NewSupervisor(store ActiveStore, factory RuntimeFactory, publisher EventPublisher, node string, interval time.Duration) *Supervisor { + if interval <= 0 { + interval = 30 * time.Second + } + return &Supervisor{store: store, factory: factory, publisher: publisher, node: node, interval: interval} +} + +func (s *Supervisor) Run(ctx context.Context) { + ticker := time.NewTicker(s.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _ = s.Tick(ctx) + } + } +} + +func (s *Supervisor) Tick(ctx context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + states, err := s.store.ActiveStates() + if err != nil { + return fmt.Errorf("load active autoscaling workloads: %w", err) + } + var firstErr error + for deployment, state := range states { + session, err := s.factory.Build(ctx, deployment, state) + if err == nil { + _, err = NewRunner(s.store, session.Executor, s.publisher, s.node).Reconcile(ctx, deployment, session.Input, state.Route) + if session.Close != nil { + if closeErr := session.Close(); err == nil { + err = closeErr + } + } + } + if err != nil { + s.publishFailure(deployment, err) + if firstErr == nil { + firstErr = err + } + } + } + return firstErr +} + +func (s *Supervisor) publishFailure(deployment string, err error) { + if s.publisher == nil { + return + } + _, _ = s.publisher.Publish(events.Event{ + Source: "capacity", Type: "autoscale.observation_failed", Severity: events.SeverityCritical, + Title: "Autoscaling observation failed", Message: err.Error(), + Scope: events.Scope{Node: s.node, Deployment: deployment}, + CorrelationKey: "autoscale-observation:" + s.node + ":" + deployment, OccurredAt: time.Now(), + }) +} diff --git a/internal/autoscale/supervisor_test.go b/internal/autoscale/supervisor_test.go new file mode 100644 index 0000000..f1a1279 --- /dev/null +++ b/internal/autoscale/supervisor_test.go @@ -0,0 +1,55 @@ +package autoscale + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/flatrun/agent/internal/orchestrator" +) + +type supervisorStore struct { + runnerStore + states map[string]State +} + +func (s *supervisorStore) ActiveStates() (map[string]State, error) { return s.states, nil } + +type runtimeFactoryStub struct { + err error + builtFor string + executor *runnerExecutor +} + +func (f *runtimeFactoryStub) Build(_ context.Context, deployment string, state State) (RuntimeSession, error) { + f.builtFor = deployment + return RuntimeSession{Input: Input{Now: time.Now(), Replicas: state.Replicas}, Executor: f.executor}, f.err +} + +func TestSupervisorReconcilesActiveWorkloads(t *testing.T) { + policy := DefaultPolicy() + store := &supervisorStore{ + runnerStore: runnerStore{policy: policy, state: State{Active: true, Replicas: 1}}, + states: map[string]State{"shop": {Active: true, Replicas: 1}}, + } + factory := &runtimeFactoryStub{executor: &runnerExecutor{}} + if err := NewSupervisor(store, factory, nil, "prod-1", time.Second).Tick(context.Background()); err != nil { + t.Fatal(err) + } + if factory.builtFor != "shop" { + t.Fatalf("built for %q", factory.builtFor) + } +} + +func TestSupervisorPublishesObservationFailure(t *testing.T) { + store := &supervisorStore{ + runnerStore: runnerStore{policy: DefaultPolicy()}, + states: map[string]State{"shop": {Active: true, Provider: orchestrator.ProviderSwarm}}, + } + publisher := &runnerPublisher{} + err := NewSupervisor(store, &runtimeFactoryStub{err: errors.New("stats unavailable")}, publisher, "prod-1", time.Second).Tick(context.Background()) + if err == nil || len(publisher.events) != 1 || publisher.events[0].CorrelationKey != "autoscale-observation:prod-1:shop" { + t.Fatalf("error = %v, events = %#v", err, publisher.events) + } +} diff --git a/internal/capacity/capacity.go b/internal/capacity/capacity.go new file mode 100644 index 0000000..7cca183 --- /dev/null +++ b/internal/capacity/capacity.go @@ -0,0 +1,195 @@ +package capacity + +import ( + "math" + + "github.com/flatrun/agent/pkg/config" +) + +type Pressure string + +const ( + PressureNone Pressure = "none" + PressureAllocation Pressure = "allocation" + PressureHost Pressure = "host" +) + +type Action string + +const ( + ActionNone Action = "none" + ActionIncreaseMemory Action = "increase_memory" + ActionIncreaseCPU Action = "increase_cpu" + ActionAddReplica Action = "add_replica" + ActionNotify Action = "notify" +) + +type Host struct { + CPUCores float64 `json:"cpu_cores"` + CPUUsagePercent float64 `json:"cpu_usage_percent"` + MemoryTotal uint64 `json:"memory_total"` + MemoryAvailable uint64 `json:"memory_available"` +} + +type Container struct { + ID string `json:"id"` + Name string `json:"name"` + CPUPercent float64 `json:"cpu_percent"` + CPULimit float64 `json:"cpu_limit"` + MemoryUsage uint64 `json:"memory_usage"` + MemoryLimit uint64 `json:"memory_limit"` +} + +type Policy struct { + AllocationThresholdPercent float64 `json:"allocation_threshold_percent"` + HostThresholdPercent float64 `json:"host_threshold_percent"` + HostMemoryReserve uint64 `json:"host_memory_reserve"` + HostCPUReserve float64 `json:"host_cpu_reserve"` + MemoryStepPercent float64 `json:"memory_step_percent"` + CPUStepPercent float64 `json:"cpu_step_percent"` + MaxMemory uint64 `json:"max_memory"` + MaxCPU float64 `json:"max_cpu"` + AllowVertical bool `json:"allow_vertical"` + AllowHorizontal bool `json:"allow_horizontal"` +} + +type Diagnosis struct { + Pressure Pressure `json:"pressure"` + Resource string `json:"resource,omitempty"` + Action Action `json:"action"` + CurrentLimit float64 `json:"current_limit,omitempty"` + RecommendedLimit float64 `json:"recommended_limit,omitempty"` + Reason string `json:"reason"` +} + +type Offer struct { + Enabled bool `json:"enabled"` + AvailableCPU float64 `json:"available_cpu"` + AvailableMemory uint64 `json:"available_memory"` + Reason string `json:"reason"` +} + +func DefaultPolicy() Policy { + return Policy{ + AllocationThresholdPercent: 90, + HostThresholdPercent: 85, + HostMemoryReserve: 512 * 1024 * 1024, + HostCPUReserve: 0.25, + MemoryStepPercent: 50, + CPUStepPercent: 50, + AllowVertical: true, + AllowHorizontal: true, + } +} + +func PolicyFromConfig(cfg config.CapacityConfig) Policy { + policy := DefaultPolicy() + if cfg.AllocationThresholdPercent > 0 { + policy.AllocationThresholdPercent = cfg.AllocationThresholdPercent + } + if cfg.HostThresholdPercent > 0 { + policy.HostThresholdPercent = cfg.HostThresholdPercent + } + if cfg.HostMemoryReserve > 0 { + policy.HostMemoryReserve = cfg.HostMemoryReserve + } + if cfg.HostCPUReserve > 0 { + policy.HostCPUReserve = cfg.HostCPUReserve + } + if cfg.MemoryStepPercent > 0 { + policy.MemoryStepPercent = cfg.MemoryStepPercent + } + if cfg.CPUStepPercent > 0 { + policy.CPUStepPercent = cfg.CPUStepPercent + } + policy.MaxMemory = cfg.MaxMemory + policy.MaxCPU = cfg.MaxCPU + if cfg.AllowVertical != nil { + policy.AllowVertical = *cfg.AllowVertical + } + if cfg.AllowHorizontal != nil { + policy.AllowHorizontal = *cfg.AllowHorizontal + } + return policy +} + +func Diagnose(host Host, container Container, policy Policy) Diagnosis { + if policy.AllocationThresholdPercent <= 0 { + policy = DefaultPolicy() + } + + if container.MemoryLimit > 0 { + utilization := float64(container.MemoryUsage) / float64(container.MemoryLimit) * 100 + if utilization >= policy.AllocationThresholdPercent { + next := grow(float64(container.MemoryLimit), policy.MemoryStepPercent, float64(policy.MaxMemory)) + required := uint64(math.Max(0, next-float64(container.MemoryLimit))) + if policy.AllowVertical && next > float64(container.MemoryLimit) && host.MemoryAvailable >= policy.HostMemoryReserve+required { + return Diagnosis{Pressure: PressureAllocation, Resource: "memory", Action: ActionIncreaseMemory, CurrentLimit: float64(container.MemoryLimit), RecommendedLimit: next, Reason: "Container memory is constrained while the host has reserved headroom"} + } + return exhaustedAction("memory", policy) + } + } + + if container.CPULimit > 0 { + utilization := container.CPUPercent / (container.CPULimit * 100) * 100 + if utilization >= policy.AllocationThresholdPercent { + next := grow(container.CPULimit, policy.CPUStepPercent, policy.MaxCPU) + available := host.CPUCores * math.Max(0, 100-host.CPUUsagePercent) / 100 + if policy.AllowVertical && next > container.CPULimit && available >= policy.HostCPUReserve+(next-container.CPULimit) { + return Diagnosis{Pressure: PressureAllocation, Resource: "cpu", Action: ActionIncreaseCPU, CurrentLimit: container.CPULimit, RecommendedLimit: next, Reason: "Container CPU is constrained while the host has reserved headroom"} + } + return exhaustedAction("cpu", policy) + } + } + + memoryPressure := host.MemoryTotal > 0 && float64(host.MemoryTotal-host.MemoryAvailable)/float64(host.MemoryTotal)*100 >= policy.HostThresholdPercent + if memoryPressure || host.CPUUsagePercent >= policy.HostThresholdPercent { + resource := "cpu" + if memoryPressure { + resource = "memory" + } + return Diagnosis{Pressure: PressureHost, Resource: resource, Action: horizontalOrNotify(policy), Reason: "Host capacity is below its configured safety reserve"} + } + + return Diagnosis{Pressure: PressureNone, Action: ActionNone, Reason: "Container allocation and host capacity are within policy"} +} + +func FleetOffer(host Host, policy Policy, enabled bool) Offer { + if !enabled { + return Offer{Reason: "This server has not permitted fleet workloads"} + } + + availableCPU := host.CPUCores*math.Max(0, 100-host.CPUUsagePercent)/100 - policy.HostCPUReserve + if availableCPU < 0 { + availableCPU = 0 + } + availableMemory := uint64(0) + if host.MemoryAvailable > policy.HostMemoryReserve { + availableMemory = host.MemoryAvailable - policy.HostMemoryReserve + } + enabled = availableCPU > 0 && availableMemory > 0 + reason := "CPU and memory are available above the host safety reserve" + if !enabled { + reason = "Host capacity is at or below its safety reserve" + } + return Offer{Enabled: enabled, AvailableCPU: availableCPU, AvailableMemory: availableMemory, Reason: reason} +} + +func grow(current, percent, maximum float64) float64 { + next := current * (1 + percent/100) + if maximum > 0 && next > maximum { + return maximum + } + return next +} + +func exhaustedAction(resource string, policy Policy) Diagnosis { + return Diagnosis{Pressure: PressureHost, Resource: resource, Action: horizontalOrNotify(policy), Reason: "Container allocation is constrained and the host cannot safely increase it"} +} + +func horizontalOrNotify(policy Policy) Action { + if policy.AllowHorizontal { + return ActionAddReplica + } + return ActionNotify +} diff --git a/internal/capacity/capacity_test.go b/internal/capacity/capacity_test.go new file mode 100644 index 0000000..43b5e53 --- /dev/null +++ b/internal/capacity/capacity_test.go @@ -0,0 +1,88 @@ +package capacity + +import ( + "testing" + + "github.com/flatrun/agent/pkg/config" +) + +func TestDiagnoseIncreasesContainerMemoryBeforeScaling(t *testing.T) { + policy := DefaultPolicy() + host := Host{MemoryTotal: 16 << 30, MemoryAvailable: 8 << 30, CPUCores: 8, CPUUsagePercent: 20} + container := Container{MemoryUsage: 950 << 20, MemoryLimit: 1 << 30} + + diagnosis := Diagnose(host, container, policy) + if diagnosis.Pressure != PressureAllocation || diagnosis.Action != ActionIncreaseMemory { + t.Fatalf("diagnosis = %#v", diagnosis) + } + if diagnosis.RecommendedLimit != float64(1536<<20) { + t.Fatalf("recommended memory = %.0f", diagnosis.RecommendedLimit) + } +} + +func TestDiagnoseAddsReplicaWhenHostCannotResize(t *testing.T) { + policy := DefaultPolicy() + host := Host{MemoryTotal: 4 << 30, MemoryAvailable: 600 << 20, CPUCores: 2, CPUUsagePercent: 80} + container := Container{MemoryUsage: 950 << 20, MemoryLimit: 1 << 30} + + diagnosis := Diagnose(host, container, policy) + if diagnosis.Pressure != PressureHost || diagnosis.Action != ActionAddReplica { + t.Fatalf("diagnosis = %#v", diagnosis) + } +} + +func TestDiagnoseReportsUnlimitedContainerAsHealthyWhenHostHasHeadroom(t *testing.T) { + policy := DefaultPolicy() + host := Host{MemoryTotal: 16 << 30, MemoryAvailable: 10 << 30, CPUCores: 8, CPUUsagePercent: 10} + + diagnosis := Diagnose(host, Container{MemoryUsage: 6 << 30, CPUPercent: 200}, policy) + if diagnosis.Pressure != PressureNone || diagnosis.Action != ActionNone { + t.Fatalf("diagnosis = %#v", diagnosis) + } +} + +func TestPolicyFromConfigPreservesExplicitScalingChoices(t *testing.T) { + disabled := false + policy := PolicyFromConfig(config.CapacityConfig{ + AllocationThresholdPercent: 75, + HostThresholdPercent: 80, + AllowVertical: &disabled, + AllowHorizontal: &disabled, + }) + + if policy.AllocationThresholdPercent != 75 || policy.HostThresholdPercent != 80 { + t.Fatalf("thresholds = %.0f, %.0f", policy.AllocationThresholdPercent, policy.HostThresholdPercent) + } + if policy.AllowVertical || policy.AllowHorizontal { + t.Fatalf("scaling choices = vertical:%t horizontal:%t", policy.AllowVertical, policy.AllowHorizontal) + } + if policy.HostMemoryReserve == 0 || policy.MemoryStepPercent == 0 { + t.Fatalf("defaults were not retained: %#v", policy) + } +} + +func TestFleetOfferKeepsHostReserve(t *testing.T) { + policy := DefaultPolicy() + offer := FleetOffer(Host{ + CPUCores: 8, + CPUUsagePercent: 25, + MemoryAvailable: 8 << 30, + }, policy, true) + + if !offer.Enabled { + t.Fatalf("offer = %#v", offer) + } + if offer.AvailableCPU != 5.75 { + t.Fatalf("available CPU = %.2f", offer.AvailableCPU) + } + if offer.AvailableMemory != 7680<<20 { + t.Fatalf("available memory = %d", offer.AvailableMemory) + } +} + +func TestFleetOfferRequiresOperatorPermission(t *testing.T) { + offer := FleetOffer(Host{CPUCores: 8, MemoryAvailable: 8 << 30}, DefaultPolicy(), false) + if offer.Enabled || offer.AvailableCPU != 0 || offer.AvailableMemory != 0 { + t.Fatalf("offer = %#v", offer) + } +} diff --git a/internal/cluster/capabilities.go b/internal/cluster/capabilities.go new file mode 100644 index 0000000..87ff1fc --- /dev/null +++ b/internal/cluster/capabilities.go @@ -0,0 +1,45 @@ +package cluster + +type Capability string + +const ( + CapabilityFleetRead Capability = "fleet.read" + CapabilityDeploymentsRead Capability = "deployments.read" + CapabilityDeploymentsRun Capability = "deployments.run" + CapabilityCapacityRead Capability = "capacity.read" + CapabilityCapacityOffer Capability = "capacity.offer" + CapabilityEventsPublish Capability = "events.publish" + CapabilityRoutingManage Capability = "routing.manage" +) + +type Grant struct { + Capability Capability `json:"capability"` + Deployments []string `json:"deployments,omitempty"` + MaxCPU float64 `json:"max_cpu,omitempty"` + MaxMemory uint64 `json:"max_memory,omitempty"` + MaxReplicas int `json:"max_replicas,omitempty"` +} + +type PeerPolicy struct { + Peer string `json:"peer"` + Grants []Grant `json:"grants"` +} + +func DefaultPeerGrants() []Grant { + return []Grant{ + {Capability: CapabilityFleetRead}, + {Capability: CapabilityDeploymentsRead}, + {Capability: CapabilityDeploymentsRun}, + {Capability: CapabilityCapacityRead}, + } +} + +func ValidCapability(value Capability) bool { + switch value { + case CapabilityFleetRead, CapabilityDeploymentsRead, CapabilityDeploymentsRun, + CapabilityCapacityRead, CapabilityCapacityOffer, CapabilityEventsPublish, CapabilityRoutingManage: + return true + default: + return false + } +} diff --git a/internal/cluster/db.go b/internal/cluster/db.go index 87a658c..ee562a8 100644 --- a/internal/cluster/db.go +++ b/internal/cluster/db.go @@ -1,17 +1,25 @@ package cluster import ( + "context" "crypto/sha256" "database/sql" + "embed" "encoding/hex" + "encoding/json" + "io/fs" "os" "path/filepath" "sync" "time" + "github.com/pressly/goose/v3" _ "modernc.org/sqlite" ) +//go:embed migrations/*.sql +var migrationFiles embed.FS + type Peer struct { ID int64 `json:"id"` Name string `json:"name"` @@ -60,6 +68,10 @@ func NewDB(deploymentsPath string) (*DB, error) { conn.Close() return nil, err } + if err := db.repair(); err != nil { + conn.Close() + return nil, err + } return db, nil } @@ -71,35 +83,31 @@ func (db *DB) Close() error { } func (db *DB) migrate() error { - schema := ` - CREATE TABLE IF NOT EXISTS peers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT UNIQUE NOT NULL, - url TEXT NOT NULL, - api_key_hash TEXT NOT NULL, - api_key_encrypted TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'active', - created_at DATETIME DEFAULT CURRENT_TIMESTAMP, - last_seen_at DATETIME - ); - - CREATE TABLE IF NOT EXISTS invites ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - token_hash TEXT UNIQUE NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - created_by INTEGER NOT NULL, - accepted_peer TEXT, - expires_at DATETIME NOT NULL, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP - ); - - CREATE INDEX IF NOT EXISTS idx_peers_name ON peers(name); - CREATE INDEX IF NOT EXISTS idx_peers_status ON peers(status); - CREATE INDEX IF NOT EXISTS idx_invites_token_hash ON invites(token_hash); - CREATE INDEX IF NOT EXISTS idx_invites_status ON invites(status); - ` - - _, err := db.conn.Exec(schema) + migrations, err := fs.Sub(migrationFiles, "migrations") + if err != nil { + return err + } + provider, err := goose.NewProvider( + goose.DialectSQLite3, + db.conn, + migrations, + goose.WithTableName("cluster_schema_version"), + ) + if err != nil { + return err + } + _, err = provider.Up(context.Background()) + return err +} + +func (db *DB) repair() error { + grants, err := json.Marshal(DefaultPeerGrants()) + if err != nil { + return err + } + _, err = db.conn.Exec(` + INSERT OR IGNORE INTO peer_policies (peer_name, grants_json) + SELECT name, ? FROM peers`, grants) return err } @@ -120,7 +128,18 @@ func (db *DB) CreatePeer(peer *Peer) (int64, error) { if err != nil { return 0, err } - return result.LastInsertId() + id, err := result.LastInsertId() + if err != nil { + return 0, err + } + grants, err := json.Marshal(DefaultPeerGrants()) + if err != nil { + return 0, err + } + if _, err := db.conn.Exec(`INSERT OR IGNORE INTO peer_policies (peer_name, grants_json) VALUES (?, ?)`, peer.Name, grants); err != nil { + return 0, err + } + return id, nil } func (db *DB) GetPeer(name string) (*Peer, error) { @@ -180,10 +199,50 @@ func (db *DB) DeletePeer(name string) error { db.mu.Lock() defer db.mu.Unlock() + if _, err := db.conn.Exec(`DELETE FROM peer_policies WHERE peer_name = ?`, name); err != nil { + return err + } _, err := db.conn.Exec(`DELETE FROM peers WHERE name = ?`, name) return err } +func (db *DB) GetPeerPolicy(name string) (*PeerPolicy, error) { + db.mu.RLock() + defer db.mu.RUnlock() + + var raw string + if err := db.conn.QueryRow(`SELECT grants_json FROM peer_policies WHERE peer_name = ?`, name).Scan(&raw); err != nil { + return nil, err + } + var grants []Grant + if err := json.Unmarshal([]byte(raw), &grants); err != nil { + return nil, err + } + return &PeerPolicy{Peer: name, Grants: grants}, nil +} + +func (db *DB) SetPeerPolicy(policy PeerPolicy) error { + db.mu.Lock() + defer db.mu.Unlock() + + grants, err := json.Marshal(policy.Grants) + if err != nil { + return err + } + result, err := db.conn.Exec(`UPDATE peer_policies SET grants_json = ?, updated_at = ? WHERE peer_name = ?`, grants, time.Now(), policy.Peer) + if err != nil { + return err + } + rows, err := result.RowsAffected() + if err != nil { + return err + } + if rows == 0 { + return sql.ErrNoRows + } + return nil +} + func (db *DB) UpdateLastSeen(name string) error { db.mu.Lock() defer db.mu.Unlock() diff --git a/internal/cluster/db_test.go b/internal/cluster/db_test.go index 3e195d8..0b2647f 100644 --- a/internal/cluster/db_test.go +++ b/internal/cluster/db_test.go @@ -40,6 +40,119 @@ func TestNewDB(t *testing.T) { } } +func TestNewDBRecordsSchemaVersion(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + var version int64 + if err := db.conn.QueryRow(`SELECT MAX(version_id) FROM cluster_schema_version WHERE is_applied = 1`).Scan(&version); err != nil { + t.Fatalf("read schema version: %v", err) + } + if version != 2 { + t.Fatalf("schema version = %d, want 2", version) + } +} + +func TestNewDBMigratesExistingPeersWithDefaultPolicy(t *testing.T) { + tmpDir := t.TempDir() + dbDir := filepath.Join(tmpDir, ".flatrun") + if err := os.MkdirAll(dbDir, 0755); err != nil { + t.Fatalf("create database directory: %v", err) + } + conn, err := sql.Open("sqlite", "file:"+filepath.Join(dbDir, "cluster.db")) + if err != nil { + t.Fatalf("open legacy database: %v", err) + } + _, err = conn.Exec(` + CREATE TABLE peers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT UNIQUE NOT NULL, + url TEXT NOT NULL, + api_key_hash TEXT NOT NULL, + api_key_encrypted TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + last_seen_at DATETIME + ); + CREATE TABLE invites ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + token_hash TEXT UNIQUE NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + created_by INTEGER NOT NULL, + accepted_peer TEXT, + expires_at DATETIME NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP + ); + INSERT INTO peers (name, url, api_key_hash, api_key_encrypted) + VALUES ('prod-2', 'https://prod-2.example.com', 'hash', 'encrypted'); + `) + if err != nil { + _ = conn.Close() + t.Fatalf("seed legacy database: %v", err) + } + if err := conn.Close(); err != nil { + t.Fatalf("close legacy database: %v", err) + } + + db, err := NewDB(tmpDir) + if err != nil { + t.Fatalf("migrate legacy database: %v", err) + } + defer db.Close() + + peer, err := db.GetPeer("prod-2") + if err != nil { + t.Fatalf("read migrated peer: %v", err) + } + if peer.URL != "https://prod-2.example.com" { + t.Fatalf("peer URL = %q", peer.URL) + } + policy, err := db.GetPeerPolicy("prod-2") + if err != nil { + t.Fatalf("read migrated peer policy: %v", err) + } + if len(policy.Grants) != len(DefaultPeerGrants()) { + t.Fatalf("policy grants = %#v", policy.Grants) + } +} + +func TestNewDBRepairsMissingPeerPolicy(t *testing.T) { + tmpDir := t.TempDir() + db, err := NewDB(tmpDir) + if err != nil { + t.Fatalf("open database: %v", err) + } + peer := &Peer{ + Name: "prod-2", + URL: "https://prod-2.example.com", + APIKeyHash: "hash", + APIKeyEncrypted: "encrypted", + Status: "active", + } + if _, err := db.CreatePeer(peer); err != nil { + t.Fatalf("create peer: %v", err) + } + if _, err := db.conn.Exec(`DELETE FROM peer_policies WHERE peer_name = ?`, peer.Name); err != nil { + t.Fatalf("remove peer policy: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close database: %v", err) + } + + db, err = NewDB(tmpDir) + if err != nil { + t.Fatalf("reopen database: %v", err) + } + defer db.Close() + policy, err := db.GetPeerPolicy(peer.Name) + if err != nil { + t.Fatalf("read repaired peer policy: %v", err) + } + if len(policy.Grants) != len(DefaultPeerGrants()) { + t.Fatalf("policy grants = %#v", policy.Grants) + } +} + func TestDBPath(t *testing.T) { tmpDir, err := os.MkdirTemp("", "cluster_test") if err != nil { @@ -188,6 +301,55 @@ func TestDeletePeer(t *testing.T) { } } +func TestPeerPolicyDefaultsAndUpdates(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + _, err := db.CreatePeer(&Peer{ + Name: "policy-peer", URL: "https://peer.example.com", + APIKeyHash: "h", APIKeyEncrypted: "e", Status: "active", + }) + if err != nil { + t.Fatalf("CreatePeer failed: %v", err) + } + + policy, err := db.GetPeerPolicy("policy-peer") + if err != nil { + t.Fatalf("GetPeerPolicy failed: %v", err) + } + if len(policy.Grants) != len(DefaultPeerGrants()) { + t.Fatalf("default grants = %#v", policy.Grants) + } + + policy.Grants = []Grant{{Capability: CapabilityCapacityOffer, MaxCPU: 2, MaxMemory: 2 << 30, MaxReplicas: 3}} + if err := db.SetPeerPolicy(*policy); err != nil { + t.Fatalf("SetPeerPolicy failed: %v", err) + } + updated, err := db.GetPeerPolicy("policy-peer") + if err != nil { + t.Fatalf("GetPeerPolicy after update failed: %v", err) + } + if len(updated.Grants) != 1 || updated.Grants[0].MaxReplicas != 3 { + t.Fatalf("updated policy = %#v", updated) + } +} + +func TestDeletePeerDeletesPolicy(t *testing.T) { + db, cleanup := setupTestDB(t) + defer cleanup() + + _, _ = db.CreatePeer(&Peer{ + Name: "policy-delete", URL: "https://peer.example.com", + APIKeyHash: "h", APIKeyEncrypted: "e", Status: "active", + }) + if err := db.DeletePeer("policy-delete"); err != nil { + t.Fatalf("DeletePeer failed: %v", err) + } + if _, err := db.GetPeerPolicy("policy-delete"); err == nil { + t.Fatal("policy should be deleted with its peer") + } +} + func TestUpdateLastSeen(t *testing.T) { db, cleanup := setupTestDB(t) defer cleanup() diff --git a/internal/cluster/manager.go b/internal/cluster/manager.go index 1d8cb6f..ce4c98b 100644 --- a/internal/cluster/manager.go +++ b/internal/cluster/manager.go @@ -12,14 +12,21 @@ import ( "log" "sync" "time" + + "github.com/flatrun/agent/internal/events" ) +type EventPublisher interface { + Publish(events.Event) (events.IngestResult, error) +} + type PeerStatus struct { Name string `json:"name"` URL string `json:"url"` Online bool `json:"online"` LastSeen time.Time `json:"last_seen"` Error string `json:"error,omitempty"` + Probed bool `json:"-"` } type Result struct { @@ -36,6 +43,7 @@ type Manager struct { healthInterval time.Duration requestTimeout time.Duration encryptionKey []byte + publisher EventPublisher cancel context.CancelFunc } @@ -52,6 +60,12 @@ func NewManager(db *DB, serverName string, healthInterval, requestTimeout time.D } } +func (m *Manager) SetEventPublisher(publisher EventPublisher) { + m.mu.Lock() + m.publisher = publisher + m.mu.Unlock() +} + func (m *Manager) Start(ctx context.Context) error { ctx, m.cancel = context.WithCancel(ctx) @@ -128,7 +142,11 @@ func (m *Manager) checkAllPeers(ctx context.Context) { m.mu.Lock() st, exists := m.status[p.name] + wasOnline := exists && st.Online + wasKnown := exists && st.Probed + publisher := m.publisher if exists { + st.Probed = true if err != nil { st.Online = false st.Error = err.Error() @@ -140,6 +158,14 @@ func (m *Manager) checkAllPeers(ctx context.Context) { } } m.mu.Unlock() + + event := fleetHealthEvent(p.name, wasKnown, wasOnline, err) + if publisher == nil || event == nil { + continue + } + if _, publishErr := publisher.Publish(*event); publishErr != nil { + log.Printf("Warning: Failed to publish Fleet health event for %s: %v", p.name, publishErr) + } } for _, name := range seenNames { @@ -147,6 +173,29 @@ func (m *Manager) checkAllPeers(ctx context.Context) { } } +func fleetHealthEvent(name string, wasKnown, wasOnline bool, healthErr error) *events.Event { + isOnline := healthErr == nil + if (!wasKnown && isOnline) || (wasKnown && wasOnline == isOnline) { + return nil + } + event := &events.Event{ + Source: "fleet", Scope: events.Scope{Node: name}, CorrelationKey: "node:" + name, OccurredAt: time.Now(), + } + if healthErr != nil { + event.Type = "node.unavailable" + event.Severity = events.SeverityCritical + event.Title = name + " is unavailable" + event.Message = "The Fleet node stopped responding. Related deployment failures will be grouped into this incident." + return event + } + event.Type = "node.recovered" + event.Severity = events.SeverityInfo + event.Title = name + " recovered" + event.Message = "The Fleet node is responding again." + event.Resolved = true + return event +} + func (m *Manager) GetPeer(name string) (*Client, error) { m.mu.RLock() defer m.mu.RUnlock() diff --git a/internal/cluster/manager_events_test.go b/internal/cluster/manager_events_test.go new file mode 100644 index 0000000..335854f --- /dev/null +++ b/internal/cluster/manager_events_test.go @@ -0,0 +1,26 @@ +package cluster + +import ( + "errors" + "testing" +) + +func TestFleetHealthEventOpensAndResolvesOneNodeIncident(t *testing.T) { + failed := fleetHealthEvent("prod2", false, false, errors.New("connection refused")) + if failed == nil || failed.Type != "node.unavailable" || failed.CorrelationKey != "node:prod2" || failed.Resolved { + t.Fatalf("failed event = %#v", failed) + } + if repeated := fleetHealthEvent("prod2", true, false, errors.New("connection refused")); repeated != nil { + t.Fatalf("repeated event = %#v", repeated) + } + recovered := fleetHealthEvent("prod2", true, false, nil) + if recovered == nil || recovered.Type != "node.recovered" || recovered.CorrelationKey != failed.CorrelationKey || !recovered.Resolved { + t.Fatalf("recovered event = %#v", recovered) + } +} + +func TestFleetHealthEventDoesNotAnnounceInitialSuccess(t *testing.T) { + if event := fleetHealthEvent("prod2", false, false, nil); event != nil { + t.Fatalf("event = %#v", event) + } +} diff --git a/internal/cluster/migrations/00001_initial.sql b/internal/cluster/migrations/00001_initial.sql new file mode 100644 index 0000000..f72d808 --- /dev/null +++ b/internal/cluster/migrations/00001_initial.sql @@ -0,0 +1,30 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS peers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT UNIQUE NOT NULL, + url TEXT NOT NULL, + api_key_hash TEXT NOT NULL, + api_key_encrypted TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + last_seen_at DATETIME +); + +CREATE TABLE IF NOT EXISTS invites ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + token_hash TEXT UNIQUE NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + created_by INTEGER NOT NULL, + accepted_peer TEXT, + expires_at DATETIME NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_peers_name ON peers(name); +CREATE INDEX IF NOT EXISTS idx_peers_status ON peers(status); +CREATE INDEX IF NOT EXISTS idx_invites_token_hash ON invites(token_hash); +CREATE INDEX IF NOT EXISTS idx_invites_status ON invites(status); + +-- +goose Down +DROP TABLE IF EXISTS invites; +DROP TABLE IF EXISTS peers; diff --git a/internal/cluster/migrations/00002_peer_policies.sql b/internal/cluster/migrations/00002_peer_policies.sql new file mode 100644 index 0000000..bca1e44 --- /dev/null +++ b/internal/cluster/migrations/00002_peer_policies.sql @@ -0,0 +1,9 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS peer_policies ( + peer_name TEXT PRIMARY KEY, + grants_json TEXT NOT NULL, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- +goose Down +DROP TABLE IF EXISTS peer_policies; diff --git a/internal/docker/stats.go b/internal/docker/stats.go index 6910f94..36d69b4 100644 --- a/internal/docker/stats.go +++ b/internal/docker/stats.go @@ -81,26 +81,71 @@ func GetAllContainerStats() ([]ContainerStats, error) { func listContainerDeploymentLabels() map[string]string { labels := make(map[string]string) - cmd := exec.Command("docker", "ps", "-a", "--format", "{{.ID}}|{{.Names}}|{{.Label \"com.docker.compose.project\"}}") + cmd := exec.Command("docker", "ps", "-a", "--format", "{{.ID}}|{{.Names}}|{{.Label \"com.docker.compose.project\"}}|{{.Label \"flatrun.deployment\"}}") output, err := cmd.Output() if err != nil { log.Printf("warning: failed to list container deployment labels: %v", err) return labels } - for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") { + return parseContainerDeploymentLabels(string(output)) +} + +func parseContainerDeploymentLabels(output string) map[string]string { + labels := make(map[string]string) + for _, line := range strings.Split(strings.TrimSpace(output), "\n") { if line == "" { continue } - parts := strings.SplitN(line, "|", 3) - if len(parts) != 3 || parts[2] == "" { + parts := strings.SplitN(line, "|", 4) + if len(parts) != 4 { continue } - labels[parts[0]] = parts[2] - labels[parts[1]] = parts[2] + deployment := parts[3] + if deployment == "" { + deployment = parts[2] + } + if deployment != "" { + labels[parts[0]] = deployment + labels[parts[1]] = deployment + } } return labels } +func GetManagedDeploymentStats(deployment string) ([]ContainerStats, error) { + if strings.TrimSpace(deployment) == "" { + return []ContainerStats{}, nil + } + ps := exec.Command("docker", "ps", "-q", "--filter", "label=flatrun.deployment="+deployment) + containerIDs, err := ps.Output() + if err != nil { + return nil, err + } + ids := strings.Fields(string(containerIDs)) + if len(ids) == 0 { + return []ContainerStats{}, nil + } + args := append([]string{"stats", "--no-stream", "--format", "{{json .}}"}, ids...) + output, err := exec.Command("docker", args...).Output() + if err != nil { + return nil, err + } + result := make([]ContainerStats, 0, len(ids)) + for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") { + if line == "" { + continue + } + var raw dockerStatsJSON + if err := json.Unmarshal([]byte(line), &raw); err != nil { + continue + } + stat := parseStats(&raw) + stat.DeploymentName = deployment + result = append(result, *stat) + } + return result, nil +} + func GetDeploymentStats(projectName string) ([]ContainerStats, error) { if projectName == "" { return []ContainerStats{}, nil diff --git a/internal/docker/stats_test.go b/internal/docker/stats_test.go new file mode 100644 index 0000000..7db45df --- /dev/null +++ b/internal/docker/stats_test.go @@ -0,0 +1,13 @@ +package docker + +import "testing" + +func TestParseContainerDeploymentLabelsPrefersManagedWorkload(t *testing.T) { + labels := parseContainerDeploymentLabels("abc|shop.1.task|legacy|shop\ndef|db|database|\n") + if labels["abc"] != "shop" || labels["shop.1.task"] != "shop" { + t.Fatalf("managed labels = %#v", labels) + } + if labels["def"] != "database" || labels["db"] != "database" { + t.Fatalf("Compose labels = %#v", labels) + } +} diff --git a/internal/events/events.go b/internal/events/events.go new file mode 100644 index 0000000..c908224 --- /dev/null +++ b/internal/events/events.go @@ -0,0 +1,165 @@ +package events + +import ( + "fmt" + "sync" + "time" +) + +type Severity string + +const ( + SeverityInfo Severity = "info" + SeverityWarning Severity = "warning" + SeverityCritical Severity = "critical" +) + +type Scope struct { + Node string `json:"node,omitempty"` + Deployment string `json:"deployment,omitempty"` + Container string `json:"container,omitempty"` +} + +type Event struct { + ID string `json:"id"` + Source string `json:"source"` + Type string `json:"type"` + Severity Severity `json:"severity"` + Title string `json:"title"` + Message string `json:"message"` + Scope Scope `json:"scope"` + CorrelationKey string `json:"correlation_key,omitempty"` + OccurredAt time.Time `json:"occurred_at"` + Attributes map[string]any `json:"attributes,omitempty"` + Resolved bool `json:"resolved,omitempty"` +} + +type NotificationAction string + +const ( + NotificationNone NotificationAction = "none" + NotificationOpened NotificationAction = "opened" + NotificationUpdated NotificationAction = "updated" + NotificationResolved NotificationAction = "resolved" +) + +type IngestResult struct { + Incident Incident `json:"incident"` + Notification NotificationAction `json:"notification"` +} + +type Correlator struct { + mu sync.Mutex + incidents map[string]Incident + lastNotifiedAt map[string]time.Time + updateInterval time.Duration +} + +func NewCorrelator(updateInterval time.Duration) *Correlator { + return &Correlator{ + incidents: make(map[string]Incident), + lastNotifiedAt: make(map[string]time.Time), + updateInterval: updateInterval, + } +} + +func (c *Correlator) Ingest(event Event) IngestResult { + c.mu.Lock() + defer c.mu.Unlock() + + key := CorrelationKey(event) + incident, exists := c.incidents[key] + action := NotificationNone + if !exists || incident.Status == IncidentResolved { + incident = Incident{ + ID: fmt.Sprintf("%s:%d", key, event.OccurredAt.UnixNano()), + CorrelationKey: key, + Status: IncidentOpen, + Severity: event.Severity, + Title: event.Title, + FirstEventAt: event.OccurredAt, + } + action = NotificationOpened + } + + incident.EventCount++ + incident.LastEventAt = event.OccurredAt + incident.LastEvent = event + if severityRank(event.Severity) > severityRank(incident.Severity) { + incident.Severity = event.Severity + } + if event.Resolved { + incident.Status = IncidentResolved + action = NotificationResolved + } else if action == NotificationNone && c.updateInterval > 0 && event.OccurredAt.Sub(c.lastNotifiedAt[key]) >= c.updateInterval { + action = NotificationUpdated + } + + c.incidents[key] = incident + if action != NotificationNone { + c.lastNotifiedAt[key] = event.OccurredAt + } + return IngestResult{Incident: incident, Notification: action} +} + +func (c *Correlator) List() []Incident { + c.mu.Lock() + defer c.mu.Unlock() + result := make([]Incident, 0, len(c.incidents)) + for _, incident := range c.incidents { + result = append(result, incident) + } + return result +} + +func (c *Correlator) Restore(incidents []Incident) { + c.mu.Lock() + defer c.mu.Unlock() + for _, incident := range incidents { + c.incidents[incident.CorrelationKey] = incident + c.lastNotifiedAt[incident.CorrelationKey] = incident.LastEventAt + } +} + +func severityRank(severity Severity) int { + switch severity { + case SeverityCritical: + return 3 + case SeverityWarning: + return 2 + default: + return 1 + } +} + +type IncidentStatus string + +const ( + IncidentOpen IncidentStatus = "open" + IncidentResolved IncidentStatus = "resolved" +) + +type Incident struct { + ID string `json:"id"` + CorrelationKey string `json:"correlation_key"` + Status IncidentStatus `json:"status"` + Severity Severity `json:"severity"` + Title string `json:"title"` + EventCount int `json:"event_count"` + FirstEventAt time.Time `json:"first_event_at"` + LastEventAt time.Time `json:"last_event_at"` + LastEvent Event `json:"last_event"` +} + +func CorrelationKey(event Event) string { + if event.CorrelationKey != "" { + return event.CorrelationKey + } + if event.Scope.Node != "" { + return "node:" + event.Scope.Node + } + if event.Scope.Deployment != "" { + return "deployment:" + event.Scope.Deployment + } + return event.Source + ":" + event.Type +} diff --git a/internal/events/events_test.go b/internal/events/events_test.go new file mode 100644 index 0000000..4edecf0 --- /dev/null +++ b/internal/events/events_test.go @@ -0,0 +1,45 @@ +package events + +import ( + "testing" + "time" +) + +func TestCorrelatorSuppressesDeploymentFloodDuringNodeIncident(t *testing.T) { + correlator := NewCorrelator(15 * time.Minute) + started := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) + opened := correlator.Ingest(Event{ + Source: "fleet", Type: "node.unavailable", Severity: SeverityCritical, + Title: "prod2 unavailable", Scope: Scope{Node: "prod2"}, OccurredAt: started, + }) + if opened.Notification != NotificationOpened { + t.Fatalf("opened notification = %q", opened.Notification) + } + + for i := 0; i < 14; i++ { + result := correlator.Ingest(Event{ + Source: "capacity", Type: "deployment.unavailable", Severity: SeverityCritical, + Title: "Deployment unavailable", Scope: Scope{Node: "prod2", Deployment: "app"}, + CorrelationKey: "node:prod2", OccurredAt: started.Add(time.Duration(i+1) * 10 * time.Second), + }) + if result.Notification != NotificationNone { + t.Fatalf("event %d notification = %q", i, result.Notification) + } + } + + incidents := correlator.List() + if len(incidents) != 1 || incidents[0].EventCount != 15 { + t.Fatalf("incidents = %#v", incidents) + } +} + +func TestCorrelatorSendsOneRecoveryNotification(t *testing.T) { + correlator := NewCorrelator(15 * time.Minute) + started := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) + correlator.Ingest(Event{Source: "fleet", Type: "node.unavailable", Severity: SeverityCritical, Title: "prod2 unavailable", Scope: Scope{Node: "prod2"}, OccurredAt: started}) + + resolved := correlator.Ingest(Event{Source: "fleet", Type: "node.available", Severity: SeverityInfo, Title: "prod2 recovered", Scope: Scope{Node: "prod2"}, OccurredAt: started.Add(10 * time.Minute), Resolved: true}) + if resolved.Notification != NotificationResolved || resolved.Incident.Status != IncidentResolved { + t.Fatalf("resolved = %#v", resolved) + } +} diff --git a/internal/events/migrations/00001_initial.sql b/internal/events/migrations/00001_initial.sql new file mode 100644 index 0000000..2704123 --- /dev/null +++ b/internal/events/migrations/00001_initial.sql @@ -0,0 +1,20 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS incidents ( + correlation_key TEXT PRIMARY KEY, + payload BLOB NOT NULL, + last_event_at DATETIME NOT NULL +); + +CREATE TABLE IF NOT EXISTS events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + incident_id TEXT NOT NULL, + payload BLOB NOT NULL, + occurred_at DATETIME NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_events_incident_id ON events(incident_id); +CREATE INDEX IF NOT EXISTS idx_events_occurred_at ON events(occurred_at); + +-- +goose Down +DROP TABLE IF EXISTS events; +DROP TABLE IF EXISTS incidents; diff --git a/internal/events/store.go b/internal/events/store.go new file mode 100644 index 0000000..df6f9f8 --- /dev/null +++ b/internal/events/store.go @@ -0,0 +1,106 @@ +package events + +import ( + "context" + "database/sql" + "embed" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/pressly/goose/v3" + _ "modernc.org/sqlite" +) + +//go:embed migrations/*.sql +var migrationFiles embed.FS + +type Store struct { + db *sql.DB +} + +func NewStore(basePath string) (*Store, error) { + dir := filepath.Join(basePath, ".flatrun") + if err := os.MkdirAll(dir, 0700); err != nil { + return nil, err + } + db, err := sql.Open("sqlite", "file:"+filepath.Join(dir, "events.db")+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + if err != nil { + return nil, err + } + store := &Store{db: db} + if err := store.migrate(); err != nil { + _ = db.Close() + return nil, err + } + return store, nil +} + +func (s *Store) migrate() error { + migrations, err := fs.Sub(migrationFiles, "migrations") + if err != nil { + return err + } + provider, err := goose.NewProvider( + goose.DialectSQLite3, + s.db, + migrations, + goose.WithTableName("events_schema_version"), + ) + if err != nil { + return err + } + _, err = provider.Up(context.Background()) + return err +} + +func (s *Store) Record(event Event, incident Incident) error { + eventPayload, err := json.Marshal(event) + if err != nil { + return err + } + incidentPayload, err := json.Marshal(incident) + if err != nil { + return err + } + tx, err := s.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.Exec(`INSERT INTO events (incident_id, payload, occurred_at) VALUES (?, ?, ?)`, incident.ID, eventPayload, event.OccurredAt); err != nil { + return err + } + if _, err := tx.Exec(`INSERT INTO incidents (correlation_key, payload, last_event_at) VALUES (?, ?, ?) + ON CONFLICT(correlation_key) DO UPDATE SET payload = excluded.payload, last_event_at = excluded.last_event_at`, incident.CorrelationKey, incidentPayload, incident.LastEventAt); err != nil { + return err + } + return tx.Commit() +} + +func (s *Store) ListIncidents() ([]Incident, error) { + rows, err := s.db.Query(`SELECT payload FROM incidents ORDER BY last_event_at DESC`) + if err != nil { + return nil, err + } + defer rows.Close() + var incidents []Incident + for rows.Next() { + var payload []byte + if err := rows.Scan(&payload); err != nil { + return nil, err + } + var incident Incident + if err := json.Unmarshal(payload, &incident); err != nil { + return nil, fmt.Errorf("decode incident: %w", err) + } + incidents = append(incidents, incident) + } + return incidents, rows.Err() +} + +func (s *Store) Close() error { + return s.db.Close() +} diff --git a/internal/events/store_test.go b/internal/events/store_test.go new file mode 100644 index 0000000..26280f5 --- /dev/null +++ b/internal/events/store_test.go @@ -0,0 +1,44 @@ +package events + +import ( + "testing" + "time" +) + +func TestStoreRecordsMigrationVersionAndPreservesIncidents(t *testing.T) { + dir := t.TempDir() + store, err := NewStore(dir) + if err != nil { + t.Fatalf("NewStore failed: %v", err) + } + var version int64 + if err := store.db.QueryRow(`SELECT MAX(version_id) FROM events_schema_version WHERE is_applied = 1`).Scan(&version); err != nil { + t.Fatalf("read schema version: %v", err) + } + if version != 1 { + t.Fatalf("schema version = %d, want 1", version) + } + + now := time.Now().UTC() + event := Event{ID: "event-1", Source: "capacity", Type: "host.pressure", Severity: SeverityWarning, OccurredAt: now} + incident := Incident{ID: "incident-1", CorrelationKey: "node:prod-1", Status: IncidentOpen, Severity: SeverityWarning, LastEventAt: now} + if err := store.Record(event, incident); err != nil { + t.Fatalf("Record failed: %v", err) + } + if err := store.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } + + reopened, err := NewStore(dir) + if err != nil { + t.Fatalf("reopen store: %v", err) + } + defer reopened.Close() + incidents, err := reopened.ListIncidents() + if err != nil { + t.Fatalf("ListIncidents failed: %v", err) + } + if len(incidents) != 1 || incidents[0].ID != "incident-1" { + t.Fatalf("incidents = %#v", incidents) + } +} diff --git a/internal/nginx/manager.go b/internal/nginx/manager.go index 5a6f6f8..6fad5b8 100644 --- a/internal/nginx/manager.go +++ b/internal/nginx/manager.go @@ -4,10 +4,12 @@ import ( "bytes" "fmt" "log" + "net" "os" "os/exec" "path/filepath" "sort" + "strconv" "strings" "sync" "text/template" @@ -221,6 +223,61 @@ func (m *Manager) RenderVirtualHost(deployment *models.Deployment) (string, erro return m.generateMultiDomainConfig(deployment) } +type UpstreamBackend struct { + Address string + Healthy bool + Weight int +} + +func (m *Manager) RenderVirtualHostWithBackends(deployment *models.Deployment, backends map[string][]UpstreamBackend) (string, error) { + if deployment.Metadata == nil { + return "", fmt.Errorf("deployment has no metadata") + } + if len(deployment.Metadata.GetDomains()) == 0 { + return "", nil + } + if err := validateUpstreamBackends(backends); err != nil { + return "", err + } + m.mu.RLock() + defer m.mu.RUnlock() + return m.renderMultiDomainConfigWithBackends(deployment, m.keepaliveSupported(), backends) +} + +func validateUpstreamBackends(backends map[string][]UpstreamBackend) error { + for service, entries := range backends { + if strings.TrimSpace(service) == "" || len(entries) == 0 { + return fmt.Errorf("backend service and targets are required") + } + for _, backend := range entries { + host, rawPort, err := net.SplitHostPort(backend.Address) + if err != nil || !safeBackendHost(host) { + return fmt.Errorf("backend address %q is invalid", backend.Address) + } + port, portErr := strconv.Atoi(rawPort) + if portErr != nil || port < 1 || port > 65535 || backend.Weight < 0 { + return fmt.Errorf("backend address %q is invalid", backend.Address) + } + } + } + return nil +} + +func safeBackendHost(host string) bool { + if net.ParseIP(host) != nil { + return true + } + if host == "" { + return false + } + for _, value := range host { + if (value < 'a' || value > 'z') && (value < 'A' || value > 'Z') && (value < '0' || value > '9') && value != '.' && value != '-' && value != '_' { + return false + } + } + return true +} + func (m *Manager) VirtualHostExists(deploymentName string) bool { configFile := filepath.Join(m.configPath, deploymentName+".conf") _, err := os.Stat(configFile) @@ -589,6 +646,10 @@ func (m *Manager) generateMultiDomainConfig(deployment *models.Deployment) (stri } func (m *Manager) renderMultiDomainConfig(deployment *models.Deployment, keepalive bool) (string, error) { + return m.renderMultiDomainConfigWithBackends(deployment, keepalive, nil) +} + +func (m *Manager) renderMultiDomainConfigWithBackends(deployment *models.Deployment, keepalive bool, backendOverrides map[string][]UpstreamBackend) (string, error) { domains := deployment.Metadata.GetDomains() if len(domains) == 0 { return "", fmt.Errorf("no domains configured") @@ -624,7 +685,7 @@ func (m *Manager) renderMultiDomainConfig(deployment *models.Deployment, keepali servers := m.groupDomainsByHost(domains, deployment.Name, m.deploymentComposeContent(deployment)) - upstreams := assignUpstreams(servers, keepalive) + upstreams := assignUpstreams(servers, keepalive, backendOverrides) data := multiRouteTemplateData{ DeploymentName: deployment.Name, @@ -673,8 +734,8 @@ func (m *Manager) renderMultiDomainConfig(deployment *models.Deployment, keepali // service:port target and returns no blocks, so the generated config is // unchanged. With keepalive on it points each location at a shared, deduplicated // upstream block so requests to the same service:port reuse one connection pool. -func assignUpstreams(servers []serverData, keepalive bool) []upstreamData { - if !keepalive { +func assignUpstreams(servers []serverData, keepalive bool, backendOverrides map[string][]UpstreamBackend) []upstreamData { + if !keepalive && len(backendOverrides) == 0 { for si := range servers { for li := range servers[si].Locations { loc := &servers[si].Locations[li] @@ -691,11 +752,20 @@ func assignUpstreams(servers []serverData, keepalive bool) []upstreamData { for li := range servers[si].Locations { loc := &servers[si].Locations[li] target := fmt.Sprintf("%s:%d", loc.Service, loc.ContainerPort) + overrideKey := fmt.Sprintf("%s:%d", loc.RouteService, loc.ContainerPort) + overrides := backendOverrides[overrideKey] + if len(overrides) > 0 { + target = "override:" + overrideKey + } name, ok := byTarget[target] if !ok { name = upstreamNameFor(loc.Service, loc.ContainerPort, used) byTarget[target] = name - upstreams = append(upstreams, upstreamData{Name: name, Target: target}) + targets := []UpstreamBackend{{Address: target, Healthy: true}} + if len(overrides) > 0 { + targets = append([]UpstreamBackend(nil), overrides...) + } + upstreams = append(upstreams, upstreamData{Name: name, Targets: targets, Keepalive: keepalive}) } loc.Upstream = name } @@ -755,10 +825,12 @@ func (m *Manager) groupDomainsByHost(domains []models.DomainConfig, deploymentNa // Route to the service's unique container name, not the bare Compose service // name (which is not unique across deployments sharing the proxy network and // resolves via embedded DNS to an arbitrary deployment's container). - service := d.Service + routeService := d.Service + service := routeService if service == "" { log.Printf("[proxy] warning: domain %q has no service set for deployment %q, falling back to deployment name", d.Domain, deploymentName) service = deploymentName + routeService = deploymentName } else { service = docker.ContainerNameForService(composeContent, deploymentName, service) } @@ -776,6 +848,7 @@ func (m *Manager) groupDomainsByHost(domains []models.DomainConfig, deploymentNa locations = append(locations, locationData{ Path: path, Service: service, + RouteService: routeService, ContainerPort: port, Protocol: "http", StripPrefix: d.StripPrefix, @@ -897,8 +970,9 @@ type multiRouteTemplateData struct { } type upstreamData struct { - Name string - Target string + Name string + Targets []UpstreamBackend + Keepalive bool } type serverData struct { @@ -914,6 +988,7 @@ type serverData struct { type locationData struct { Path string Service string + RouteService string ContainerPort int Protocol string StripPrefix bool @@ -1082,13 +1157,20 @@ server { // container-restart rediscovery that the variable proxy_pass path provided, // while keepalive reuses connections across ordinary requests. const upstreamBlocks = `{{- range .Upstreams}} +{{- $pool := .}} upstream {{.Name}} { +{{- if .Keepalive}} zone {{.Name}} 64k; resolver 127.0.0.11 valid=30s ipv6=off; - server {{.Target}} resolve; +{{- end}} +{{- range .Targets}} + server {{.Address}}{{if .Weight}} weight={{.Weight}}{{end}}{{if not .Healthy}} down{{end}}{{if $pool.Keepalive}} resolve{{end}}; +{{- end}} +{{- if .Keepalive}} keepalive 16; keepalive_timeout 60s; keepalive_requests 1000; +{{- end}} } {{end -}} ` diff --git a/internal/nginx/manager_test.go b/internal/nginx/manager_test.go index aa71e1c..8f43ebe 100644 --- a/internal/nginx/manager_test.go +++ b/internal/nginx/manager_test.go @@ -2398,7 +2398,7 @@ func TestAssignUpstreams(t *testing.T) { {Service: "app", ContainerPort: 80}, }}, } - ups := assignUpstreams(servers, true) + ups := assignUpstreams(servers, true, nil) if len(ups) != 2 { t.Fatalf("expected 2 deduped upstreams, got %d: %+v", len(ups), ups) } @@ -2411,7 +2411,7 @@ func TestAssignUpstreams(t *testing.T) { } off := []serverData{{Locations: []locationData{{Service: "app", ContainerPort: 80}}}} - if ups := assignUpstreams(off, false); ups != nil { + if ups := assignUpstreams(off, false, nil); ups != nil { t.Errorf("expected no upstream blocks when keepalive is off, got %+v", ups) } if off[0].Locations[0].Upstream != "app:80" { @@ -2419,6 +2419,72 @@ func TestAssignUpstreams(t *testing.T) { } } +func TestAssignUpstreamsKeepsOverridePortsDistinct(t *testing.T) { + servers := []serverData{{Locations: []locationData{ + {Service: "web", RouteService: "web", ContainerPort: 8080}, + {Service: "web", RouteService: "web", ContainerPort: 9090}, + }}} + overrides := map[string][]UpstreamBackend{ + "web:8080": {{Address: "10.0.0.8:8080", Healthy: true}}, + "web:9090": {{Address: "10.0.0.9:9090", Healthy: true}}, + } + + upstreams := assignUpstreams(servers, false, overrides) + if len(upstreams) != 2 || upstreams[0].Name == upstreams[1].Name { + t.Fatalf("upstreams = %#v", upstreams) + } + if upstreams[0].Targets[0].Address != "10.0.0.8:8080" || upstreams[1].Targets[0].Address != "10.0.0.9:9090" { + t.Fatalf("upstreams = %#v", upstreams) + } +} + +func TestRenderMultiDomain_BackendOverridesPreserveDeploymentConfig(t *testing.T) { + compose := "name: tenant-a\nservices:\n web:\n container_name: tenant-a-web\n" + m, deployment := newManagerWithDeployment(t, []models.DomainConfig{ + {ID: "d1", Service: "web", ContainerPort: 8080, Domain: "app.example.com", SSL: models.SSLConfig{Enabled: true}}, + }, compose) + deployment.Metadata.Security = &models.DeploymentSecurityConfig{Enabled: true, BlockedIPs: []string{"192.0.2.10"}} + + config, err := m.renderMultiDomainConfigWithBackends(deployment, false, map[string][]UpstreamBackend{ + "web:8080": { + {Address: "10.42.0.8:8080", Healthy: true, Weight: 2}, + {Address: "10.42.1.9:8080", Healthy: false, Weight: 1}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, expected := range []string{ + "listen 443 ssl", "deny 192.0.2.10", "server 10.42.0.8:8080 weight=2;", + "server 10.42.1.9:8080 weight=1 down;", "set $upstream flatrun_tenant-a-web_8080;", + } { + if !strings.Contains(config, expected) { + t.Fatalf("missing %q in:\n%s", expected, config) + } + } + for _, unsupported := range []string{"keepalive 16", " resolve;", "zone flatrun_"} { + if strings.Contains(config, unsupported) { + t.Fatalf("unsupported upstream directive %q remains in:\n%s", unsupported, config) + } + } + if strings.Contains(config, "server tenant-a-web:8080 resolve;") { + t.Fatalf("Compose backend remains in overridden route:\n%s", config) + } +} + +func TestRenderVirtualHostWithBackendsRejectsDirectiveInjection(t *testing.T) { + m, deployment := newManagerWithDeployment(t, []models.DomainConfig{ + {ID: "d1", Service: "web", ContainerPort: 8080, Domain: "app.example.com"}, + }, "services:\n web:\n image: app\n") + + _, err := m.RenderVirtualHostWithBackends(deployment, map[string][]UpstreamBackend{ + "web": {{Address: "10.42.0.8:8080; include /etc/nginx/nginx.conf", Healthy: true}}, + }) + if err == nil { + t.Fatal("unsafe backend address was accepted") + } +} + // Dot, hyphen and underscore are valid in both container and upstream names, so // they survive and keep distinct container names distinct. func TestUpstreamNameForPreservesContainerChars(t *testing.T) { diff --git a/internal/notify/events_test.go b/internal/notify/events_test.go new file mode 100644 index 0000000..f38667e --- /dev/null +++ b/internal/notify/events_test.go @@ -0,0 +1,128 @@ +package notify + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/flatrun/agent/internal/events" +) + +func TestIncidentsFallsBackWhenStoreCannotOpen(t *testing.T) { + base := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(base, []byte("file"), 0600); err != nil { + t.Fatal(err) + } + service := NewService(base) + if incidents := service.Incidents(); len(incidents) != 0 { + t.Fatalf("incidents = %#v", incidents) + } +} + +func TestPublishSendsOneNotificationForCorrelatedFailure(t *testing.T) { + service := NewService(t.TempDir()) + if err := service.Save(Config{Targets: []Target{{ID: "email", Name: "Email", URL: "smtp://test", Enabled: true}}}); err != nil { + t.Fatal(err) + } + var deliveries []string + service.send = func(_, message string) error { + deliveries = append(deliveries, message) + return nil + } + started := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) + + _, err := service.Publish(events.Event{Source: "fleet", Type: "node.unavailable", Severity: events.SeverityCritical, Title: "prod2 unavailable", Message: "The node stopped responding.", Scope: events.Scope{Node: "prod2"}, OccurredAt: started}) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 20; i++ { + _, err = service.Publish(events.Event{Source: "capacity", Type: "deployment.unavailable", Severity: events.SeverityCritical, Title: "Deployment unavailable", Message: "A dependent deployment is unavailable.", Scope: events.Scope{Node: "prod2", Deployment: "app"}, CorrelationKey: "node:prod2", OccurredAt: started.Add(time.Duration(i+1) * 10 * time.Second)}) + if err != nil { + t.Fatal(err) + } + } + if len(deliveries) != 1 { + t.Fatalf("deliveries = %d", len(deliveries)) + } + if !strings.Contains(deliveries[0], "prod2 unavailable") { + t.Fatalf("delivery = %q", deliveries[0]) + } +} + +func TestPublishFiltersTargetsByTopicAndNode(t *testing.T) { + service := NewService(t.TempDir()) + if err := service.Save(Config{Targets: []Target{ + {ID: "prod", URL: "generic+https://prod.example.test", Enabled: true, Topics: []string{"fleet"}, Nodes: []string{"prod2"}}, + {ID: "dev", URL: "generic+https://dev.example.test", Enabled: true, Topics: []string{"fleet"}, Nodes: []string{"dev1"}}, + }}); err != nil { + t.Fatal(err) + } + var targets []string + service.send = func(target, _ string) error { + targets = append(targets, target) + return nil + } + + _, err := service.Publish(events.Event{Source: "fleet", Type: "node.unavailable", Severity: events.SeverityCritical, Title: "prod2 unavailable", Scope: events.Scope{Node: "prod2"}, OccurredAt: time.Now()}) + if err != nil { + t.Fatal(err) + } + if len(targets) != 1 || !strings.Contains(targets[0], "prod.example.test") { + t.Fatalf("targets = %#v", targets) + } +} + +func TestIncidentsSurviveServiceRestart(t *testing.T) { + basePath := t.TempDir() + service := NewService(basePath) + started := time.Date(2026, 8, 22, 12, 0, 0, 0, time.UTC) + if _, err := service.Publish(events.Event{Source: "fleet", Type: "node.unavailable", Severity: events.SeverityCritical, Title: "prod2 unavailable", Scope: events.Scope{Node: "prod2"}, OccurredAt: started}); err != nil { + t.Fatal(err) + } + if err := service.Close(); err != nil { + t.Fatal(err) + } + + restarted := NewService(basePath) + defer restarted.Close() + incidents := restarted.Incidents() + if len(incidents) != 1 || incidents[0].CorrelationKey != "node:prod2" { + t.Fatalf("incidents = %#v", incidents) + } + result, err := restarted.Publish(events.Event{Source: "capacity", Type: "deployment.unavailable", Severity: events.SeverityCritical, Title: "app unavailable", Scope: events.Scope{Node: "prod2", Deployment: "app"}, CorrelationKey: "node:prod2", OccurredAt: started.Add(time.Minute)}) + if err != nil { + t.Fatal(err) + } + if result.Incident.EventCount != 2 || result.Notification != events.NotificationNone { + t.Fatalf("result = %#v", result) + } +} + +func TestPublishUsesMatchingRules(t *testing.T) { + service := NewService(t.TempDir()) + defer service.Close() + if err := service.Save(Config{ + Targets: []Target{ + {ID: "email", URL: "generic+https://email.example.test", Enabled: true}, + {ID: "webhook", URL: "generic+https://hook.example.test", Enabled: true}, + }, + Rules: []Rule{{ID: "critical", Enabled: true, Topics: []string{"fleet"}, Severities: []events.Severity{events.SeverityCritical}, TargetIDs: []string{"email"}}}, + }); err != nil { + t.Fatal(err) + } + var targets []string + service.send = func(target, _ string) error { + targets = append(targets, target) + return nil + } + + _, err := service.Publish(events.Event{Source: "fleet", Type: "node.unavailable", Severity: events.SeverityCritical, Title: "prod2 unavailable", Scope: events.Scope{Node: "prod2"}, OccurredAt: time.Now()}) + if err != nil { + t.Fatal(err) + } + if len(targets) != 1 || !strings.Contains(targets[0], "email.example.test") { + t.Fatalf("targets = %#v", targets) + } +} diff --git a/internal/notify/notify.go b/internal/notify/notify.go index d1d5c00..f8af11d 100644 --- a/internal/notify/notify.go +++ b/internal/notify/notify.go @@ -12,7 +12,9 @@ import ( "path/filepath" "strings" "sync" + "time" + "github.com/flatrun/agent/internal/events" "github.com/nicholas-fedor/shoutrrr" "github.com/nicholas-fedor/shoutrrr/pkg/router" "gopkg.in/yaml.v3" @@ -26,10 +28,14 @@ const MaskedURL = "********" // Target is one delivery destination. URL is a shoutrrr service URL, e.g. // "smtp://user:pass@host:587/?from=x&to=y" or "generic+https://example.com/hook". type Target struct { - ID string `yaml:"id" json:"id"` - Name string `yaml:"name" json:"name"` - URL string `yaml:"url" json:"url"` - Enabled bool `yaml:"enabled" json:"enabled"` + ID string `yaml:"id" json:"id"` + Name string `yaml:"name" json:"name"` + URL string `yaml:"url" json:"url"` + Enabled bool `yaml:"enabled" json:"enabled"` + Topics []string `yaml:"topics,omitempty" json:"topics,omitempty"` + Severities []events.Severity `yaml:"severities,omitempty" json:"severities,omitempty"` + Nodes []string `yaml:"nodes,omitempty" json:"nodes,omitempty"` + Deployments []string `yaml:"deployments,omitempty" json:"deployments,omitempty"` } // MarshalJSON masks the credential-bearing URL. YAML persistence does not use @@ -40,25 +46,196 @@ func (t Target) MarshalJSON() ([]byte, error) { if masked.URL != "" { masked.URL = MaskedURL } - return json.Marshal(masked) + return json.Marshal(struct { + alias + Kind string `json:"kind"` + }{alias: masked, Kind: targetKind(t.URL)}) +} + +func targetKind(rawURL string) string { + switch { + case strings.HasPrefix(rawURL, "smtp://"): + return "email" + case strings.HasPrefix(rawURL, "generic+"): + return "webhook" + default: + return "custom" + } } // Config is the persisted notification settings. type Config struct { Targets []Target `yaml:"targets" json:"targets"` + Rules []Rule `yaml:"rules,omitempty" json:"rules,omitempty"` +} + +type Rule struct { + ID string `yaml:"id" json:"id"` + Name string `yaml:"name" json:"name"` + Enabled bool `yaml:"enabled" json:"enabled"` + Topics []string `yaml:"topics,omitempty" json:"topics,omitempty"` + EventTypes []string `yaml:"event_types,omitempty" json:"event_types,omitempty"` + Severities []events.Severity `yaml:"severities,omitempty" json:"severities,omitempty"` + Nodes []string `yaml:"nodes,omitempty" json:"nodes,omitempty"` + Deployments []string `yaml:"deployments,omitempty" json:"deployments,omitempty"` + Notifications []events.NotificationAction `yaml:"notifications,omitempty" json:"notifications,omitempty"` + TargetIDs []string `yaml:"target_ids" json:"target_ids"` } // Service loads/saves targets and delivers messages. type Service struct { - path string - mu sync.RWMutex - send func(url, message string) error // overridable in tests + path string + mu sync.RWMutex + send func(url, message string) error // overridable in tests + events *events.Correlator + store *events.Store + storeErr error } func NewService(basePath string) *Service { - return &Service{ - path: filepath.Join(basePath, ".flatrun", "notifications.yml"), + service := &Service{ + path: filepath.Join(basePath, ".flatrun", "notifications.yml"), + events: events.NewCorrelator(15 * time.Minute), + } + service.store, service.storeErr = events.NewStore(basePath) + if service.storeErr == nil { + incidents, err := service.store.ListIncidents() + if err != nil { + service.storeErr = err + } else { + service.events.Restore(incidents) + } + } + return service +} + +func (s *Service) Publish(event events.Event) (events.IngestResult, error) { + if s.storeErr != nil { + return events.IngestResult{}, s.storeErr + } + result := s.events.Ingest(event) + if err := s.store.Record(event, result.Incident); err != nil { + return result, err + } + if result.Notification == events.NotificationNone { + return result, nil + } + + kind := KindNegative + message := event.Message + switch result.Notification { + case events.NotificationResolved: + kind = KindPositive + message = fmt.Sprintf("Resolved after %d related events. %s", result.Incident.EventCount, event.Message) + case events.NotificationUpdated: + message = fmt.Sprintf("%d related events are grouped in this incident. %s", result.Incident.EventCount, event.Message) + } + if event.Severity == events.SeverityInfo && result.Notification != events.NotificationResolved { + kind = KindGeneric + } + + notification := Notification{ + Kind: kind, + Title: event.Title, + Message: message, + Panels: []Panel{{ + Title: "Incident ID", + Value: result.Incident.ID, + Detail: fmt.Sprintf("Source: %s", event.Source), + }}, + } + return result, s.deliverEvent(event, result.Notification, notification) +} + +func (s *Service) Incidents() []events.Incident { + if s.store == nil || s.storeErr != nil { + return s.events.List() + } + incidents, err := s.store.ListIncidents() + if err != nil { + return s.events.List() + } + return incidents +} + +func (s *Service) Close() error { + if s.store == nil { + return nil + } + return s.store.Close() +} + +func (s *Service) deliverEvent(event events.Event, action events.NotificationAction, notification Notification) error { + cfg := s.Load() + selected := matchingRuleTargets(cfg.Rules, event, action) + var firstErr error + for _, target := range cfg.Targets { + if !target.Enabled || target.URL == "" || !targetMatches(target, event) || (len(cfg.Rules) > 0 && !selected[target.ID]) { + continue + } + if err := s.deliver(target.URL, notification); err != nil && firstErr == nil { + firstErr = err + } } + return firstErr +} + +func matchingRuleTargets(rules []Rule, event events.Event, action events.NotificationAction) map[string]bool { + selected := make(map[string]bool) + for _, rule := range rules { + if !rule.Enabled || !matchesString(rule.Topics, event.Source) || !matchesString(rule.EventTypes, event.Type) || + !matchesSeverity(rule.Severities, event.Severity) || !matchesString(rule.Nodes, event.Scope.Node) || + !matchesString(rule.Deployments, event.Scope.Deployment) || !matchesAction(rule.Notifications, action) { + continue + } + for _, id := range rule.TargetIDs { + selected[id] = true + } + } + return selected +} + +func matchesAction(filter []events.NotificationAction, value events.NotificationAction) bool { + if len(filter) == 0 { + return true + } + for _, candidate := range filter { + if candidate == value { + return true + } + } + return false +} + +func targetMatches(target Target, event events.Event) bool { + return matchesString(target.Topics, event.Source) && + matchesSeverity(target.Severities, event.Severity) && + matchesString(target.Nodes, event.Scope.Node) && + matchesString(target.Deployments, event.Scope.Deployment) +} + +func matchesString(filter []string, value string) bool { + if len(filter) == 0 { + return true + } + for _, candidate := range filter { + if candidate == value { + return true + } + } + return false +} + +func matchesSeverity(filter []events.Severity, value events.Severity) bool { + if len(filter) == 0 { + return true + } + for _, candidate := range filter { + if candidate == value { + return true + } + } + return false } func (s *Service) Load() Config { @@ -91,6 +268,7 @@ func (s *Service) Save(cfg Config) error { // unchanged must not overwrite the real URL with the mask. func (s *Service) Update(cfg Config) error { stored := s.Load() + cfg.Rules = stored.Rules byID := make(map[string]string, len(stored.Targets)) for _, t := range stored.Targets { byID[t.ID] = t.URL @@ -103,6 +281,12 @@ func (s *Service) Update(cfg Config) error { return s.Save(cfg) } +func (s *Service) UpdateRules(rules []Rule) error { + cfg := s.Load() + cfg.Rules = rules + return s.Save(cfg) +} + // Test sends a message to a single URL, so an admin can verify a target before saving it. func (s *Service) Test(url string) error { if url == "" { @@ -123,6 +307,18 @@ func (s *Service) NotifyTargets(title, message string, ids []string) error { return s.NotifyNotificationTargets(Notification{Title: title, Message: message}, ids) } +func (s *Service) TestTarget(id string) error { + for _, target := range s.Load().Targets { + if target.ID == id { + if !target.Enabled { + return fmt.Errorf("target is disabled") + } + return s.deliver(target.URL, Notification{Title: "FlatRun test", Message: "Notification delivery is working."}) + } + } + return fmt.Errorf("target not found") +} + func (s *Service) NotifyNotificationTargets(notification Notification, ids []string) error { cfg := s.Load() var only map[string]bool diff --git a/internal/orchestrator/k3s.go b/internal/orchestrator/k3s.go new file mode 100644 index 0000000..7979584 --- /dev/null +++ b/internal/orchestrator/k3s.go @@ -0,0 +1,360 @@ +package orchestrator + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net" + "os/exec" + "sort" + "strconv" + "strings" +) + +type kubectlRunner interface { + Run(context.Context, []byte, ...string) ([]byte, error) +} + +type commandKubectl struct { + binary string +} + +func (r commandKubectl) Run(ctx context.Context, input []byte, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, r.binary, args...) + cmd.Stdin = bytes.NewReader(input) + return cmd.CombinedOutput() +} + +type K3sProvider struct { + runner kubectlRunner + kubeconfig string + namespace string +} + +func NewK3sProvider(kubeconfig, namespace string) *K3sProvider { + if namespace == "" { + namespace = "default" + } + return &K3sProvider{ + runner: commandKubectl{binary: "kubectl"}, + kubeconfig: strings.TrimSpace(kubeconfig), + namespace: strings.TrimSpace(namespace), + } +} + +func (p *K3sProvider) ID() ProviderID { return ProviderK3s } + +func (p *K3sProvider) Ready(ctx context.Context) error { + if _, err := p.run(ctx, nil, "cluster-info"); err != nil { + return fmt.Errorf("K3s is not available: %w", err) + } + return nil +} + +func (p *K3sProvider) Validate(_ context.Context, workload Workload) error { + if strings.TrimSpace(workload.ID) == "" { + return fmt.Errorf("Workload ID is required") + } + if strings.TrimSpace(workload.Image) == "" { + return fmt.Errorf("Workload image is required") + } + if workload.Replicas < 1 { + return fmt.Errorf("Replicas must be at least one") + } + if workload.Stateful && workload.Replicas > 1 { + return fmt.Errorf("Stateful workloads cannot use multiple replicas without a storage policy") + } + if workload.Port < 0 || workload.Port > 65535 { + return fmt.Errorf("Workload port is invalid") + } + return nil +} + +func (p *K3sProvider) Apply(ctx context.Context, workload Workload) (Status, error) { + if err := p.Validate(ctx, workload); err != nil { + return Status{}, err + } + manifest, err := json.Marshal(k3sManifest(workload)) + if err != nil { + return Status{}, err + } + if _, err := p.run(ctx, manifest, "apply", "-f", "-"); err != nil { + return Status{}, fmt.Errorf("apply K3s workload: %w", err) + } + return p.Status(ctx, workload.ID) +} + +func (p *K3sProvider) Resize(ctx context.Context, id string, resources Resources) (Status, error) { + patch := map[string]any{"spec": map[string]any{"template": map[string]any{"spec": map[string]any{"containers": []any{map[string]any{ + "name": id, "resources": k3sResources(resources), + }}}}}} + encoded, _ := json.Marshal(patch) + if _, err := p.run(ctx, nil, "patch", "deployment", id, "--type", "strategic", "-p", string(encoded)); err != nil { + return Status{}, fmt.Errorf("resize K3s workload: %w", err) + } + return p.Status(ctx, id) +} + +func (p *K3sProvider) Scale(ctx context.Context, id string, replicas int) (Status, error) { + if replicas < 0 { + return Status{}, fmt.Errorf("Replicas cannot be negative") + } + if _, err := p.run(ctx, nil, "scale", "deployment", id, "--replicas", strconv.Itoa(replicas)); err != nil { + return Status{}, fmt.Errorf("scale K3s workload: %w", err) + } + return p.Status(ctx, id) +} + +func (p *K3sProvider) Status(ctx context.Context, id string) (Status, error) { + deploymentRaw, err := p.run(ctx, nil, "get", "deployment", id, "-o", "json") + if err != nil { + return Status{}, fmt.Errorf("inspect K3s workload: %w", err) + } + var deployment struct { + Metadata struct { + Labels map[string]string `json:"labels"` + } `json:"metadata"` + Spec struct { + Replicas int `json:"replicas"` + } `json:"spec"` + Status struct { + Available int `json:"availableReplicas"` + } `json:"status"` + } + if err := json.Unmarshal(deploymentRaw, &deployment); err != nil { + return Status{}, fmt.Errorf("decode K3s workload: %w", err) + } + podsRaw, err := p.run(ctx, nil, "get", "pods", "-l", "flatrun.workload="+id, "-o", "json") + if err != nil { + return Status{}, fmt.Errorf("list K3s workload pods: %w", err) + } + var pods struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + Node string `json:"nodeName"` + } `json:"spec"` + Status struct { + IP string `json:"podIP"` + Phase string `json:"phase"` + Conditions []struct{ Type, Status string } `json:"conditions"` + } `json:"status"` + } `json:"items"` + } + if err := json.Unmarshal(podsRaw, &pods); err != nil { + return Status{}, fmt.Errorf("decode K3s workload pods: %w", err) + } + status := Status{Workload: id, Desired: deployment.Spec.Replicas, Available: deployment.Status.Available} + port := deployment.Metadata.Labels["flatrun.port"] + for _, pod := range pods.Items { + ready := false + for _, condition := range pod.Status.Conditions { + if condition.Type == "Ready" && condition.Status == "True" { + ready = true + } + } + address := "" + if pod.Status.IP != "" && port != "" { + address = net.JoinHostPort(pod.Status.IP, port) + } + status.Instances = append(status.Instances, Instance{ID: pod.Metadata.Name, Node: pod.Spec.Node, Address: address, Healthy: pod.Status.Phase == "Running", Ready: ready}) + } + return status, nil +} + +func (p *K3sProvider) Metrics(ctx context.Context, id string) (Usage, error) { + deploymentRaw, err := p.run(ctx, nil, "get", "deployment", id, "-o", "json") + if err != nil { + return Usage{}, fmt.Errorf("inspect K3s workload resources: %w", err) + } + var deployment struct { + Spec struct { + Replicas int `json:"replicas"` + Template struct { + Spec struct { + Containers []struct { + Resources struct { + Limits map[string]string `json:"limits"` + } `json:"resources"` + } `json:"containers"` + } `json:"spec"` + } `json:"template"` + } `json:"spec"` + } + if err := json.Unmarshal(deploymentRaw, &deployment); err != nil { + return Usage{}, fmt.Errorf("decode K3s workload resources: %w", err) + } + if deployment.Spec.Replicas < 1 || len(deployment.Spec.Template.Spec.Containers) == 0 { + return Usage{}, fmt.Errorf("K3s workload has no measurable replicas") + } + limits := deployment.Spec.Template.Spec.Containers[0].Resources.Limits + cpuLimit, err := parseCPUQuantity(limits["cpu"]) + if err != nil || cpuLimit <= 0 { + return Usage{}, fmt.Errorf("K3s workload needs a CPU limit for autoscaling") + } + memoryLimit, err := parseMemoryQuantity(limits["memory"]) + if err != nil || memoryLimit <= 0 { + return Usage{}, fmt.Errorf("K3s workload needs a memory limit for autoscaling") + } + metricsRaw, err := p.run(ctx, nil, "get", "--raw", "/apis/metrics.k8s.io/v1beta1/namespaces/"+p.namespace+"/pods?labelSelector=flatrun.workload%3D"+id) + if err != nil { + return Usage{}, fmt.Errorf("read K3s Metrics API: %w", err) + } + var metrics struct { + Items []struct { + Containers []struct { + Usage map[string]string `json:"usage"` + } `json:"containers"` + } `json:"items"` + } + if err := json.Unmarshal(metricsRaw, &metrics); err != nil { + return Usage{}, fmt.Errorf("decode K3s Metrics API: %w", err) + } + var cpuUsed float64 + var memoryUsed float64 + for _, pod := range metrics.Items { + for _, container := range pod.Containers { + cpu, cpuErr := parseCPUQuantity(container.Usage["cpu"]) + memory, memoryErr := parseMemoryQuantity(container.Usage["memory"]) + if cpuErr != nil || memoryErr != nil { + return Usage{}, fmt.Errorf("decode K3s container usage") + } + cpuUsed += cpu + memoryUsed += memory + } + } + replicas := float64(deployment.Spec.Replicas) + return Usage{ + CPUPercent: cpuUsed / (cpuLimit * replicas) * 100, + MemoryPercent: memoryUsed / (memoryLimit * replicas) * 100, + }, nil +} + +func parseCPUQuantity(value string) (float64, error) { + multiplier := 1.0 + for suffix, factor := range map[string]float64{"n": 1e-9, "u": 1e-6, "m": 1e-3} { + if strings.HasSuffix(value, suffix) { + multiplier = factor + value = strings.TrimSuffix(value, suffix) + break + } + } + parsed, err := strconv.ParseFloat(value, 64) + return parsed * multiplier, err +} + +func parseMemoryQuantity(value string) (float64, error) { + multipliers := map[string]float64{ + "Ki": 1 << 10, "Mi": 1 << 20, "Gi": 1 << 30, "Ti": 1 << 40, + "K": 1e3, "M": 1e6, "G": 1e9, "T": 1e12, + } + multiplier := 1.0 + for suffix, factor := range multipliers { + if strings.HasSuffix(value, suffix) { + multiplier = factor + value = strings.TrimSuffix(value, suffix) + break + } + } + parsed, err := strconv.ParseFloat(value, 64) + return parsed * multiplier, err +} + +func (p *K3sProvider) Remove(ctx context.Context, id string) error { + if _, err := p.run(ctx, nil, "delete", "deployment", id, "--ignore-not-found=true"); err != nil { + return fmt.Errorf("remove K3s workload: %w", err) + } + return nil +} + +func (p *K3sProvider) run(ctx context.Context, input []byte, args ...string) ([]byte, error) { + base := make([]string, 0, len(args)+4) + if p.kubeconfig != "" { + base = append(base, "--kubeconfig", p.kubeconfig) + } + base = append(base, "--namespace", p.namespace) + output, err := p.runner.Run(ctx, input, append(base, args...)...) + if err != nil { + message := strings.TrimSpace(string(output)) + if message != "" { + return nil, fmt.Errorf("%s", message) + } + return nil, err + } + return output, nil +} + +func k3sManifest(workload Workload) map[string]any { + labels := map[string]string{"app.kubernetes.io/name": workload.ID, "flatrun.workload": workload.ID} + for key, value := range workload.Labels { + labels[key] = value + } + if workload.Port > 0 { + labels["flatrun.port"] = strconv.Itoa(workload.Port) + } + container := map[string]any{"name": workload.ID, "image": workload.Image, "resources": k3sResources(workload.Resources)} + if len(workload.Environment) > 0 { + keys := make([]string, 0, len(workload.Environment)) + for key := range workload.Environment { + keys = append(keys, key) + } + sort.Strings(keys) + environment := make([]any, 0, len(keys)) + for _, key := range keys { + environment = append(environment, map[string]string{"name": key, "value": workload.Environment[key]}) + } + container["env"] = environment + } + if len(workload.Entrypoint) > 0 { + container["command"] = workload.Entrypoint + } + if len(workload.Command) > 0 { + container["args"] = workload.Command + } + if workload.WorkingDir != "" { + container["workingDir"] = workload.WorkingDir + } + if workload.Port > 0 { + container["ports"] = []any{map[string]any{"containerPort": workload.Port}} + } + deployment := map[string]any{ + "apiVersion": "apps/v1", "kind": "Deployment", + "metadata": map[string]any{"name": workload.ID, "labels": labels}, + "spec": map[string]any{"replicas": workload.Replicas, "selector": map[string]any{"matchLabels": map[string]string{"flatrun.workload": workload.ID}}, "template": map[string]any{ + "metadata": map[string]any{"labels": labels}, "spec": map[string]any{"containers": []any{container}}, + }}, + } + items := []any{deployment} + if workload.Port > 0 { + items = append(items, map[string]any{ + "apiVersion": "v1", "kind": "Service", + "metadata": map[string]any{"name": workload.ID, "labels": labels}, + "spec": map[string]any{ + "selector": map[string]string{"flatrun.workload": workload.ID}, + "ports": []any{map[string]any{"name": "http", "port": workload.Port, "targetPort": workload.Port}}, + }, + }) + } + return map[string]any{"apiVersion": "v1", "kind": "List", "items": items} +} + +func k3sResources(resources Resources) map[string]any { + requests := map[string]string{} + limits := map[string]string{} + if resources.CPURequest > 0 { + requests["cpu"] = fmt.Sprintf("%gm", resources.CPURequest*1000) + } + if resources.MemoryRequest > 0 { + requests["memory"] = strconv.FormatUint(resources.MemoryRequest, 10) + } + if resources.CPULimit > 0 { + limits["cpu"] = fmt.Sprintf("%gm", resources.CPULimit*1000) + } + if resources.MemoryLimit > 0 { + limits["memory"] = strconv.FormatUint(resources.MemoryLimit, 10) + } + return map[string]any{"requests": requests, "limits": limits} +} diff --git a/internal/orchestrator/k3s_test.go b/internal/orchestrator/k3s_test.go new file mode 100644 index 0000000..103a995 --- /dev/null +++ b/internal/orchestrator/k3s_test.go @@ -0,0 +1,121 @@ +package orchestrator + +import ( + "context" + "encoding/json" + "reflect" + "testing" +) + +type kubectlCall struct { + input []byte + args []string +} + +type fakeKubectl struct { + calls []kubectlCall + responses [][]byte +} + +func (f *fakeKubectl) Run(_ context.Context, input []byte, args ...string) ([]byte, error) { + f.calls = append(f.calls, kubectlCall{input: input, args: append([]string(nil), args...)}) + if len(f.responses) == 0 { + return nil, nil + } + response := f.responses[0] + f.responses = f.responses[1:] + return response, nil +} + +func TestK3sApplyUsesConfiguredClusterAndNamespace(t *testing.T) { + runner := &fakeKubectl{responses: [][]byte{ + nil, + []byte(`{"metadata":{"labels":{"flatrun.port":"8080"}},"spec":{"replicas":2},"status":{"availableReplicas":2}}`), + []byte(`{"items":[]}`), + }} + provider := NewK3sProvider("/etc/rancher/k3s.yaml", "apps") + provider.runner = runner + + status, err := provider.Apply(context.Background(), Workload{ID: "shop", Image: "shop:1", Port: 8080, Replicas: 2, Environment: map[string]string{"APP_ENV": "production"}, Command: []string{"serve"}}) + if err != nil { + t.Fatal(err) + } + if status.Desired != 2 || status.Available != 2 { + t.Fatalf("status = %#v", status) + } + want := []string{"--kubeconfig", "/etc/rancher/k3s.yaml", "--namespace", "apps", "apply", "-f", "-"} + if !reflect.DeepEqual(runner.calls[0].args, want) { + t.Fatalf("apply args = %#v", runner.calls[0].args) + } + var manifest map[string]any + if err := json.Unmarshal(runner.calls[0].input, &manifest); err != nil { + t.Fatal(err) + } + if manifest["kind"] != "List" { + t.Fatalf("manifest = %#v", manifest) + } + items := manifest["items"].([]any) + deployment := items[0].(map[string]any) + service := items[1].(map[string]any) + if deployment["kind"] != "Deployment" || service["kind"] != "Service" { + t.Fatalf("items = %#v", items) + } + spec := deployment["spec"].(map[string]any) + template := spec["template"].(map[string]any) + podSpec := template["spec"].(map[string]any) + container := podSpec["containers"].([]any)[0].(map[string]any) + if len(container["env"].([]any)) != 1 || len(container["args"].([]any)) != 1 { + t.Fatalf("container = %#v", container) + } +} + +func TestK3sStatusReturnsRoutableReadyPods(t *testing.T) { + runner := &fakeKubectl{responses: [][]byte{ + []byte(`{"metadata":{"labels":{"flatrun.port":"8080"}},"spec":{"replicas":1},"status":{"availableReplicas":1}}`), + []byte(`{"items":[{"metadata":{"name":"shop-a"},"spec":{"nodeName":"prod2"},"status":{"podIP":"10.42.0.8","phase":"Running","conditions":[{"type":"Ready","status":"True"}]}}]}`), + }} + provider := NewK3sProvider("", "apps") + provider.runner = runner + + status, err := provider.Status(context.Background(), "shop") + if err != nil { + t.Fatal(err) + } + if len(status.Instances) != 1 || status.Instances[0].Address != "10.42.0.8:8080" || !status.Instances[0].Ready { + t.Fatalf("status = %#v", status) + } +} + +func TestK3sResizeUsesStrategicMerge(t *testing.T) { + runner := &fakeKubectl{responses: [][]byte{ + nil, + []byte(`{"metadata":{"labels":{"flatrun.port":"8080"}},"spec":{"replicas":1},"status":{"availableReplicas":1}}`), + []byte(`{"items":[]}`), + }} + provider := NewK3sProvider("", "apps") + provider.runner = runner + + if _, err := provider.Resize(context.Background(), "shop", Resources{CPURequest: 0.25, CPULimit: 0.5, MemoryRequest: 128 << 20, MemoryLimit: 256 << 20}); err != nil { + t.Fatal(err) + } + wantPrefix := []string{"--namespace", "apps", "patch", "deployment", "shop", "--type", "strategic", "-p"} + if len(runner.calls) == 0 || !reflect.DeepEqual(runner.calls[0].args[:len(wantPrefix)], wantPrefix) { + t.Fatalf("patch args = %#v", runner.calls[0].args) + } +} + +func TestK3sMetricsReturnsLimitUtilization(t *testing.T) { + runner := &fakeKubectl{responses: [][]byte{ + []byte(`{"spec":{"replicas":2,"template":{"spec":{"containers":[{"resources":{"limits":{"cpu":"500m","memory":"256Mi"}}}]}}}}`), + []byte(`{"items":[{"containers":[{"usage":{"cpu":"250m","memory":"128Mi"}}]},{"containers":[{"usage":{"cpu":"500m","memory":"256Mi"}}]}]}`), + }} + provider := NewK3sProvider("", "apps") + provider.runner = runner + usage, err := provider.Metrics(context.Background(), "shop") + if err != nil { + t.Fatal(err) + } + if usage.CPUPercent != 75 || usage.MemoryPercent != 75 { + t.Fatalf("usage = %#v", usage) + } +} diff --git a/internal/orchestrator/provider.go b/internal/orchestrator/provider.go new file mode 100644 index 0000000..5e22f8c --- /dev/null +++ b/internal/orchestrator/provider.go @@ -0,0 +1,81 @@ +package orchestrator + +import "context" + +type ProviderID string + +const ( + ProviderStandalone ProviderID = "standalone" + ProviderSwarm ProviderID = "swarm" + ProviderK3s ProviderID = "k3s" +) + +type Resources struct { + CPURequest float64 `json:"cpu_request,omitempty"` + CPULimit float64 `json:"cpu_limit,omitempty"` + MemoryRequest uint64 `json:"memory_request,omitempty"` + MemoryLimit uint64 `json:"memory_limit,omitempty"` +} + +type Health struct { + Path string `json:"path,omitempty"` + IntervalSeconds int `json:"interval_seconds,omitempty"` + TimeoutSeconds int `json:"timeout_seconds,omitempty"` + HealthyThreshold int `json:"healthy_threshold,omitempty"` +} + +type Placement struct { + Constraints []string `json:"constraints,omitempty"` + MaxReplicasPerNode uint64 `json:"max_replicas_per_node,omitempty"` +} + +type Workload struct { + ID string `json:"id"` + Image string `json:"image"` + Port int `json:"port,omitempty"` + Replicas int `json:"replicas"` + Resources Resources `json:"resources"` + Health Health `json:"health"` + Labels map[string]string `json:"labels,omitempty"` + Environment map[string]string `json:"environment,omitempty"` + Entrypoint []string `json:"entrypoint,omitempty"` + Command []string `json:"command,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` + Networks []string `json:"networks,omitempty"` + Placement Placement `json:"placement,omitempty"` + Stateful bool `json:"stateful"` +} + +type Instance struct { + ID string `json:"id"` + Node string `json:"node"` + Address string `json:"address"` + Healthy bool `json:"healthy"` + Ready bool `json:"ready"` +} + +type Status struct { + Workload string `json:"workload"` + Desired int `json:"desired"` + Available int `json:"available"` + Instances []Instance `json:"instances"` +} + +type Usage struct { + CPUPercent float64 `json:"cpu_percent"` + MemoryPercent float64 `json:"memory_percent"` +} + +type MetricsProvider interface { + Metrics(context.Context, string) (Usage, error) +} + +type Provider interface { + ID() ProviderID + Validate(context.Context, Workload) error + Apply(context.Context, Workload) (Status, error) + Resize(context.Context, string, Resources) (Status, error) + Scale(context.Context, string, int) (Status, error) + Status(context.Context, string) (Status, error) + Remove(context.Context, string) error +} diff --git a/internal/orchestrator/swarm.go b/internal/orchestrator/swarm.go new file mode 100644 index 0000000..4266665 --- /dev/null +++ b/internal/orchestrator/swarm.go @@ -0,0 +1,313 @@ +package orchestrator + +import ( + "context" + "fmt" + "net" + "slices" + "sort" + "strconv" + "strings" + + "github.com/containerd/errdefs" + "github.com/moby/moby/api/types/swarm" + "github.com/moby/moby/client" +) + +type swarmClient interface { + SwarmInspect(context.Context, client.SwarmInspectOptions) (client.SwarmInspectResult, error) + Info(context.Context, client.InfoOptions) (client.SystemInfoResult, error) + NodeInspect(context.Context, string, client.NodeInspectOptions) (client.NodeInspectResult, error) + NodeUpdate(context.Context, string, client.NodeUpdateOptions) (client.NodeUpdateResult, error) + ServiceCreate(context.Context, client.ServiceCreateOptions) (client.ServiceCreateResult, error) + ServiceInspect(context.Context, string, client.ServiceInspectOptions) (client.ServiceInspectResult, error) + ServiceUpdate(context.Context, string, client.ServiceUpdateOptions) (client.ServiceUpdateResult, error) + ServiceRemove(context.Context, string, client.ServiceRemoveOptions) (client.ServiceRemoveResult, error) + TaskList(context.Context, client.TaskListOptions) (client.TaskListResult, error) +} + +type NodeIdentity struct { + ID string `json:"id"` + Hostname string `json:"hostname"` + ClusterID string `json:"cluster_id"` +} + +type SwarmProvider struct { + client swarmClient +} + +func NewSwarmProvider(client swarmClient) *SwarmProvider { + return &SwarmProvider{client: client} +} + +func NewSwarmProviderFromEnv() (*SwarmProvider, error) { + cli, err := client.New(client.FromEnv) + if err != nil { + return nil, err + } + return NewSwarmProvider(cli), nil +} + +func (p *SwarmProvider) Ready(ctx context.Context) error { + if _, err := p.client.SwarmInspect(ctx, client.SwarmInspectOptions{}); err != nil { + return fmt.Errorf("Docker Swarm is not available: %w", err) + } + return nil +} + +func (p *SwarmProvider) Close() error { + if closer, ok := p.client.(interface{ Close() error }); ok { + return closer.Close() + } + return nil +} + +func (p *SwarmProvider) EnsureLocalNodeLabel(ctx context.Context, key, value string) (NodeIdentity, error) { + identity, inspected, err := p.localNode(ctx) + if err != nil { + return NodeIdentity{}, err + } + if inspected.Node.Spec.Labels == nil { + inspected.Node.Spec.Labels = make(map[string]string) + } + if inspected.Node.Spec.Labels[key] != value { + inspected.Node.Spec.Labels[key] = value + if _, err := p.client.NodeUpdate(ctx, inspected.Node.ID, client.NodeUpdateOptions{Version: inspected.Node.Version, Spec: inspected.Node.Spec}); err != nil { + return NodeIdentity{}, fmt.Errorf("label local Swarm node: %w", err) + } + } + return identity, nil +} + +func (p *SwarmProvider) LocalNodeIdentity(ctx context.Context) (NodeIdentity, error) { + identity, _, err := p.localNode(ctx) + return identity, err +} + +func (p *SwarmProvider) localNode(ctx context.Context) (NodeIdentity, client.NodeInspectResult, error) { + cluster, err := p.client.SwarmInspect(ctx, client.SwarmInspectOptions{}) + if err != nil { + return NodeIdentity{}, client.NodeInspectResult{}, fmt.Errorf("inspect Docker Swarm: %w", err) + } + info, err := p.client.Info(ctx, client.InfoOptions{}) + if err != nil { + return NodeIdentity{}, client.NodeInspectResult{}, fmt.Errorf("inspect Docker host: %w", err) + } + if info.Info.Swarm.NodeID == "" { + return NodeIdentity{}, client.NodeInspectResult{}, fmt.Errorf("Docker host is not a Swarm node") + } + inspected, err := p.client.NodeInspect(ctx, info.Info.Swarm.NodeID, client.NodeInspectOptions{}) + if err != nil { + return NodeIdentity{}, client.NodeInspectResult{}, fmt.Errorf("inspect local Swarm node: %w", err) + } + return NodeIdentity{ID: inspected.Node.ID, Hostname: inspected.Node.Description.Hostname, ClusterID: cluster.Swarm.ID}, inspected, nil +} + +func (p *SwarmProvider) Placement(ctx context.Context, id string) (Placement, error) { + service, err := p.client.ServiceInspect(ctx, id, client.ServiceInspectOptions{}) + if err != nil { + return Placement{}, err + } + if service.Service.Spec.TaskTemplate.Placement == nil { + return Placement{}, nil + } + return Placement{ + Constraints: slices.Clone(service.Service.Spec.TaskTemplate.Placement.Constraints), + MaxReplicasPerNode: service.Service.Spec.TaskTemplate.Placement.MaxReplicas, + }, nil +} + +func (p *SwarmProvider) ID() ProviderID { + return ProviderSwarm +} + +func (p *SwarmProvider) Validate(_ context.Context, workload Workload) error { + if strings.TrimSpace(workload.ID) == "" { + return fmt.Errorf("Workload ID is required") + } + if strings.TrimSpace(workload.Image) == "" { + return fmt.Errorf("Workload image is required") + } + if workload.Replicas < 1 { + return fmt.Errorf("Replicas must be at least one") + } + if workload.Stateful && workload.Replicas > 1 { + return fmt.Errorf("Stateful workloads cannot use multiple replicas without a storage policy") + } + if workload.Port < 0 || workload.Port > 65535 { + return fmt.Errorf("Workload port is invalid") + } + return nil +} + +func (p *SwarmProvider) Apply(ctx context.Context, workload Workload) (Status, error) { + if err := p.Validate(ctx, workload); err != nil { + return Status{}, err + } + inspected, err := p.client.ServiceInspect(ctx, workload.ID, client.ServiceInspectOptions{}) + if err != nil { + if !errdefs.IsNotFound(err) { + return Status{}, fmt.Errorf("inspect Swarm service: %w", err) + } + if _, err := p.client.ServiceCreate(ctx, client.ServiceCreateOptions{Spec: swarmSpec(workload)}); err != nil { + return Status{}, fmt.Errorf("create Swarm service: %w", err) + } + return p.Status(ctx, workload.ID) + } + if _, err := p.client.ServiceUpdate(ctx, inspected.Service.ID, client.ServiceUpdateOptions{ + Version: inspected.Service.Version, + Spec: swarmSpec(workload), + }); err != nil { + return Status{}, fmt.Errorf("update Swarm service: %w", err) + } + return p.Status(ctx, workload.ID) +} + +func (p *SwarmProvider) Resize(ctx context.Context, id string, resources Resources) (Status, error) { + service, err := p.client.ServiceInspect(ctx, id, client.ServiceInspectOptions{}) + if err != nil { + return Status{}, err + } + service.Service.Spec.TaskTemplate.Resources = swarmResources(resources) + if _, err := p.client.ServiceUpdate(ctx, service.Service.ID, client.ServiceUpdateOptions{ + Version: service.Service.Version, + Spec: service.Service.Spec, + }); err != nil { + return Status{}, fmt.Errorf("resize Swarm service: %w", err) + } + return p.Status(ctx, id) +} + +func (p *SwarmProvider) Scale(ctx context.Context, id string, replicas int) (Status, error) { + if replicas < 0 { + return Status{}, fmt.Errorf("Replicas cannot be negative") + } + service, err := p.client.ServiceInspect(ctx, id, client.ServiceInspectOptions{}) + if err != nil { + return Status{}, err + } + count := uint64(replicas) + service.Service.Spec.Mode = swarm.ServiceMode{Replicated: &swarm.ReplicatedService{Replicas: &count}} + if _, err := p.client.ServiceUpdate(ctx, service.Service.ID, client.ServiceUpdateOptions{ + Version: service.Service.Version, + Spec: service.Service.Spec, + }); err != nil { + return Status{}, fmt.Errorf("scale Swarm service: %w", err) + } + return p.Status(ctx, id) +} + +func (p *SwarmProvider) Status(ctx context.Context, id string) (Status, error) { + service, err := p.client.ServiceInspect(ctx, id, client.ServiceInspectOptions{}) + if err != nil { + return Status{}, err + } + tasks, err := p.client.TaskList(ctx, client.TaskListOptions{Filters: make(client.Filters).Add("service", service.Service.ID)}) + if err != nil { + return Status{}, fmt.Errorf("list Swarm tasks: %w", err) + } + status := Status{Workload: id} + port := servicePort(service.Service.Spec) + if service.Service.Spec.Mode.Replicated != nil && service.Service.Spec.Mode.Replicated.Replicas != nil { + status.Desired = int(*service.Service.Spec.Mode.Replicated.Replicas) + } + for _, task := range tasks.Items { + if task.DesiredState == swarm.TaskStateShutdown || task.DesiredState == swarm.TaskStateRemove { + continue + } + running := task.Status.State == swarm.TaskStateRunning + if running { + status.Available++ + } + status.Instances = append(status.Instances, Instance{ + ID: task.ID, Node: task.NodeID, Address: taskAddress(task, port), Healthy: running, Ready: running, + }) + } + return status, nil +} + +func (p *SwarmProvider) Remove(ctx context.Context, id string) error { + if _, err := p.client.ServiceRemove(ctx, id, client.ServiceRemoveOptions{}); err != nil && !errdefs.IsNotFound(err) { + return fmt.Errorf("remove Swarm service: %w", err) + } + return nil +} + +func swarmSpec(workload Workload) swarm.ServiceSpec { + replicas := uint64(workload.Replicas) + labels := make(map[string]string, len(workload.Labels)+1) + for key, value := range workload.Labels { + labels[key] = value + } + labels["flatrun.workload"] = workload.ID + if workload.Port > 0 { + labels["flatrun.port"] = strconv.Itoa(workload.Port) + } + return swarm.ServiceSpec{ + Annotations: swarm.Annotations{Name: workload.ID, Labels: labels}, + TaskTemplate: swarm.TaskSpec{ + ContainerSpec: &swarm.ContainerSpec{Image: workload.Image, Labels: labels, Env: workloadEnvironment(workload.Environment), Command: workload.Entrypoint, Args: workload.Command, Dir: workload.WorkingDir}, + Resources: swarmResources(workload.Resources), + Networks: swarmNetworks(workload.Networks), + Placement: &swarm.Placement{ + Constraints: workload.Placement.Constraints, + MaxReplicas: workload.Placement.MaxReplicasPerNode, + }, + }, + Mode: swarm.ServiceMode{Replicated: &swarm.ReplicatedService{Replicas: &replicas}}, + } +} + +func swarmNetworks(networks []string) []swarm.NetworkAttachmentConfig { + result := make([]swarm.NetworkAttachmentConfig, 0, len(networks)) + for _, network := range networks { + if strings.TrimSpace(network) != "" { + result = append(result, swarm.NetworkAttachmentConfig{Target: network}) + } + } + return result +} + +func workloadEnvironment(environment map[string]string) []string { + keys := make([]string, 0, len(environment)) + for key := range environment { + keys = append(keys, key) + } + sort.Strings(keys) + values := make([]string, 0, len(keys)) + for _, key := range keys { + values = append(values, key+"="+environment[key]) + } + return values +} + +func servicePort(spec swarm.ServiceSpec) int { + value := spec.Annotations.Labels["flatrun.port"] + port, _ := strconv.Atoi(value) + return port +} + +func taskAddress(task swarm.Task, port int) string { + if port == 0 { + return "" + } + for _, attachment := range task.NetworksAttachments { + for _, address := range attachment.Addresses { + return net.JoinHostPort(address.Addr().String(), strconv.Itoa(port)) + } + } + return "" +} + +func swarmResources(resources Resources) *swarm.ResourceRequirements { + return &swarm.ResourceRequirements{ + Limits: &swarm.Limit{ + NanoCPUs: int64(resources.CPULimit * 1_000_000_000), + MemoryBytes: int64(resources.MemoryLimit), + }, + Reservations: &swarm.Resources{ + NanoCPUs: int64(resources.CPURequest * 1_000_000_000), + MemoryBytes: int64(resources.MemoryRequest), + }, + } +} diff --git a/internal/orchestrator/swarm_test.go b/internal/orchestrator/swarm_test.go new file mode 100644 index 0000000..3d3c60f --- /dev/null +++ b/internal/orchestrator/swarm_test.go @@ -0,0 +1,145 @@ +package orchestrator + +import ( + "context" + "net/netip" + "testing" + + "github.com/containerd/errdefs" + "github.com/moby/moby/api/types/swarm" + "github.com/moby/moby/api/types/system" + "github.com/moby/moby/client" +) + +type fakeSwarmClient struct { + service swarm.Service + tasks []swarm.Task + created *swarm.ServiceSpec + updated *swarm.ServiceSpec + node swarm.Node +} + +func (f *fakeSwarmClient) Info(_ context.Context, _ client.InfoOptions) (client.SystemInfoResult, error) { + return client.SystemInfoResult{Info: system.Info{Swarm: swarm.Info{NodeID: f.node.ID}}}, nil +} + +func (f *fakeSwarmClient) NodeInspect(_ context.Context, _ string, _ client.NodeInspectOptions) (client.NodeInspectResult, error) { + return client.NodeInspectResult{Node: f.node}, nil +} + +func (f *fakeSwarmClient) NodeUpdate(_ context.Context, _ string, options client.NodeUpdateOptions) (client.NodeUpdateResult, error) { + f.node.Spec = options.Spec + return client.NodeUpdateResult{}, nil +} + +func (f *fakeSwarmClient) SwarmInspect(_ context.Context, _ client.SwarmInspectOptions) (client.SwarmInspectResult, error) { + return client.SwarmInspectResult{Swarm: swarm.Swarm{ClusterInfo: swarm.ClusterInfo{ID: "swarm-1"}}}, nil +} + +func (f *fakeSwarmClient) ServiceCreate(_ context.Context, options client.ServiceCreateOptions) (client.ServiceCreateResult, error) { + f.created = &options.Spec + f.service = swarm.Service{ID: "service-1", Spec: options.Spec} + return client.ServiceCreateResult{ID: f.service.ID}, nil +} + +func (f *fakeSwarmClient) ServiceInspect(_ context.Context, _ string, _ client.ServiceInspectOptions) (client.ServiceInspectResult, error) { + if f.service.ID == "" { + return client.ServiceInspectResult{}, errdefs.ErrNotFound + } + return client.ServiceInspectResult{Service: f.service}, nil +} + +func (f *fakeSwarmClient) ServiceUpdate(_ context.Context, _ string, options client.ServiceUpdateOptions) (client.ServiceUpdateResult, error) { + f.updated = &options.Spec + f.service.Spec = options.Spec + return client.ServiceUpdateResult{}, nil +} + +func (f *fakeSwarmClient) ServiceRemove(_ context.Context, _ string, _ client.ServiceRemoveOptions) (client.ServiceRemoveResult, error) { + f.service = swarm.Service{} + return client.ServiceRemoveResult{}, nil +} + +func (f *fakeSwarmClient) TaskList(_ context.Context, _ client.TaskListOptions) (client.TaskListResult, error) { + return client.TaskListResult{Items: f.tasks}, nil +} + +func TestSwarmProviderCreatesReplicatedService(t *testing.T) { + client := &fakeSwarmClient{tasks: []swarm.Task{{ + ID: "task-1", NodeID: "node-1", DesiredState: swarm.TaskStateRunning, + Status: swarm.TaskStatus{State: swarm.TaskStateRunning}, + NetworksAttachments: []swarm.NetworkAttachment{{Addresses: []netip.Prefix{netip.MustParsePrefix("10.0.0.12/24")}}}, + }}} + provider := NewSwarmProvider(client) + + status, err := provider.Apply(context.Background(), Workload{ + ID: "shop", Image: "example/shop:1", Port: 8080, Replicas: 1, + Resources: Resources{CPURequest: 0.5, CPULimit: 1, MemoryRequest: 256 << 20, MemoryLimit: 512 << 20}, + Environment: map[string]string{"APP_ENV": "production"}, Entrypoint: []string{"/app/entrypoint"}, Command: []string{"serve"}, WorkingDir: "/app", + Networks: []string{"proxy"}, Placement: Placement{Constraints: []string{"node.hostname==prod-1"}}, + }) + if err != nil { + t.Fatalf("Apply failed: %v", err) + } + if client.created == nil || client.created.Annotations.Name != "shop" { + t.Fatalf("created spec = %#v", client.created) + } + if client.created.TaskTemplate.Resources.Limits.NanoCPUs != 1_000_000_000 { + t.Fatalf("CPU limit = %d", client.created.TaskTemplate.Resources.Limits.NanoCPUs) + } + container := client.created.TaskTemplate.ContainerSpec + if len(container.Env) != 1 || container.Env[0] != "APP_ENV=production" || container.Command[0] != "/app/entrypoint" || container.Args[0] != "serve" || container.Dir != "/app" { + t.Fatalf("container spec = %#v", container) + } + if len(client.created.TaskTemplate.Networks) != 1 || client.created.TaskTemplate.Networks[0].Target != "proxy" { + t.Fatalf("networks = %#v", client.created.TaskTemplate.Networks) + } + if len(client.created.TaskTemplate.Placement.Constraints) != 1 || client.created.TaskTemplate.Placement.Constraints[0] != "node.hostname==prod-1" { + t.Fatalf("placement = %#v", client.created.TaskTemplate.Placement) + } + if status.Desired != 1 || status.Available != 1 || len(status.Instances) != 1 { + t.Fatalf("status = %#v", status) + } + if status.Instances[0].Address != "10.0.0.12:8080" { + t.Fatalf("instance address = %q", status.Instances[0].Address) + } +} + +func TestSwarmProviderScalesExistingService(t *testing.T) { + replicas := uint64(1) + client := &fakeSwarmClient{service: swarm.Service{ + ID: "service-1", + Spec: swarm.ServiceSpec{Mode: swarm.ServiceMode{Replicated: &swarm.ReplicatedService{Replicas: &replicas}}}, + }} + provider := NewSwarmProvider(client) + + status, err := provider.Scale(context.Background(), "shop", 3) + if err != nil { + t.Fatalf("Scale failed: %v", err) + } + if client.updated == nil || *client.updated.Mode.Replicated.Replicas != 3 { + t.Fatalf("updated spec = %#v", client.updated) + } + if status.Desired != 3 { + t.Fatalf("status = %#v", status) + } +} + +func TestSwarmProviderRejectsUnsafeStatefulReplication(t *testing.T) { + provider := NewSwarmProvider(&fakeSwarmClient{}) + err := provider.Validate(context.Background(), Workload{ID: "database", Image: "postgres:17", Replicas: 2, Stateful: true}) + if err == nil { + t.Fatal("stateful replication should require a storage policy") + } +} + +func TestSwarmProviderLabelsLocalNodeForCapacityGrant(t *testing.T) { + client := &fakeSwarmClient{node: swarm.Node{ID: "node-1", Description: swarm.NodeDescription{Hostname: "prod-1"}}} + identity, err := NewSwarmProvider(client).EnsureLocalNodeLabel(context.Background(), "flatrun.capacity.origin", "true") + if err != nil { + t.Fatal(err) + } + if identity.ID != "node-1" || identity.Hostname != "prod-1" || identity.ClusterID != "swarm-1" || client.node.Spec.Labels["flatrun.capacity.origin"] != "true" { + t.Fatalf("identity = %#v, node = %#v", identity, client.node) + } +} diff --git a/internal/routing/adapters.go b/internal/routing/adapters.go new file mode 100644 index 0000000..f103c02 --- /dev/null +++ b/internal/routing/adapters.go @@ -0,0 +1,184 @@ +package routing + +import ( + "context" + "fmt" + "net" + "regexp" + "sort" + "strconv" + "strings" + "sync" + + "gopkg.in/yaml.v3" +) + +type ConfigWriter interface { + Apply(context.Context, string, string, []byte) error + Remove(context.Context, string) error +} + +type routeProvider struct { + id ProviderID + writer ConfigWriter + render func(Route) ([]byte, error) + mu sync.RWMutex + routes map[string]Route +} + +func NewNginxProvider(writer ConfigWriter) Provider { + return &routeProvider{id: ProviderNginx, writer: writer, render: renderNginx, routes: make(map[string]Route)} +} + +func NewTraefikProvider(writer ConfigWriter) Provider { + return &routeProvider{id: ProviderTraefik, writer: writer, render: renderTraefik, routes: make(map[string]Route)} +} + +func (p *routeProvider) ID() ProviderID { + return p.id +} + +func (p *routeProvider) Validate(_ context.Context, route Route) error { + return validateRoute(route) +} + +func validateRoute(route Route) error { + if !safeRouteID.MatchString(route.ID) { + return fmt.Errorf("Route ID is invalid") + } + if strings.TrimSpace(route.Domain) == "" || strings.ContainsAny(route.Domain, " /\\") { + return fmt.Errorf("Route domain is invalid") + } + if route.Protocol != "http" && route.Protocol != "https" { + return fmt.Errorf("Route protocol must be http or https") + } + if route.Path != "" && !strings.HasPrefix(route.Path, "/") { + return fmt.Errorf("Route path must start with a slash") + } + if len(route.Backends) == 0 { + return fmt.Errorf("Route needs at least one backend") + } + for _, backend := range route.Backends { + if strings.TrimSpace(backend.ID) == "" { + return fmt.Errorf("Backend ID is required") + } + host, portValue, err := net.SplitHostPort(backend.Address) + if err != nil { + return fmt.Errorf("Backend %q address is invalid: %w", backend.ID, err) + } + if net.ParseIP(host) == nil && !safeHost.MatchString(host) { + return fmt.Errorf("Backend %q host is invalid", backend.ID) + } + port, err := strconv.Atoi(portValue) + if err != nil || port < 1 || port > 65535 { + return fmt.Errorf("Backend %q port is invalid", backend.ID) + } + if backend.Weight < 0 { + return fmt.Errorf("Backend %q weight cannot be negative", backend.ID) + } + } + return nil +} + +func (p *routeProvider) Reconcile(ctx context.Context, route Route) error { + if err := p.Validate(ctx, route); err != nil { + return err + } + content, err := p.render(route) + if err != nil { + return err + } + if err := p.writer.Apply(ctx, route.ID, string(p.id), content); err != nil { + return err + } + p.mu.Lock() + p.routes[route.ID] = cloneRoute(route) + p.mu.Unlock() + return nil +} + +func (p *routeProvider) Drain(ctx context.Context, routeID, backendID string) error { + p.mu.RLock() + route, ok := p.routes[routeID] + route = cloneRoute(route) + p.mu.RUnlock() + if !ok { + return fmt.Errorf("Route %q is not managed", routeID) + } + found := false + for index := range route.Backends { + if route.Backends[index].ID == backendID { + route.Backends[index].Healthy = false + found = true + } + } + if !found { + return fmt.Errorf("Backend %q is not part of route %q", backendID, routeID) + } + return p.Reconcile(ctx, route) +} + +func (p *routeProvider) Remove(ctx context.Context, routeID string) error { + if err := p.writer.Remove(ctx, routeID); err != nil { + return err + } + p.mu.Lock() + delete(p.routes, routeID) + p.mu.Unlock() + return nil +} + +var safeID = regexp.MustCompile(`[^a-zA-Z0-9_.-]+`) +var safeHost = regexp.MustCompile(`^[a-zA-Z0-9.-]+$`) +var safeRouteID = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_.-]*$`) + +func renderNginx(route Route) ([]byte, error) { + name := "flatrun_" + safeID.ReplaceAllString(route.ID, "_") + path := route.Path + if path == "" { + path = "/" + } + backends := append([]Backend(nil), route.Backends...) + sort.Slice(backends, func(i, j int) bool { return backends[i].ID < backends[j].ID }) + var output strings.Builder + fmt.Fprintf(&output, "upstream %s {\n", name) + for _, backend := range backends { + fmt.Fprintf(&output, " server %s", backend.Address) + if backend.Weight > 0 { + fmt.Fprintf(&output, " weight=%d", backend.Weight) + } + if !backend.Healthy { + output.WriteString(" down") + } + output.WriteString(";\n") + } + output.WriteString("}\n\nserver {\n listen 80;\n") + fmt.Fprintf(&output, " server_name %s;\n\n", route.Domain) + fmt.Fprintf(&output, " location %s {\n", path) + fmt.Fprintf(&output, " proxy_pass %s://%s;\n", route.Protocol, name) + output.WriteString(" proxy_set_header Host $host;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n }\n}\n") + return []byte(output.String()), nil +} + +func renderTraefik(route Route) ([]byte, error) { + path := route.Path + if path == "" { + path = "/" + } + servers := make([]map[string]string, 0, len(route.Backends)) + for _, backend := range route.Backends { + if backend.Healthy { + servers = append(servers, map[string]string{"url": route.Protocol + "://" + backend.Address}) + } + } + config := map[string]any{"http": map[string]any{ + "routers": map[string]any{route.ID: map[string]any{ + "rule": fmt.Sprintf("Host(`%s`) && PathPrefix(`%s`)", route.Domain, path), + "service": route.ID, + }}, + "services": map[string]any{route.ID: map[string]any{ + "loadBalancer": map[string]any{"servers": servers}, + }}, + }} + return yaml.Marshal(config) +} diff --git a/internal/routing/adapters_test.go b/internal/routing/adapters_test.go new file mode 100644 index 0000000..d30384c --- /dev/null +++ b/internal/routing/adapters_test.go @@ -0,0 +1,99 @@ +package routing + +import ( + "context" + "strings" + "testing" +) + +type memoryConfigWriter struct { + format string + content string + removed string +} + +func (w *memoryConfigWriter) Apply(_ context.Context, _ string, format string, content []byte) error { + w.format = format + w.content = string(content) + return nil +} + +func (w *memoryConfigWriter) Remove(_ context.Context, id string) error { + w.removed = id + return nil +} + +func testRoute() Route { + return Route{ + ID: "shop", Domain: "shop.example.com", Path: "/", Protocol: "http", + Backends: []Backend{ + {ID: "replica-b", Address: "10.0.0.12:8080", Healthy: true, Weight: 1}, + {ID: "replica-a", Address: "10.0.0.11:8080", Healthy: true, Weight: 2}, + }, + } +} + +func TestNginxProviderRendersWeightedUpstreamAndDrainsBackend(t *testing.T) { + writer := &memoryConfigWriter{} + provider := NewNginxProvider(writer) + if err := provider.Reconcile(context.Background(), testRoute()); err != nil { + t.Fatalf("Reconcile failed: %v", err) + } + for _, expected := range []string{ + "upstream flatrun_shop", + "server 10.0.0.11:8080 weight=2;", + "server 10.0.0.12:8080 weight=1;", + "proxy_pass http://flatrun_shop;", + } { + if !strings.Contains(writer.content, expected) { + t.Fatalf("missing %q in:\n%s", expected, writer.content) + } + } + if err := provider.Drain(context.Background(), "shop", "replica-b"); err != nil { + t.Fatalf("Drain failed: %v", err) + } + if !strings.Contains(writer.content, "server 10.0.0.12:8080 weight=1 down;") { + t.Fatalf("drained backend remains active:\n%s", writer.content) + } +} + +func TestNginxProviderOwnsReconciledBackendSlice(t *testing.T) { + writer := &memoryConfigWriter{} + provider := NewNginxProvider(writer) + route := testRoute() + if err := provider.Reconcile(context.Background(), route); err != nil { + t.Fatal(err) + } + route.Backends[0].Healthy = false + if err := provider.Drain(context.Background(), "shop", "replica-a"); err != nil { + t.Fatal(err) + } + if strings.Contains(writer.content, "server 10.0.0.12:8080 weight=1 down;") { + t.Fatalf("caller mutation changed the stored route:\n%s", writer.content) + } +} + +func TestTraefikProviderExcludesDrainedBackend(t *testing.T) { + writer := &memoryConfigWriter{} + provider := NewTraefikProvider(writer) + if err := provider.Reconcile(context.Background(), testRoute()); err != nil { + t.Fatalf("Reconcile failed: %v", err) + } + if writer.format != "traefik" || !strings.Contains(writer.content, "Host(`shop.example.com`) && PathPrefix(`/`)") { + t.Fatalf("Traefik config:\n%s", writer.content) + } + if err := provider.Drain(context.Background(), "shop", "replica-a"); err != nil { + t.Fatalf("Drain failed: %v", err) + } + if strings.Contains(writer.content, "10.0.0.11:8080") || !strings.Contains(writer.content, "10.0.0.12:8080") { + t.Fatalf("Traefik drain config:\n%s", writer.content) + } +} + +func TestRoutingProvidersRejectUnsafeAddresses(t *testing.T) { + route := testRoute() + route.Backends[0].Address = "10.0.0.12:8080; include /etc/passwd" + if err := NewNginxProvider(&memoryConfigWriter{}).Validate(context.Background(), route); err == nil { + t.Fatal("invalid backend address should be rejected") + } +} diff --git a/internal/routing/k3s_ingress.go b/internal/routing/k3s_ingress.go new file mode 100644 index 0000000..aab8eb2 --- /dev/null +++ b/internal/routing/k3s_ingress.go @@ -0,0 +1,144 @@ +package routing + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net" + "os/exec" + "strconv" + "strings" + "sync" +) + +type KubernetesRunner interface { + Run(context.Context, []byte, ...string) ([]byte, error) +} + +type kubectlCommand struct{} + +func (kubectlCommand) Run(ctx context.Context, input []byte, args ...string) ([]byte, error) { + command := exec.CommandContext(ctx, "kubectl", args...) + command.Stdin = bytes.NewReader(input) + return command.CombinedOutput() +} + +type K3sIngressProvider struct { + runner KubernetesRunner + kubeconfig string + namespace string + mu sync.RWMutex + routes map[string]Route +} + +func NewK3sIngressProvider(kubeconfig, namespace string, restored ...Route) Provider { + if strings.TrimSpace(namespace) == "" { + namespace = "default" + } + provider := &K3sIngressProvider{ + runner: kubectlCommand{}, kubeconfig: strings.TrimSpace(kubeconfig), + namespace: strings.TrimSpace(namespace), routes: make(map[string]Route), + } + for _, route := range restored { + provider.routes[route.ID] = cloneRoute(route) + } + return provider +} + +func NewK3sIngressProviderWithRunner(kubeconfig, namespace string, runner KubernetesRunner) Provider { + provider := NewK3sIngressProvider(kubeconfig, namespace).(*K3sIngressProvider) + provider.runner = runner + return provider +} + +func (p *K3sIngressProvider) ID() ProviderID { return ProviderTraefik } + +func (p *K3sIngressProvider) Validate(_ context.Context, route Route) error { + if err := validateRoute(route); err != nil { + return err + } + if strings.TrimSpace(route.Service) == "" { + return fmt.Errorf("Route service is required") + } + return nil +} + +func (p *K3sIngressProvider) Reconcile(ctx context.Context, route Route) error { + if err := p.Validate(ctx, route); err != nil { + return err + } + _, portValue, err := net.SplitHostPort(route.Backends[0].Address) + if err != nil { + return fmt.Errorf("resolve route service port: %w", err) + } + port, err := strconv.Atoi(portValue) + if err != nil { + return fmt.Errorf("resolve route service port: %w", err) + } + path := route.Path + if path == "" { + path = "/" + } + manifest := map[string]any{ + "apiVersion": "networking.k8s.io/v1", "kind": "Ingress", + "metadata": map[string]any{"name": route.ID, "labels": map[string]string{"flatrun.route": route.ID}}, + "spec": map[string]any{"rules": []any{map[string]any{ + "host": route.Domain, "http": map[string]any{"paths": []any{map[string]any{ + "path": path, "pathType": "Prefix", "backend": map[string]any{"service": map[string]any{ + "name": route.Service, "port": map[string]any{"number": port}, + }}, + }}}, + }}}, + } + content, err := json.Marshal(manifest) + if err != nil { + return err + } + if _, err := p.run(ctx, content, "apply", "-f", "-"); err != nil { + return fmt.Errorf("apply K3s ingress: %w", err) + } + p.mu.Lock() + p.routes[route.ID] = cloneRoute(route) + p.mu.Unlock() + return nil +} + +func (p *K3sIngressProvider) Drain(_ context.Context, routeID, backendID string) error { + p.mu.RLock() + _, exists := p.routes[routeID] + p.mu.RUnlock() + if !exists { + return fmt.Errorf("Route %q is not managed", routeID) + } + return nil +} + +func (p *K3sIngressProvider) Remove(ctx context.Context, routeID string) error { + if !safeRouteID.MatchString(routeID) { + return fmt.Errorf("Route ID is invalid") + } + if _, err := p.run(ctx, nil, "delete", "ingress", routeID, "--ignore-not-found=true"); err != nil { + return fmt.Errorf("remove K3s ingress: %w", err) + } + p.mu.Lock() + delete(p.routes, routeID) + p.mu.Unlock() + return nil +} + +func (p *K3sIngressProvider) run(ctx context.Context, input []byte, args ...string) ([]byte, error) { + base := make([]string, 0, len(args)+4) + if p.kubeconfig != "" { + base = append(base, "--kubeconfig", p.kubeconfig) + } + base = append(base, "--namespace", p.namespace) + output, err := p.runner.Run(ctx, input, append(base, args...)...) + if err != nil { + if message := strings.TrimSpace(string(output)); message != "" { + return nil, fmt.Errorf("%s", message) + } + return nil, err + } + return output, nil +} diff --git a/internal/routing/k3s_ingress_test.go b/internal/routing/k3s_ingress_test.go new file mode 100644 index 0000000..4d263c9 --- /dev/null +++ b/internal/routing/k3s_ingress_test.go @@ -0,0 +1,37 @@ +package routing + +import ( + "context" + "encoding/json" + "testing" +) + +type kubernetesRunnerStub struct { + input []byte + args []string +} + +func (r *kubernetesRunnerStub) Run(_ context.Context, input []byte, args ...string) ([]byte, error) { + r.input = append([]byte(nil), input...) + r.args = append([]string(nil), args...) + return nil, nil +} + +func TestK3sIngressRoutesThroughWorkloadService(t *testing.T) { + runner := &kubernetesRunnerStub{} + provider := NewK3sIngressProviderWithRunner("/etc/rancher/k3s.yaml", "apps", runner) + err := provider.Reconcile(context.Background(), Route{ + ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http", + Backends: []Backend{{ID: "pod-one", Address: "10.42.0.8:8080", Healthy: true}}, + }) + if err != nil { + t.Fatal(err) + } + var manifest map[string]any + if err := json.Unmarshal(runner.input, &manifest); err != nil { + t.Fatal(err) + } + if manifest["kind"] != "Ingress" || runner.args[0] != "--kubeconfig" || runner.args[4] != "apply" { + t.Fatalf("manifest = %#v, args = %#v", manifest, runner.args) + } +} diff --git a/internal/routing/managed_nginx.go b/internal/routing/managed_nginx.go new file mode 100644 index 0000000..0c5501b --- /dev/null +++ b/internal/routing/managed_nginx.go @@ -0,0 +1,147 @@ +package routing + +import ( + "context" + "fmt" + "net" + "sync" + + "github.com/flatrun/agent/internal/nginx" + "github.com/flatrun/agent/pkg/models" +) + +type DeploymentSource interface { + GetDeployment(string) (*models.Deployment, error) +} + +type ManagedNginx interface { + RenderVirtualHost(*models.Deployment) (string, error) + RenderVirtualHostWithBackends(*models.Deployment, map[string][]nginx.UpstreamBackend) (string, error) + GetVirtualHost(string) (string, error) + WriteVirtualHost(string, string) error + TestConfig() error + Reload() error +} + +type managedNginxProvider struct { + manager ManagedNginx + deployments DeploymentSource + mu sync.RWMutex + routes map[string]Route +} + +func NewManagedNginxProvider(manager ManagedNginx, deployments DeploymentSource, restored ...Route) Provider { + routes := make(map[string]Route, len(restored)) + for _, route := range restored { + routes[route.ID] = cloneRoute(route) + } + return &managedNginxProvider{manager: manager, deployments: deployments, routes: routes} +} + +func (p *managedNginxProvider) ID() ProviderID { return ProviderNginx } + +func (p *managedNginxProvider) Validate(_ context.Context, route Route) error { + if route.Service == "" { + return fmt.Errorf("Route service is required") + } + return validateRoute(route) +} + +func (p *managedNginxProvider) Reconcile(ctx context.Context, route Route) error { + if err := ctx.Err(); err != nil { + return err + } + if err := p.Validate(ctx, route); err != nil { + return err + } + deployment, err := p.deployments.GetDeployment(route.ID) + if err != nil { + return fmt.Errorf("load deployment for route: %w", err) + } + backends := make([]nginx.UpstreamBackend, 0, len(route.Backends)) + for _, backend := range route.Backends { + backends = append(backends, nginx.UpstreamBackend{Address: backend.Address, Healthy: backend.Healthy, Weight: backend.Weight}) + } + _, port, err := net.SplitHostPort(route.Backends[0].Address) + if err != nil { + return fmt.Errorf("resolve managed backend port: %w", err) + } + content, err := p.manager.RenderVirtualHostWithBackends(deployment, map[string][]nginx.UpstreamBackend{route.Service + ":" + port: backends}) + if err != nil { + return fmt.Errorf("render deployment route: %w", err) + } + previousContent, previousErr := p.manager.GetVirtualHost(route.ID) + if err := p.manager.WriteVirtualHost(route.ID, content); err != nil { + return fmt.Errorf("write deployment route: %w", err) + } + if err := p.manager.TestConfig(); err != nil { + if previousErr == nil { + _ = p.manager.WriteVirtualHost(route.ID, previousContent) + } + return fmt.Errorf("test Nginx configuration: %w", err) + } + if err := p.manager.Reload(); err != nil { + return fmt.Errorf("reload Nginx: %w", err) + } + p.mu.Lock() + p.routes[route.ID] = cloneRoute(route) + p.mu.Unlock() + return nil +} + +func cloneRoute(route Route) Route { + route.Backends = append([]Backend(nil), route.Backends...) + return route +} + +func (p *managedNginxProvider) Drain(ctx context.Context, routeID, backendID string) error { + route, ok := p.route(routeID) + if !ok { + return fmt.Errorf("Route %q is not managed", routeID) + } + found := false + for index := range route.Backends { + if route.Backends[index].ID == backendID { + route.Backends[index].Healthy = false + found = true + } + } + if !found { + return fmt.Errorf("Backend %q is not part of route %q", backendID, routeID) + } + return p.Reconcile(ctx, route) +} + +func (p *managedNginxProvider) Remove(ctx context.Context, routeID string) error { + if err := ctx.Err(); err != nil { + return err + } + deployment, err := p.deployments.GetDeployment(routeID) + if err != nil { + return fmt.Errorf("load deployment for route: %w", err) + } + content, err := p.manager.RenderVirtualHost(deployment) + if err != nil { + return fmt.Errorf("render deployment route: %w", err) + } + if err := p.manager.WriteVirtualHost(routeID, content); err != nil { + return fmt.Errorf("write deployment route: %w", err) + } + if err := p.manager.TestConfig(); err != nil { + return fmt.Errorf("test Nginx configuration: %w", err) + } + if err := p.manager.Reload(); err != nil { + return fmt.Errorf("reload Nginx: %w", err) + } + p.mu.Lock() + delete(p.routes, routeID) + p.mu.Unlock() + return nil +} + +func (p *managedNginxProvider) route(id string) (Route, bool) { + p.mu.RLock() + defer p.mu.RUnlock() + route, ok := p.routes[id] + return cloneRoute(route), ok +} diff --git a/internal/routing/managed_nginx_test.go b/internal/routing/managed_nginx_test.go new file mode 100644 index 0000000..0bd4124 --- /dev/null +++ b/internal/routing/managed_nginx_test.go @@ -0,0 +1,109 @@ +package routing + +import ( + "context" + "strings" + "testing" + + "github.com/flatrun/agent/internal/nginx" + "github.com/flatrun/agent/pkg/models" +) + +type managedNginxRecorder struct { + backends map[string][]nginx.UpstreamBackend + content string + reloads int +} + +func (m *managedNginxRecorder) RenderVirtualHost(_ *models.Deployment) (string, error) { + return "compose deployment config", nil +} + +func (m *managedNginxRecorder) RenderVirtualHostWithBackends(_ *models.Deployment, backends map[string][]nginx.UpstreamBackend) (string, error) { + m.backends = backends + return "preserved deployment config", nil +} +func (m *managedNginxRecorder) GetVirtualHost(string) (string, error) { return "previous config", nil } +func (m *managedNginxRecorder) WriteVirtualHost(_, content string) error { + m.content = content + return nil +} +func (m *managedNginxRecorder) TestConfig() error { return nil } +func (m *managedNginxRecorder) Reload() error { m.reloads++; return nil } + +type deploymentSourceStub struct{ deployment *models.Deployment } + +func (s deploymentSourceStub) GetDeployment(string) (*models.Deployment, error) { + return s.deployment, nil +} + +func TestManagedNginxProviderReconcilesAndDrainsDeploymentBackends(t *testing.T) { + manager := &managedNginxRecorder{} + provider := NewManagedNginxProvider(manager, deploymentSourceStub{deployment: &models.Deployment{Name: "shop"}}) + route := Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http", Backends: []Backend{ + {ID: "one", Address: "10.42.0.8:8080", Healthy: true, Weight: 1}, + {ID: "two", Address: "10.42.1.9:8080", Healthy: true, Weight: 1}, + }} + if err := provider.Reconcile(context.Background(), route); err != nil { + t.Fatal(err) + } + if manager.content != "preserved deployment config" || manager.reloads != 1 || len(manager.backends["web:8080"]) != 2 { + t.Fatalf("manager = %#v", manager) + } + if err := provider.Drain(context.Background(), "shop", "two"); err != nil { + t.Fatal(err) + } + if manager.backends["web:8080"][1].Healthy || manager.reloads != 2 { + t.Fatalf("drained backends = %#v", manager.backends["web:8080"]) + } +} + +func TestManagedNginxProviderRequiresDeploymentService(t *testing.T) { + provider := NewManagedNginxProvider(&managedNginxRecorder{}, deploymentSourceStub{deployment: &models.Deployment{Name: "shop"}}) + err := provider.Validate(context.Background(), Route{ID: "shop", Domain: "shop.example.com", Protocol: "http", Backends: []Backend{{ID: "one", Address: "10.0.0.1:80", Healthy: true}}}) + if err == nil || !strings.Contains(err.Error(), "service") { + t.Fatalf("error = %v", err) + } +} + +func TestManagedNginxProviderRestoresComposeRouteOnRemove(t *testing.T) { + manager := &managedNginxRecorder{} + provider := NewManagedNginxProvider(manager, deploymentSourceStub{deployment: &models.Deployment{Name: "shop"}}) + if err := provider.Remove(context.Background(), "shop"); err != nil { + t.Fatal(err) + } + if manager.content != "compose deployment config" || manager.reloads != 1 { + t.Fatalf("manager = %#v", manager) + } +} + +func TestManagedNginxProviderRestoresRouteWithoutReloading(t *testing.T) { + manager := &managedNginxRecorder{} + route := Route{ID: "shop", Service: "web", Domain: "shop.example.com", Protocol: "http", Backends: []Backend{{ID: "one", Address: "10.42.0.8:8080", Healthy: true}}} + provider := NewManagedNginxProvider(manager, deploymentSourceStub{deployment: &models.Deployment{Name: "shop"}}, route) + + if manager.reloads != 0 { + t.Fatalf("reloads = %d", manager.reloads) + } + if err := provider.Drain(context.Background(), "shop", "one"); err != nil { + t.Fatal(err) + } + if manager.reloads != 1 || manager.backends["web:8080"][0].Healthy { + t.Fatalf("manager = %#v", manager) + } +} + +func TestManagedNginxProviderOwnsReturnedBackendSlice(t *testing.T) { + route := Route{ID: "shop", Backends: []Backend{{ID: "one", Healthy: true}}} + provider := NewManagedNginxProvider(&managedNginxRecorder{}, deploymentSourceStub{}, route).(*managedNginxProvider) + + loaded, ok := provider.route("shop") + if !ok { + t.Fatal("route was not restored") + } + loaded.Backends[0].Healthy = false + stored, _ := provider.route("shop") + if !stored.Backends[0].Healthy { + t.Fatal("caller mutation changed the stored route") + } +} diff --git a/internal/routing/nginx_writer.go b/internal/routing/nginx_writer.go new file mode 100644 index 0000000..6e1881d --- /dev/null +++ b/internal/routing/nginx_writer.go @@ -0,0 +1,62 @@ +package routing + +import ( + "context" + "fmt" +) + +type NginxManager interface { + WriteVirtualHost(string, string) error + DeleteVirtualHost(string) error + TestConfig() error + Reload() error +} + +type NginxWriter struct { + manager NginxManager +} + +func NewNginxWriter(manager NginxManager) *NginxWriter { + return &NginxWriter{manager: manager} +} + +func (w *NginxWriter) Apply(ctx context.Context, id, provider string, content []byte) error { + if err := ctx.Err(); err != nil { + return err + } + if provider != string(ProviderNginx) { + return fmt.Errorf("Nginx writer cannot apply %q configuration", provider) + } + if !safeRouteID.MatchString(id) { + return fmt.Errorf("Route ID is invalid") + } + if err := w.manager.WriteVirtualHost(id, string(content)); err != nil { + return fmt.Errorf("write Nginx route: %w", err) + } + if err := w.manager.TestConfig(); err != nil { + return fmt.Errorf("test Nginx configuration: %w", err) + } + if err := w.manager.Reload(); err != nil { + return fmt.Errorf("reload Nginx: %w", err) + } + return nil +} + +func (w *NginxWriter) Remove(ctx context.Context, id string) error { + if err := ctx.Err(); err != nil { + return err + } + if !safeRouteID.MatchString(id) { + return fmt.Errorf("Route ID is invalid") + } + if err := w.manager.DeleteVirtualHost(id); err != nil { + return fmt.Errorf("delete Nginx route: %w", err) + } + if err := w.manager.TestConfig(); err != nil { + return fmt.Errorf("test Nginx configuration: %w", err) + } + if err := w.manager.Reload(); err != nil { + return fmt.Errorf("reload Nginx: %w", err) + } + return nil +} diff --git a/internal/routing/nginx_writer_test.go b/internal/routing/nginx_writer_test.go new file mode 100644 index 0000000..d55da75 --- /dev/null +++ b/internal/routing/nginx_writer_test.go @@ -0,0 +1,55 @@ +package routing + +import ( + "context" + "testing" +) + +type recordingNginx struct { + written string + deleted string + tested int + reloaded int +} + +func (m *recordingNginx) WriteVirtualHost(id, content string) error { + m.written = id + ":" + content + return nil +} + +func (m *recordingNginx) DeleteVirtualHost(id string) error { + m.deleted = id + return nil +} + +func (m *recordingNginx) TestConfig() error { + m.tested++ + return nil +} + +func (m *recordingNginx) Reload() error { + m.reloaded++ + return nil +} + +func TestNginxWriterAppliesAndReloadsRoute(t *testing.T) { + manager := &recordingNginx{} + writer := NewNginxWriter(manager) + if err := writer.Apply(context.Background(), "shop", "nginx", []byte("upstream shop {}")); err != nil { + t.Fatal(err) + } + if manager.written != "shop:upstream shop {}" || manager.tested != 1 || manager.reloaded != 1 { + t.Fatalf("unexpected manager calls: %+v", manager) + } +} + +func TestNginxWriterRejectsUnsafeRouteID(t *testing.T) { + manager := &recordingNginx{} + writer := NewNginxWriter(manager) + if err := writer.Apply(context.Background(), "../shop", "nginx", []byte("route")); err == nil { + t.Fatal("expected invalid route error") + } + if manager.written != "" { + t.Fatal("unsafe route was written") + } +} diff --git a/internal/routing/provider.go b/internal/routing/provider.go new file mode 100644 index 0000000..55a83dc --- /dev/null +++ b/internal/routing/provider.go @@ -0,0 +1,34 @@ +package routing + +import "context" + +type ProviderID string + +const ( + ProviderNginx ProviderID = "nginx" + ProviderTraefik ProviderID = "traefik" +) + +type Backend struct { + ID string `json:"id"` + Address string `json:"address"` + Healthy bool `json:"healthy"` + Weight int `json:"weight,omitempty"` +} + +type Route struct { + ID string `json:"id"` + Service string `json:"service,omitempty"` + Domain string `json:"domain"` + Path string `json:"path,omitempty"` + Protocol string `json:"protocol"` + Backends []Backend `json:"backends"` +} + +type Provider interface { + ID() ProviderID + Validate(context.Context, Route) error + Reconcile(context.Context, Route) error + Drain(context.Context, string, string) error + Remove(context.Context, string) error +} diff --git a/internal/system/network.go b/internal/system/network.go index 29f819a..7af65c1 100644 --- a/internal/system/network.go +++ b/internal/system/network.go @@ -38,6 +38,7 @@ type NetworkInterface struct { type ServerInfo struct { Hostname string `json:"hostname"` + AgentURL string `json:"agent_url"` PublicIPv4 string `json:"public_ipv4"` PublicIPv6 string `json:"public_ipv6"` Interfaces []NetworkInterface `json:"interfaces"` diff --git a/pkg/config/capacity_test.go b/pkg/config/capacity_test.go new file mode 100644 index 0000000..5e4cf9f --- /dev/null +++ b/pkg/config/capacity_test.go @@ -0,0 +1,28 @@ +package config + +import "testing" + +func TestCapacityDefaultsAllowVerticalAndHorizontalScaling(t *testing.T) { + cfg := &Config{} + setDefaults(cfg) + + if cfg.Capacity.AllocationThresholdPercent != 90 || cfg.Capacity.HostThresholdPercent != 85 { + t.Fatalf("thresholds = %.0f, %.0f", cfg.Capacity.AllocationThresholdPercent, cfg.Capacity.HostThresholdPercent) + } + if cfg.Capacity.AllowVertical == nil || !*cfg.Capacity.AllowVertical { + t.Fatal("vertical scaling should default to enabled") + } + if cfg.Capacity.AllowHorizontal == nil || !*cfg.Capacity.AllowHorizontal { + t.Fatal("horizontal scaling should default to enabled") + } +} + +func TestCapacityDefaultsPreserveExplicitDisabledScaling(t *testing.T) { + disabled := false + cfg := &Config{Capacity: CapacityConfig{AllowVertical: &disabled, AllowHorizontal: &disabled}} + setDefaults(cfg) + + if *cfg.Capacity.AllowVertical || *cfg.Capacity.AllowHorizontal { + t.Fatal("explicit scaling choices were overwritten") + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index b7578ec..d2a36f2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -14,11 +14,33 @@ import ( ) type ClusterConfig struct { - Enabled bool `yaml:"enabled"` - ServerName string `yaml:"server_name"` - AdvertiseURL string `yaml:"advertise_url"` - HealthInterval string `yaml:"health_interval"` - RequestTimeout string `yaml:"request_timeout"` + Enabled bool `yaml:"enabled"` + ServerName string `yaml:"server_name"` + AdvertiseURL string `yaml:"advertise_url"` + HealthInterval string `yaml:"health_interval"` + RequestTimeout string `yaml:"request_timeout"` + Orchestrator string `yaml:"orchestrator" json:"orchestrator"` + Routing string `yaml:"routing" json:"routing"` + K3s K3sConfig `yaml:"k3s" json:"k3s"` +} + +type K3sConfig struct { + Kubeconfig string `yaml:"kubeconfig" json:"kubeconfig"` + Namespace string `yaml:"namespace" json:"namespace"` +} + +type CapacityConfig struct { + AllocationThresholdPercent float64 `yaml:"allocation_threshold_percent" json:"allocation_threshold_percent"` + HostThresholdPercent float64 `yaml:"host_threshold_percent" json:"host_threshold_percent"` + HostMemoryReserve uint64 `yaml:"host_memory_reserve" json:"host_memory_reserve"` + HostCPUReserve float64 `yaml:"host_cpu_reserve" json:"host_cpu_reserve"` + MemoryStepPercent float64 `yaml:"memory_step_percent" json:"memory_step_percent"` + CPUStepPercent float64 `yaml:"cpu_step_percent" json:"cpu_step_percent"` + MaxMemory uint64 `yaml:"max_memory" json:"max_memory"` + MaxCPU float64 `yaml:"max_cpu" json:"max_cpu"` + AllowVertical *bool `yaml:"allow_vertical" json:"allow_vertical"` + AllowHorizontal *bool `yaml:"allow_horizontal" json:"allow_horizontal"` + OfferToFleet bool `yaml:"offer_to_fleet" json:"offer_to_fleet"` } type Config struct { @@ -37,6 +59,7 @@ type Config struct { Security SecurityConfig `yaml:"security"` Audit AuditConfig `yaml:"audit"` Cluster ClusterConfig `yaml:"cluster"` + Capacity CapacityConfig `yaml:"capacity"` SystemTerminal SystemTerminalConfig `yaml:"system_terminal"` Cleanup CleanupConfig `yaml:"cleanup"` Plans PlansConfig `yaml:"plans"` @@ -386,6 +409,38 @@ func setDefaults(cfg *Config) { if cfg.DefaultTimeout == 0 { cfg.DefaultTimeout = 2 * time.Minute } + if cfg.Cluster.Orchestrator == "" { + cfg.Cluster.Orchestrator = "standalone" + } + if cfg.Cluster.Routing == "" { + cfg.Cluster.Routing = "nginx" + } + if cfg.Capacity.AllocationThresholdPercent == 0 { + cfg.Capacity.AllocationThresholdPercent = 90 + } + if cfg.Capacity.HostThresholdPercent == 0 { + cfg.Capacity.HostThresholdPercent = 85 + } + if cfg.Capacity.HostMemoryReserve == 0 { + cfg.Capacity.HostMemoryReserve = 512 * 1024 * 1024 + } + if cfg.Capacity.HostCPUReserve == 0 { + cfg.Capacity.HostCPUReserve = 0.25 + } + if cfg.Capacity.MemoryStepPercent == 0 { + cfg.Capacity.MemoryStepPercent = 50 + } + if cfg.Capacity.CPUStepPercent == 0 { + cfg.Capacity.CPUStepPercent = 50 + } + if cfg.Capacity.AllowVertical == nil { + enabled := true + cfg.Capacity.AllowVertical = &enabled + } + if cfg.Capacity.AllowHorizontal == nil { + enabled := true + cfg.Capacity.AllowHorizontal = &enabled + } if cfg.Cleanup.Timeout == 0 { cfg.Cleanup.Timeout = cfg.DefaultTimeout } diff --git a/pkg/models/deployment.go b/pkg/models/deployment.go index deacc16..99b80b0 100644 --- a/pkg/models/deployment.go +++ b/pkg/models/deployment.go @@ -39,6 +39,7 @@ type ServiceMetadata struct { QuickActions []QuickAction `yaml:"quick_actions,omitempty" json:"quick_actions,omitempty"` Security *DeploymentSecurityConfig `yaml:"security,omitempty" json:"security,omitempty"` Backup *BackupSpec `yaml:"backup,omitempty" json:"backup,omitempty"` + Scaling *ScalingConfig `yaml:"scaling,omitempty" json:"scaling,omitempty"` ProtectedMode *ProtectedModeConfig `yaml:"protected_mode,omitempty" json:"protected_mode,omitempty"` RequirePlan bool `yaml:"require_plan,omitempty" json:"require_plan,omitempty"` CredentialID string `yaml:"credential_id,omitempty" json:"credential_id,omitempty"` @@ -47,6 +48,17 @@ type ServiceMetadata struct { Databases []DatabaseConfig `yaml:"databases,omitempty" json:"databases,omitempty"` } +type ScalingConfig struct { + Service string `yaml:"service" json:"service"` + Stateless bool `yaml:"stateless" json:"stateless"` + Storage ScalingStorageConfig `yaml:"storage,omitempty" json:"storage,omitempty"` +} + +type ScalingStorageConfig struct { + Mode string `yaml:"mode,omitempty" json:"mode,omitempty"` + Class string `yaml:"class,omitempty" json:"class,omitempty"` +} + type DomainConfig struct { ID string `yaml:"id" json:"id"` Service string `yaml:"service" json:"service"`