From 189045c5343d232673554effc4e406d98d51532c Mon Sep 17 00:00:00 2001 From: Sudeep Reddy <148759887+sudeepr34@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:09:55 +0530 Subject: [PATCH] feat: add node maintenance evict commands to longhornctl Add `longhornctl maintenance evict-node` and `evict-disk` commands so node maintenance can be driven from the CLI without the web UI. evict-node cordons the Kubernetes node, disables Longhorn replica scheduling on it and requests eviction of all its replicas. evict-disk does the same for a single disk identified by UUID. Both accept --wait to block until no replicas remain, and evict-node prints the kubectl drain command that finishes the maintenance workflow. Related: longhorn/longhorn#8799 Signed-off-by: Sudeep Reddy <148759887+sudeepr34@users.noreply.github.com> --- cmd/remote/longhornctl.go | 1 + cmd/remote/subcmd/maintenance.go | 133 ++++++++++++++ pkg/consts/cmd.go | 21 ++- pkg/remote/maintenance/evictor.go | 242 +++++++++++++++++++++++++ pkg/remote/maintenance/evictor_test.go | 96 ++++++++++ pkg/utils/longhorn/client.go | 8 + 6 files changed, 494 insertions(+), 7 deletions(-) create mode 100644 cmd/remote/subcmd/maintenance.go create mode 100644 pkg/remote/maintenance/evictor.go create mode 100644 pkg/remote/maintenance/evictor_test.go diff --git a/cmd/remote/longhornctl.go b/cmd/remote/longhornctl.go index 3c4750eb..fa311a61 100644 --- a/cmd/remote/longhornctl.go +++ b/cmd/remote/longhornctl.go @@ -57,6 +57,7 @@ func newCmdLonghornctl() *cobra.Command { { Message: "Operation Commands:", Commands: []*cobra.Command{ + subcmd.NewCmdMaintenance(globalOpts), subcmd.NewCmdTrim(globalOpts), subcmd.NewCmdExport(globalOpts), subcmd.NewCmdChecksum(globalOpts), diff --git a/cmd/remote/subcmd/maintenance.go b/cmd/remote/subcmd/maintenance.go new file mode 100644 index 00000000..84cf4f22 --- /dev/null +++ b/cmd/remote/subcmd/maintenance.go @@ -0,0 +1,133 @@ +package subcmd + +import ( + "context" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/remote/maintenance" + "github.com/longhorn/cli/pkg/types" + "github.com/longhorn/cli/pkg/utils" +) + +func NewCmdMaintenance(globalOpts *types.GlobalCmdOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: consts.SubCmdMaintenance, + Short: "Longhorn node maintenance operations", + } + + utils.SetGlobalOptionsRemote(cmd, globalOpts) + + cmd.AddCommand(newCmdEvictNode(globalOpts)) + cmd.AddCommand(newCmdEvictDisk(globalOpts)) + + return cmd +} + +func newCmdEvictNode(globalOpts *types.GlobalCmdOptions) *cobra.Command { + var nodeEvictor = maintenance.Evictor{} + + cmd := &cobra.Command{ + Use: consts.SubCmdEvictNode, + Short: "Evict all replicas from a node", + Long: `Prepare a Longhorn node for maintenance by cordoning the Kubernetes node, disabling replica scheduling on it and requesting eviction of all its replicas. + +Specify the following option: +- --node-id: The name of the node to evict. + +With --wait, the command blocks until no replicas remain scheduled on the node. +Once eviction is done, drain the node with the printed kubectl command before stopping or restarting it.`, + Example: `$ longhornctl maintenance evict-node --node-id="worker-1" --wait`, + + PreRun: func(cmd *cobra.Command, args []string) { + nodeEvictor.Image = globalOpts.Image + nodeEvictor.ImageRegistry = globalOpts.ImageRegistry + nodeEvictor.ImagePullSecret = globalOpts.ImagePullSecret + nodeEvictor.KubeConfigPath = globalOpts.KubeConfigPath + nodeEvictor.NodeSelector = globalOpts.NodeSelector + nodeEvictor.Tolerations = globalOpts.Tolerations + nodeEvictor.Namespace = globalOpts.Namespace + + utils.CheckErr(nodeEvictor.Validate()) + + logrus.Info("Initializing node evictor") + if err := nodeEvictor.Init(); err != nil { + utils.CheckErr(errors.Wrapf(err, "Failed to initialize node evictor for node %s", nodeEvictor.NodeID)) + } + }, + + Run: func(cmd *cobra.Command, args []string) { + log := logrus.WithField("node", nodeEvictor.NodeID) + + log.Info("Running node evictor") + if err := nodeEvictor.EvictNode(context.Background()); err != nil { + utils.CheckErr(errors.Wrapf(err, "Failed to evict node %s", nodeEvictor.NodeID)) + } + + log.Info("Completed node evictor") + }, + } + + utils.SetGlobalOptionsRemote(cmd, globalOpts) + + cmd.Flags().StringVar(&nodeEvictor.NodeID, consts.CmdOptNodeId, "", "Name of the node to evict.") + cmd.Flags().BoolVar(&nodeEvictor.Wait, consts.CmdOptWait, false, "Wait until all replicas have left the node.") + + return cmd +} + +func newCmdEvictDisk(globalOpts *types.GlobalCmdOptions) *cobra.Command { + var diskEvictor = maintenance.Evictor{} + + cmd := &cobra.Command{ + Use: consts.SubCmdEvictDisk, + Short: "Evict all replicas from a disk", + Long: `Prepare a single Longhorn disk for maintenance by disabling replica scheduling on it and requesting eviction of all its replicas. + +Specify the following options: +- --node-id: The name of the node the disk belongs to. +- --disk-uuid: The UUID of the disk to evict. + +With --wait, the command blocks until no replicas remain scheduled on the disk.`, + Example: `$ longhornctl maintenance evict-disk --node-id="worker-1" --disk-uuid="a1b2c3d4-e5f6-7890-abcd-ef1234567890" --wait`, + + PreRun: func(cmd *cobra.Command, args []string) { + diskEvictor.Image = globalOpts.Image + diskEvictor.ImageRegistry = globalOpts.ImageRegistry + diskEvictor.ImagePullSecret = globalOpts.ImagePullSecret + diskEvictor.KubeConfigPath = globalOpts.KubeConfigPath + diskEvictor.NodeSelector = globalOpts.NodeSelector + diskEvictor.Tolerations = globalOpts.Tolerations + diskEvictor.Namespace = globalOpts.Namespace + + utils.CheckErr(diskEvictor.ValidateDiskEviction()) + + logrus.Info("Initializing disk evictor") + if err := diskEvictor.Init(); err != nil { + utils.CheckErr(errors.Wrapf(err, "Failed to initialize disk evictor for disk %s", diskEvictor.DiskUUID)) + } + }, + + Run: func(cmd *cobra.Command, args []string) { + log := logrus.WithFields(logrus.Fields{"node": diskEvictor.NodeID, "disk": diskEvictor.DiskUUID}) + + log.Info("Running disk evictor") + if err := diskEvictor.EvictDisk(context.Background()); err != nil { + utils.CheckErr(errors.Wrapf(err, "Failed to evict disk %s", diskEvictor.DiskUUID)) + } + + log.Info("Completed disk evictor") + }, + } + + utils.SetGlobalOptionsRemote(cmd, globalOpts) + + cmd.Flags().StringVar(&diskEvictor.NodeID, consts.CmdOptNodeId, "", "Name of the node the disk belongs to.") + cmd.Flags().StringVar(&diskEvictor.DiskUUID, consts.CmdOptDiskUUID, "", "UUID of the disk to evict.") + cmd.Flags().BoolVar(&diskEvictor.Wait, consts.CmdOptWait, false, "Wait until all replicas have left the disk.") + + return cmd +} diff --git a/pkg/consts/cmd.go b/pkg/consts/cmd.go index 94b3175a..392c31bd 100644 --- a/pkg/consts/cmd.go +++ b/pkg/consts/cmd.go @@ -8,12 +8,13 @@ const ( const ( // The first layer of subcommands (verb) - SubCmdCheck = "check" - SubCmdDiagnose = "diagnose" - SubCmdExport = "export" - SubCmdGet = "get" - SubCmdInstall = "install" - SubCmdTrim = "trim" + SubCmdCheck = "check" + SubCmdDiagnose = "diagnose" + SubCmdExport = "export" + SubCmdGet = "get" + SubCmdInstall = "install" + SubCmdMaintenance = "maintenance" + SubCmdTrim = "trim" // The second layer of subcommands (noun) SubCmdPreflight = "preflight" @@ -21,7 +22,9 @@ const ( SubCmdVolume = "volume" // The third layer of subcommands (action to the previous layers) - SubCmdStop = "stop" + SubCmdStop = "stop" + SubCmdEvictNode = "evict-node" + SubCmdEvictDisk = "evict-disk" // Other subcommands SubCmdVersion = "version" @@ -67,6 +70,10 @@ const ( CmdOptReplicaDryRun = "dry-run" EnvReplicaDirectory = "REPLICA_DIR" SubCmdRecover = "recover" + + // Maintenance options + CmdOptDiskUUID = "disk-uuid" + CmdOptWait = "wait" ) const ( diff --git a/pkg/remote/maintenance/evictor.go b/pkg/remote/maintenance/evictor.go new file mode 100644 index 00000000..7456088d --- /dev/null +++ b/pkg/remote/maintenance/evictor.go @@ -0,0 +1,242 @@ +package maintenance + +import ( + "context" + "fmt" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + kubeclient "k8s.io/client-go/kubernetes" + + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" + + "github.com/longhorn/cli/pkg/types" + + kubeutils "github.com/longhorn/cli/pkg/utils/kubernetes" + utilslonghorn "github.com/longhorn/cli/pkg/utils/longhorn" +) + +const ( + // evictionPollInterval is how often the evictor re-checks replica counts while waiting. + evictionPollInterval = 10 * time.Second + // evictionWaitTimeout bounds how long --wait blocks before giving up. + evictionWaitTimeout = 30 * time.Minute +) + +// EvictorCmdOptions holds the options for the maintenance evict commands. +type EvictorCmdOptions struct { + types.GlobalCmdOptions + + NodeID string + DiskUUID string + Wait bool +} + +// Evictor provides functions for Longhorn node maintenance eviction. +type Evictor struct { + EvictorCmdOptions + + kubeClient *kubeclient.Clientset + longhornClient *utilslonghorn.LonghornClient +} + +// Init initializes the Evictor. +func (remote *Evictor) Init() error { + kubeClient, err := kubeutils.NewKubeClient("", remote.KubeConfigPath) + if err != nil { + return err + } + remote.kubeClient = kubeClient + + longhornClient, err := utilslonghorn.NewLonghornClient(remote.KubeConfigPath, remote.Namespace) + if err != nil { + return errors.Wrap(err, "failed to initialize Longhorn client") + } + remote.longhornClient = longhornClient + + return nil +} + +// Validate checks that the required options are set. +func (remote *Evictor) Validate() error { + if remote.NodeID == "" { + return errors.New("node ID must be specified") + } + return nil +} + +// ValidateDiskEviction checks the options required for disk eviction. +func (remote *Evictor) ValidateDiskEviction() error { + if err := remote.Validate(); err != nil { + return err + } + if remote.DiskUUID == "" { + return errors.New("disk UUID must be specified") + } + return nil +} + +// EvictNode prepares a node for maintenance: it cordons the Kubernetes node, +// disables Longhorn replica scheduling on it and requests eviction of all its +// replicas. With Wait it blocks until the node holds no replicas anymore. +func (remote *Evictor) EvictNode(ctx context.Context) error { + log := logrus.WithField("node", remote.NodeID) + + if err := remote.cordonNode(ctx); err != nil { + return err + } + + node, err := remote.longhornClient.GetNode(ctx, remote.NodeID) + if err != nil { + return errors.Wrapf(err, "failed to get Longhorn node %s", remote.NodeID) + } + + if !node.Spec.AllowScheduling || node.Spec.EvictionRequested { + log.Info("Node eviction was already requested") + } else { + node.Spec.AllowScheduling = false + node.Spec.EvictionRequested = true + if _, err := remote.longhornClient.UpdateNode(ctx, node); err != nil { + return errors.Wrapf(err, "failed to request eviction for node %s", remote.NodeID) + } + log.Info("Requested eviction for node") + } + + if remote.Wait { + log.Info("Waiting for replicas to leave the node") + if err := remote.waitForNodeEviction(ctx); err != nil { + return err + } + log.Info("All replicas evicted from node") + } + + log.Infof("To finish maintenance, drain the node:\n %s", drainCommand(remote.NodeID)) + return nil +} + +// EvictDisk requests eviction of all replicas from a single disk of a node. +// With Wait it blocks until the disk holds no replicas anymore. +func (remote *Evictor) EvictDisk(ctx context.Context) error { + log := logrus.WithFields(logrus.Fields{"node": remote.NodeID, "disk": remote.DiskUUID}) + + node, err := remote.longhornClient.GetNode(ctx, remote.NodeID) + if err != nil { + return errors.Wrapf(err, "failed to get Longhorn node %s", remote.NodeID) + } + + diskName, err := findDiskNameByUUID(node, remote.DiskUUID) + if err != nil { + return err + } + + diskSpec, ok := node.Spec.Disks[diskName] + if !ok { + return errors.Errorf("disk %q (uuid %s) not found in node %s spec", diskName, remote.DiskUUID, remote.NodeID) + } + + if diskSpec.EvictionRequested { + log.Info("Disk eviction was already requested") + } else { + diskSpec.AllowScheduling = false + diskSpec.EvictionRequested = true + node.Spec.Disks[diskName] = diskSpec + if _, err := remote.longhornClient.UpdateNode(ctx, node); err != nil { + return errors.Wrapf(err, "failed to request eviction for disk %s on node %s", remote.DiskUUID, remote.NodeID) + } + log.Info("Requested eviction for disk") + } + + if remote.Wait { + log.Info("Waiting for replicas to leave the disk") + if err := remote.waitForDiskEviction(ctx, remote.NodeID, diskName); err != nil { + return err + } + log.Info("All replicas evicted from disk") + } + + return nil +} + +// cordonNode marks the Kubernetes node unschedulable so no new pods land on it +// while it is being maintained. +func (remote *Evictor) cordonNode(ctx context.Context) error { + node, err := remote.kubeClient.CoreV1().Nodes().Get(ctx, remote.NodeID, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "failed to get Kubernetes node %s", remote.NodeID) + } + + if node.Spec.Unschedulable { + logrus.WithField("node", remote.NodeID).Info("Node is already cordoned") + return nil + } + + node.Spec.Unschedulable = true + if _, err := remote.kubeClient.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}); err != nil { + return errors.Wrapf(err, "failed to cordon Kubernetes node %s", remote.NodeID) + } + logrus.WithField("node", remote.NodeID).Info("Cordoned node") + + return nil +} + +// waitForNodeEviction polls until no replicas remain scheduled on the node. +func (remote *Evictor) waitForNodeEviction(ctx context.Context) error { + return wait.PollUntilContextTimeout(ctx, evictionPollInterval, evictionWaitTimeout, true, + func(ctx context.Context) (bool, error) { + node, err := remote.longhornClient.GetNode(ctx, remote.NodeID) + if err != nil { + return false, errors.Wrapf(err, "failed to get Longhorn node %s", remote.NodeID) + } + return countScheduledReplicas(node, "") == 0, nil + }) +} + +// waitForDiskEviction polls until no replicas remain scheduled on the disk. +func (remote *Evictor) waitForDiskEviction(ctx context.Context, nodeID, diskName string) error { + return wait.PollUntilContextTimeout(ctx, evictionPollInterval, evictionWaitTimeout, true, + func(ctx context.Context) (bool, error) { + node, err := remote.longhornClient.GetNode(ctx, nodeID) + if err != nil { + return false, errors.Wrapf(err, "failed to get Longhorn node %s", nodeID) + } + return countScheduledReplicas(node, diskName) == 0, nil + }) +} + +// findDiskNameByUUID resolves a disk UUID to its disk name in the node spec. +// The spec is keyed by disk name while users identify disks by UUID, so the +// lookup goes through the node status. +func findDiskNameByUUID(node *longhorn.Node, diskUUID string) (string, error) { + for diskName, diskStatus := range node.Status.DiskStatus { + if diskStatus != nil && diskStatus.DiskUUID == diskUUID { + return diskName, nil + } + } + return "", errors.Errorf("disk with UUID %s not found on node %s", diskUUID, node.Name) +} + +// countScheduledReplicas returns the number of replicas scheduled on the node, +// or on a single disk when diskName is not empty. +func countScheduledReplicas(node *longhorn.Node, diskName string) int { + count := 0 + for name, diskStatus := range node.Status.DiskStatus { + if diskName != "" && name != diskName { + continue + } + if diskStatus != nil { + count += len(diskStatus.ScheduledReplica) + } + } + return count +} + +// drainCommand returns the kubectl command that finishes node maintenance once +// eviction is done. Draining itself stays a kubectl operation because the drain +// helper is not vendored by the CLI. +func drainCommand(nodeID string) string { + return fmt.Sprintf("kubectl drain %s --ignore-daemonsets", nodeID) +} diff --git a/pkg/remote/maintenance/evictor_test.go b/pkg/remote/maintenance/evictor_test.go new file mode 100644 index 00000000..94f4d529 --- /dev/null +++ b/pkg/remote/maintenance/evictor_test.go @@ -0,0 +1,96 @@ +package maintenance + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" +) + +func newTestNode() *longhorn.Node { + return &longhorn.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "worker-1"}, + Spec: longhorn.NodeSpec{ + Name: "worker-1", + AllowScheduling: true, + Disks: map[string]longhorn.DiskSpec{ + "default-disk-abc": { + Path: "/var/lib/longhorn", + AllowScheduling: true, + }, + }, + }, + Status: longhorn.NodeStatus{ + DiskStatus: map[string]*longhorn.DiskStatus{ + "default-disk-abc": { + DiskUUID: "11111111-2222-3333-4444-555555555555", + ScheduledReplica: map[string]int64{ + "replica-a": 1073741824, + "replica-b": 2147483648, + }, + }, + "default-disk-def": { + DiskUUID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + ScheduledReplica: map[string]int64{}, + }, + }, + }, + } +} + +func TestFindDiskNameByUUID(t *testing.T) { + node := newTestNode() + + diskName, err := findDiskNameByUUID(node, "11111111-2222-3333-4444-555555555555") + assert.NoError(t, err) + assert.Equal(t, "default-disk-abc", diskName) + + diskName, err = findDiskNameByUUID(node, "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee") + assert.NoError(t, err) + assert.Equal(t, "default-disk-def", diskName) + + _, err = findDiskNameByUUID(node, "00000000-0000-0000-0000-000000000000") + assert.Error(t, err) + assert.Contains(t, err.Error(), "not found") + + _, err = findDiskNameByUUID(&longhorn.Node{}, "11111111-2222-3333-4444-555555555555") + assert.Error(t, err) +} + +func TestCountScheduledReplicas(t *testing.T) { + node := newTestNode() + + assert.Equal(t, 2, countScheduledReplicas(node, "")) + assert.Equal(t, 2, countScheduledReplicas(node, "default-disk-abc")) + assert.Equal(t, 0, countScheduledReplicas(node, "default-disk-def")) + assert.Equal(t, 0, countScheduledReplicas(node, "nonexistent-disk")) + assert.Equal(t, 0, countScheduledReplicas(&longhorn.Node{}, "")) +} + +func TestDrainCommand(t *testing.T) { + assert.Equal(t, + "kubectl drain worker-1 --ignore-daemonsets", + drainCommand("worker-1")) +} + +func TestValidate(t *testing.T) { + evictor := &Evictor{} + assert.Error(t, evictor.Validate()) + + evictor.NodeID = "worker-1" + assert.NoError(t, evictor.Validate()) +} + +func TestValidateDiskEviction(t *testing.T) { + evictor := &Evictor{} + assert.Error(t, evictor.ValidateDiskEviction()) + + evictor.NodeID = "worker-1" + assert.Error(t, evictor.ValidateDiskEviction()) + + evictor.DiskUUID = "11111111-2222-3333-4444-555555555555" + assert.NoError(t, evictor.ValidateDiskEviction()) +} diff --git a/pkg/utils/longhorn/client.go b/pkg/utils/longhorn/client.go index 5dca222f..80d6e936 100644 --- a/pkg/utils/longhorn/client.go +++ b/pkg/utils/longhorn/client.go @@ -72,6 +72,14 @@ func (s *LonghornClient) ListNodes(ctx context.Context) (*longhorn.NodeList, err return s.clientset.LonghornV1beta2().Nodes(s.namespace).List(ctx, metav1.ListOptions{}) } +func (s *LonghornClient) GetNode(ctx context.Context, name string) (*longhorn.Node, error) { + return s.clientset.LonghornV1beta2().Nodes(s.namespace).Get(ctx, name, metav1.GetOptions{}) +} + +func (s *LonghornClient) UpdateNode(ctx context.Context, node *longhorn.Node) (*longhorn.Node, error) { + return s.clientset.LonghornV1beta2().Nodes(s.namespace).Update(ctx, node, metav1.UpdateOptions{}) +} + func (s *LonghornClient) ListEngineImages(ctx context.Context) (*longhorn.EngineImageList, error) { return s.clientset.LonghornV1beta2().EngineImages(s.namespace).List(ctx, metav1.ListOptions{}) }