Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/local/longhornctl-local.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ func newCmdLonghornctlLocal() *cobra.Command {
Message: "Troubleshoot Commands:",
Commands: []*cobra.Command{
localsubcmd.NewCmdCheck(globalOpts),
localsubcmd.NewCmdDiagnose(globalOpts),
localsubcmd.NewCmdGet(globalOpts),
},
},
Expand Down
89 changes: 89 additions & 0 deletions cmd/local/subcmd/diagnose.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package subcmd

import (
"os"

"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"

"github.com/longhorn/cli/pkg/consts"
"github.com/longhorn/cli/pkg/types"
"github.com/longhorn/cli/pkg/utils"

localspdk "github.com/longhorn/cli/pkg/local/spdk"
)

func NewCmdDiagnose(globalOpts *types.GlobalCmdOptions) *cobra.Command {
cmd := &cobra.Command{
Use: consts.SubCmdDiagnose,
Short: "Diagnose the health of the Longhorn system",
}

utils.SetGlobalOptionsLocal(cmd, globalOpts)

cmd.AddCommand(newCmdDiagnoseV2DataEngine(globalOpts))

return cmd
}

func newCmdDiagnoseV2DataEngine(globalOpts *types.GlobalCmdOptions) *cobra.Command {
var localDiagnoser = localspdk.Diagnoser{}

cmd := &cobra.Command{
Use: consts.SubCmdV2DataEngine,
Short: "Inspect V2 Data Engine SPDK target memory, iobuf pools, and blobstore channels",
Long: `This command performs a lockless, read-only inspection of running V2 Data Engine spdk_tgt processes on the node.

It combines SPDK Unix-socket JSON-RPC queries with ELF/DWARF-guided /proc/<pid>/mem inspection to report:
- Global iobuf small/large pool configuration, live DPDK ring free counts, and per-module retry statistics.
- Per-thread iobuf wait queues and lockless module caches (including nvmf_TCP large_pool availability).
- Blobstore and per-blob spdk_bs_channel request-set pools (freeReqs, needClusterAlloc queues, and active in-flight CoW / external snapshot reads).
- Bdev shared resources with queued nomem_io requests and active bdev_no_mem_poller counts.`,

PreRun: func(cmd *cobra.Command, args []string) {
localDiagnoser.LogLevel = globalOpts.LogLevel

if err := localDiagnoser.Init(); err != nil {
utils.CheckErr(errors.Wrap(err, "Failed to initialize V2 Data Engine diagnoser"))
}
},

Run: func(cmd *cobra.Command, args []string) {
if err := localDiagnoser.Run(); err != nil {
utils.CheckErr(errors.Wrap(err, "Failed to run V2 Data Engine diagnoser"))
}

logrus.Info("Successfully collected V2 Data Engine SPDK diagnostics")
},

PostRun: func(cmd *cobra.Command, args []string) {
if err := localDiagnoser.Output(); err != nil {
utils.CheckErr(errors.Wrap(err, "Failed to output V2 Data Engine SPDK diagnostics"))
}

logrus.Info("Successfully output V2 Data Engine SPDK diagnostics")
},
}

utils.SetGlobalOptionsLocal(cmd, globalOpts)

defaultPID := utils.ConvertStringToTypeOrDefault(os.Getenv(consts.EnvSpdkPid), 0)
defaultProcDir := os.Getenv(consts.EnvProcDir)
if defaultProcDir == "" {
defaultProcDir = consts.DefaultProcDirectory
}
defaultSock := os.Getenv(consts.EnvSpdkSocket)
if defaultSock == "" {
defaultSock = consts.DefaultSpdkSocketPath
}

cmd.Flags().StringVar(&localDiagnoser.CurrentNodeID, consts.CmdOptNodeId, os.Getenv(consts.EnvCurrentNodeID), "Current node ID.")
cmd.Flags().StringVarP(&localDiagnoser.OutputFilePath, consts.CmdOptOutputFile, "o", os.Getenv(consts.EnvOutputFilePath), "Output the result to a file, default to stdout.")
cmd.Flags().StringVar(&localDiagnoser.ProcDir, consts.CmdOptProcDir, defaultProcDir, "Path to the procfs directory used to discover and inspect spdk_tgt.")
cmd.Flags().StringVar(&localDiagnoser.SpdkSocketPath, consts.CmdOptSpdkSocket, defaultSock, "Path to the SPDK JSON-RPC Unix domain socket inside the target mount namespace.")
cmd.Flags().IntVar(&localDiagnoser.TargetPID, consts.CmdOptPid, defaultPID, "Inspect a specific spdk_tgt PID instead of scanning procfs.")
cmd.Flags().BoolVar(&localDiagnoser.IncludeAll, consts.CmdOptAll, false, "Include idle per-blob channels with 0 queued or active requests.")

return cmd
}
5 changes: 3 additions & 2 deletions cmd/local/subcmd/get.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@ import (
"github.com/spf13/cobra"

"github.com/longhorn/cli/pkg/consts"
local "github.com/longhorn/cli/pkg/local/replica"
"github.com/longhorn/cli/pkg/types"
"github.com/longhorn/cli/pkg/utils"

localreplica "github.com/longhorn/cli/pkg/local/replica"
)

func NewCmdGet(globalOpts *types.GlobalCmdOptions) *cobra.Command {
Expand All @@ -27,7 +28,7 @@ func NewCmdGet(globalOpts *types.GlobalCmdOptions) *cobra.Command {
}

func newCmdGetReplica(globalOpts *types.GlobalCmdOptions) *cobra.Command {
var localGetter = local.Getter{}
var localGetter = localreplica.Getter{}

cmd := &cobra.Command{
Use: consts.SubCmdReplica,
Expand Down
112 changes: 108 additions & 4 deletions cmd/remote/subcmd/diagnose.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,33 @@ import (

"github.com/longhorn/cli/pkg/consts"
"github.com/longhorn/cli/pkg/remote/diagnose"
"github.com/longhorn/cli/pkg/remote/spdk"
"github.com/longhorn/cli/pkg/types"
"github.com/longhorn/cli/pkg/utils"
)

func NewCmdDiagnose(globalOpts *types.GlobalCmdOptions) *cobra.Command {
cmd := &cobra.Command{
Use: consts.SubCmdDiagnose,
Short: "Diagnose the health of the Longhorn system",
}

utils.SetGlobalOptionsRemote(cmd, globalOpts)

cmd.AddCommand(newCmdDiagnoseSystem(globalOpts))
cmd.AddCommand(newCmdDiagnoseV2DataEngine(globalOpts))

return cmd
}

func newCmdDiagnoseSystem(globalOpts *types.GlobalCmdOptions) *cobra.Command {
var diagnoser = diagnose.Diagnoser{}
var results diagnose.Results
var outputFormat string

cmd := &cobra.Command{
Use: consts.SubCmdDiagnose,
Short: "Diagnose the health of the Longhorn system",
Use: consts.SubCmdSystem,
Short: "Diagnose the health of Longhorn system components and volumes",
Long: `This command checks whether the Longhorn system is healthy and fully functional. It is useful for verifying a Longhorn deployment, for example in automation.

The following checks are performed, ordered so that a component is checked before the components depending on it:
Expand All @@ -31,6 +46,8 @@ The following checks are performed, ordered so that a component is checked befor
- LonghornNodes: The Longhorn nodes are ready. Other node conditions and disks that are not ready are reported as warnings.
- EngineImages: The engine images are deployed on all nodes.
- InstanceManagers: The instance managers are running.
- ShareManagers: The share managers carry the required discovery labels.
- BackingImageManagers: The backing image managers carry the required discovery labels.
- CSIDriver: The Longhorn CSIDriver (driver.longhorn.io) is registered.
- CSIPlugin: The longhorn-csi-plugin DaemonSet pods are ready.
- CSIAttacher, CSIProvisioner, CSIResizer, CSISnapshotter: The CSI sidecar Deployments have ready pods.
Expand All @@ -42,8 +59,8 @@ The command only reads the cluster state. It does not create or modify any resou
The results are printed to stdout in YAML or JSON, and the logs are printed to stderr.

The command exits with a non-zero status if any check fails.`,
Example: `$ longhornctl diagnose
$ longhornctl diagnose -o json 2>/dev/null | jq '.[] | select(.status == "fail")'`,
Example: `$ longhornctl diagnose system
$ longhornctl diagnose system -o json 2>/dev/null | jq '.[] | select(.status == "fail")'`,

PreRun: func(cmd *cobra.Command, args []string) {
if !slices.Contains(diagnose.OutputFormats, outputFormat) {
Expand Down Expand Up @@ -94,3 +111,90 @@ $ longhornctl diagnose -o json 2>/dev/null | jq '.[] | select(.status == "fail")

return cmd
}

func newCmdDiagnoseV2DataEngine(globalOpts *types.GlobalCmdOptions) *cobra.Command {
var spdkDiagnoser = spdk.Diagnoser{}

cmd := &cobra.Command{
Use: consts.SubCmdV2DataEngine,
Short: "Diagnose Longhorn V2 Data Engine SPDK memory, iobuf, blobstore channels, and cross-node deadlocks",
Long: `This command inspects running V2 Data Engine spdk_tgt processes inside instance-manager pods across Longhorn nodes without pausing the SPDK reactor (using read-only /proc/<pid>/mem pread and SPDK JSON-RPC via a temporary read-only DaemonSet).

It collects, correlates, and diagnoses:
- Global iobuf pool sizes, live DPDK rte_ring free counts, and per-module allocation/retry counters.
- Per-thread iobuf wait queues and lockless module caches (such as nvmf_TCP large_pool availability).
- Blobstore and per-blob spdk_bs_channel request-set pools (freeReqs, needClusterAlloc queues, and active in-flight CoW / external snapshot reads).
- Bdev shared resources with queued nomem_io requests and active bdev_no_mem_poller counts.
- Cross-node NVMe-oF snapshot read dependencies and distributed deadlock cycles across stalled nodes.

The results are printed to stdout in YAML, JSON, or human-readable text (-o text), and logs are printed to stderr.
YAML and JSON outputs place the high-level cluster summary, deadlock cycles, and cross-node dependencies first while preserving all raw per-node SPDK data underneath.

The command exits with a non-zero status if any V2 Data Engine SPDK target is stalled or deadlocked.`,
Example: `$ longhornctl diagnose v2-data-engine
$ longhornctl diagnose v2-data-engine -o text
$ longhornctl diagnose v2-data-engine --node-id=worker-1 -o json 2>/dev/null | jq '.nodes["worker-1"]'`,

PreRun: func(cmd *cobra.Command, args []string) {
spdkDiagnoser.Image = globalOpts.Image
spdkDiagnoser.ImageRegistry = globalOpts.ImageRegistry
spdkDiagnoser.ImagePullSecret = globalOpts.ImagePullSecret
spdkDiagnoser.KubeConfigPath = globalOpts.KubeConfigPath
spdkDiagnoser.NodeSelector = globalOpts.NodeSelector
spdkDiagnoser.Tolerations = globalOpts.Tolerations
spdkDiagnoser.Namespace = globalOpts.Namespace
spdkDiagnoser.LogLevel = globalOpts.LogLevel

utils.CheckErr(spdkDiagnoser.Validate())

logrus.Info("Initializing V2 Data Engine diagnoser")
if err := spdkDiagnoser.Init(); err != nil {
utils.CheckErr(errors.Wrap(err, "Failed to initialize V2 Data Engine diagnoser"))
}

logrus.Info("Cleaning up V2 Data Engine diagnoser")
if err := spdkDiagnoser.Cleanup(); err != nil {
utils.CheckErr(errors.Wrap(err, "Failed to cleanup V2 Data Engine diagnoser"))
}
},

Run: func(cmd *cobra.Command, args []string) {
logrus.Info("Running V2 Data Engine diagnoser")
output, err := spdkDiagnoser.Run()
if err != nil {
if cleanupErr := spdkDiagnoser.Cleanup(); cleanupErr != nil {
logrus.WithError(cleanupErr).Warn("Failed to cleanup V2 Data Engine diagnoser after error")
}
utils.CheckErr(errors.Wrap(err, "Failed to run V2 Data Engine diagnoser"))
}

if _, err := cmd.OutOrStdout().Write(output); err != nil {
if cleanupErr := spdkDiagnoser.Cleanup(); cleanupErr != nil {
logrus.WithError(cleanupErr).Warn("Failed to cleanup V2 Data Engine diagnoser after output error")
}
utils.CheckErr(errors.Wrap(err, "Failed to write V2 Data Engine diagnoser output"))
}
},

PostRun: func(cmd *cobra.Command, args []string) {
logrus.Info("Cleaning up V2 Data Engine diagnoser")
if err := spdkDiagnoser.Cleanup(); err != nil {
utils.CheckErr(errors.Wrap(err, "Failed to cleanup V2 Data Engine diagnoser"))
}

if reason := spdkDiagnoser.UnhealthyReason(); reason != "" {
utils.CheckErr(errors.Errorf("Longhorn V2 Data Engine SPDK target is unhealthy (%s)", reason))
}

logrus.Info("Completed V2 Data Engine diagnoser")
},
}

utils.SetGlobalOptionsRemote(cmd, globalOpts)

cmd.Flags().StringVar(&spdkDiagnoser.NodeID, consts.CmdOptNodeId, "", "Inspect V2 Data Engine SPDK target processes only on the specified node.")
cmd.Flags().StringVarP(&spdkDiagnoser.OutputFormat, consts.CmdOptOutput, "o", spdk.OutputFormatYAML, fmt.Sprintf("Output format of the results. One of: %v.", strings.Join(spdk.OutputFormats, ", ")))
cmd.Flags().BoolVar(&spdkDiagnoser.IncludeAll, consts.CmdOptAll, false, "Include idle per-blob channels with 0 queued or active requests.")

return cmd
}
90 changes: 90 additions & 0 deletions cmd/remote/subcmd/diagnose_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package subcmd

import (
"bytes"
"testing"

"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/longhorn/cli/pkg/consts"
"github.com/longhorn/cli/pkg/types"
)

func newTestRootWithDiagnose(globalOpts *types.GlobalCmdOptions) *cobra.Command {
root := &cobra.Command{Use: consts.CmdLonghornctlRemote}
root.PersistentFlags().StringVarP(&globalOpts.LogLevel, consts.CmdOptLogLevel, "l", "info", "log level")
root.PersistentFlags().StringVar(&globalOpts.KubeConfigPath, consts.CmdOptKubeConfigPath, "", "kubeconfig")
root.PersistentFlags().StringVar(&globalOpts.Image, consts.CmdOptImage, consts.ImageLonghornCli, "image")
root.PersistentFlags().StringVar(&globalOpts.ImageRegistry, consts.CmdOptImageRegistry, "", "image-registry")
root.PersistentFlags().StringVar(&globalOpts.ImagePullSecret, consts.CmdOptImagePullSecret, "", "image-pull-secret")
root.PersistentFlags().StringVar(&globalOpts.NodeSelector, consts.CmdOptNodeSelector, "", "node-selector")
root.PersistentFlags().StringVar(&globalOpts.Tolerations, consts.CmdOptTolerations, "", "tolerations")
root.PersistentFlags().StringVar(&globalOpts.Namespace, consts.CmdOptNamespace, consts.NamespaceLonghorn, "namespace")
root.AddCommand(NewCmdDiagnose(globalOpts))
return root
}

func TestDiagnoseSystemAndV2DataEngineSubcommands(t *testing.T) {
globalOpts := &types.GlobalCmdOptions{}
root := newTestRootWithDiagnose(globalOpts)

// 1. Bare "diagnose" acts as a pure command group advertising "system" and "v2-data-engine".
var diagHelp bytes.Buffer
root.SetOut(&diagHelp)
root.SetArgs([]string{"diagnose"})
require.NoError(t, root.Execute())
assert.Contains(t, diagHelp.String(), "system")
assert.Contains(t, diagHelp.String(), "v2-data-engine")

// 2. "diagnose system --help" hides DaemonSet pod flags while keeping API check flags.
globalOpts = &types.GlobalCmdOptions{}
root = newTestRootWithDiagnose(globalOpts)
var sysHelp bytes.Buffer
root.SetOut(&sysHelp)
root.SetArgs([]string{"diagnose", "system", "--help"})
require.NoError(t, root.Execute())
assert.Contains(t, sysHelp.String(), "--kubeconfig ")
assert.Contains(t, sysHelp.String(), "One of: yaml, json.")
assert.NotContains(t, sysHelp.String(), "--image ")
assert.NotContains(t, sysHelp.String(), "--tolerations")

// 3. "diagnose v2-data-engine --help" exposes all DaemonSet and V2 Data Engine flags, with NO aliases.
globalOpts = &types.GlobalCmdOptions{}
root = newTestRootWithDiagnose(globalOpts)
var v2Help bytes.Buffer
root.SetOut(&v2Help)
root.SetArgs([]string{"diagnose", "v2-data-engine", "--help"})
require.NoError(t, root.Execute())
assert.Contains(t, v2Help.String(), "--image ")
assert.Contains(t, v2Help.String(), "--tolerations ")
assert.Contains(t, v2Help.String(), "--node-id ")
assert.Contains(t, v2Help.String(), "--all")
assert.Contains(t, v2Help.String(), "One of: yaml, json, text.")
assert.NotContains(t, v2Help.String(), "Aliases:")

// 4. Confirm "v2" and "spdk" are NOT registered aliases under "diagnose".
globalOpts = &types.GlobalCmdOptions{}
root = newTestRootWithDiagnose(globalOpts)
v2Cmd, _, err := root.Find([]string{"diagnose", "v2-data-engine"})
require.NoError(t, err)
assert.Empty(t, v2Cmd.Aliases)
for _, disallowed := range []string{"v2", "spdk"} {
found, _, _ := root.Find([]string{"diagnose", disallowed})
assert.NotEqual(t, v2Cmd, found)
}

// 5. Parsing flags on "diagnose v2-data-engine" populates globalOpts cleanly.
require.NoError(t, v2Cmd.ParseFlags([]string{
"--image", "derekbit/longhorn-cli:iobuf2",
"--tolerations", "node-role.kubernetes.io/control-plane:NoSchedule",
"--kubeconfig", "/tmp/kubeconfig",
"--node-id", "hp-23-tink-system",
"-o", "text",
"--all",
}))
assert.Equal(t, "derekbit/longhorn-cli:iobuf2", globalOpts.Image)
assert.Equal(t, "node-role.kubernetes.io/control-plane:NoSchedule", globalOpts.Tolerations)
assert.Equal(t, "/tmp/kubeconfig", globalOpts.KubeConfigPath)
}
4 changes: 2 additions & 2 deletions docs/longhornctl.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ A CLI tool for troubleshooting and managing Longhorn operations.

```
-h, --help help for longhornctl
--image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev")
--image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev")
--image-pull-secret string Secret with registry credentials for pulling images
--image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images.
--kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used
Expand All @@ -33,4 +33,4 @@ A CLI tool for troubleshooting and managing Longhorn operations.
* [longhornctl trim](longhornctl_trim.md) - Longhorn trimming operations
* [longhornctl version](longhornctl_version.md) - Print longhornctl version

###### Auto generated by spf13/cobra on 6-Jul-2026
###### Auto generated by spf13/cobra on 19-Nov-2026
4 changes: 2 additions & 2 deletions docs/longhornctl_check.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ Longhorn checking operations

```
-h, --help help for check
--image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev")
--image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev")
--image-pull-secret string Secret with registry credentials for pulling images
--image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images.
--kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used
Expand All @@ -26,4 +26,4 @@ Longhorn checking operations
* [longhornctl](longhornctl.md) - Command-line interface for Longhorn.
* [longhornctl check preflight](longhornctl_check_preflight.md) - Run a preflight check for Longhorn

###### Auto generated by spf13/cobra on 6-Jul-2026
###### Auto generated by spf13/cobra on 19-Nov-2026
Loading
Loading