From 576fb2a0c2bea1b56f3ae2be2d9001f7bf9c2bc6 Mon Sep 17 00:00:00 2001 From: Derek Su Date: Wed, 18 Nov 2026 15:30:28 +0000 Subject: [PATCH] feat(diagnose): add system and v2-data-engine subcommands Restructure `longhornctl diagnose` into a ` ` command group consistent with other `longhornctl` commands: - `longhornctl diagnose system`: moves the existing Kubernetes and Longhorn control-plane, CSI, instance-manager, and volume health checks under the `system` subcommand. - `longhornctl diagnose v2-data-engine` (and `longhornctl-local diagnose v2-data-engine`): inspects live V2 Data Engine `spdk_tgt` processes across nodes via a temporary read-only DaemonSet without pausing the SPDK reactor. `diagnose v2-data-engine` combines non-blocking SPDK JSON-RPC queries with ELF/DWARF-guided `/proc//mem` inspection to collect and correlate: - Global `iobuf` pool capacities, live DPDK `rte_ring` free counts, per-thread wait queues, and per-module lockless caches. - Shared `blobstore` and per-blob `spdk_bs_channel` request-set pools (`freeReqs`, per-lvol `need_cluster_alloc` queues, and in-flight `cluster_cow` / `esnap_cow` operations). - Queued `bdev` `nomem_io` requests (resolved to Longhorn lvol names) and active `bdev_no_mem_poller` counts. - Cross-node NVMe-oF external snapshot read dependencies, transitive stall propagation, and distributed deadlock cycles. Results support `-o yaml` (default), `-o json`, and `-o text`. YAML and JSON outputs surface the synthesized cluster summary, deadlock cycles, and cross-node dependencies ahead of the full raw per-node forensic data, and the command exits with a non-zero status after DaemonSet cleanup if any SPDK target is stalled or deadlocked. longhorn/longhorn#14589 Signed-off-by: Derek Su Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- cmd/local/longhornctl-local.go | 1 + cmd/local/subcmd/diagnose.go | 89 ++ cmd/local/subcmd/get.go | 5 +- cmd/remote/subcmd/diagnose.go | 112 ++- cmd/remote/subcmd/diagnose_test.go | 90 ++ docs/longhornctl.md | 4 +- docs/longhornctl_check.md | 4 +- docs/longhornctl_check_preflight.md | 4 +- docs/longhornctl_checksum.md | 4 +- docs/longhornctl_checksum_volume.md | 4 +- docs/longhornctl_diagnose.md | 2 +- docs/longhornctl_doc.md | 4 +- docs/longhornctl_export.md | 4 +- docs/longhornctl_export_replica.md | 6 +- docs/longhornctl_export_replica_stop.md | 4 +- docs/longhornctl_get.md | 4 +- docs/longhornctl_get_replica.md | 4 +- docs/longhornctl_global-options.md | 4 +- docs/longhornctl_install.md | 4 +- docs/longhornctl_install_preflight.md | 4 +- docs/longhornctl_install_preflight_stop.md | 4 +- docs/longhornctl_trim.md | 4 +- docs/longhornctl_trim_volume.md | 4 +- docs/longhornctl_version.md | 4 +- pkg/consts/cmd.go | 11 +- pkg/consts/environment.go | 3 + pkg/consts/spdk.go | 8 +- pkg/local/spdk/getter.go | 197 ++++ pkg/local/spdk/layout.go | 736 ++++++++++++++ pkg/local/spdk/mem.go | 674 +++++++++++++ pkg/local/spdk/mem_test.go | 471 +++++++++ pkg/local/spdk/rpc.go | 241 +++++ pkg/remote/spdk/getter.go | 1003 ++++++++++++++++++++ pkg/remote/spdk/getter_test.go | 272 ++++++ pkg/types/spdk.go | 193 ++++ 35 files changed, 4134 insertions(+), 48 deletions(-) create mode 100644 cmd/local/subcmd/diagnose.go create mode 100644 cmd/remote/subcmd/diagnose_test.go create mode 100644 pkg/local/spdk/getter.go create mode 100644 pkg/local/spdk/layout.go create mode 100644 pkg/local/spdk/mem.go create mode 100644 pkg/local/spdk/mem_test.go create mode 100644 pkg/local/spdk/rpc.go create mode 100644 pkg/remote/spdk/getter.go create mode 100644 pkg/remote/spdk/getter_test.go create mode 100644 pkg/types/spdk.go diff --git a/cmd/local/longhornctl-local.go b/cmd/local/longhornctl-local.go index 4334e718b..9b1bdb80d 100644 --- a/cmd/local/longhornctl-local.go +++ b/cmd/local/longhornctl-local.go @@ -59,6 +59,7 @@ func newCmdLonghornctlLocal() *cobra.Command { Message: "Troubleshoot Commands:", Commands: []*cobra.Command{ localsubcmd.NewCmdCheck(globalOpts), + localsubcmd.NewCmdDiagnose(globalOpts), localsubcmd.NewCmdGet(globalOpts), }, }, diff --git a/cmd/local/subcmd/diagnose.go b/cmd/local/subcmd/diagnose.go new file mode 100644 index 000000000..af143b718 --- /dev/null +++ b/cmd/local/subcmd/diagnose.go @@ -0,0 +1,89 @@ +package subcmd + +import ( + "os" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/types" + "github.com/longhorn/cli/pkg/utils" + + localspdk "github.com/longhorn/cli/pkg/local/spdk" +) + +func NewCmdDiagnose(globalOpts *types.GlobalCmdOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: consts.SubCmdDiagnose, + Short: "Diagnose the health of the Longhorn system", + } + + utils.SetGlobalOptionsLocal(cmd, globalOpts) + + cmd.AddCommand(newCmdDiagnoseV2DataEngine(globalOpts)) + + return cmd +} + +func newCmdDiagnoseV2DataEngine(globalOpts *types.GlobalCmdOptions) *cobra.Command { + var localDiagnoser = localspdk.Diagnoser{} + + cmd := &cobra.Command{ + Use: consts.SubCmdV2DataEngine, + Short: "Inspect V2 Data Engine SPDK target memory, iobuf pools, and blobstore channels", + Long: `This command performs a lockless, read-only inspection of running V2 Data Engine spdk_tgt processes on the node. + +It combines SPDK Unix-socket JSON-RPC queries with ELF/DWARF-guided /proc//mem inspection to report: +- Global iobuf small/large pool configuration, live DPDK ring free counts, and per-module retry statistics. +- Per-thread iobuf wait queues and lockless module caches (including nvmf_TCP large_pool availability). +- Blobstore and per-blob spdk_bs_channel request-set pools (freeReqs, needClusterAlloc queues, and active in-flight CoW / external snapshot reads). +- Bdev shared resources with queued nomem_io requests and active bdev_no_mem_poller counts.`, + + PreRun: func(cmd *cobra.Command, args []string) { + localDiagnoser.LogLevel = globalOpts.LogLevel + + if err := localDiagnoser.Init(); err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to initialize V2 Data Engine diagnoser")) + } + }, + + Run: func(cmd *cobra.Command, args []string) { + if err := localDiagnoser.Run(); err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to run V2 Data Engine diagnoser")) + } + + logrus.Info("Successfully collected V2 Data Engine SPDK diagnostics") + }, + + PostRun: func(cmd *cobra.Command, args []string) { + if err := localDiagnoser.Output(); err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to output V2 Data Engine SPDK diagnostics")) + } + + logrus.Info("Successfully output V2 Data Engine SPDK diagnostics") + }, + } + + utils.SetGlobalOptionsLocal(cmd, globalOpts) + + defaultPID := utils.ConvertStringToTypeOrDefault(os.Getenv(consts.EnvSpdkPid), 0) + defaultProcDir := os.Getenv(consts.EnvProcDir) + if defaultProcDir == "" { + defaultProcDir = consts.DefaultProcDirectory + } + defaultSock := os.Getenv(consts.EnvSpdkSocket) + if defaultSock == "" { + defaultSock = consts.DefaultSpdkSocketPath + } + + cmd.Flags().StringVar(&localDiagnoser.CurrentNodeID, consts.CmdOptNodeId, os.Getenv(consts.EnvCurrentNodeID), "Current node ID.") + cmd.Flags().StringVarP(&localDiagnoser.OutputFilePath, consts.CmdOptOutputFile, "o", os.Getenv(consts.EnvOutputFilePath), "Output the result to a file, default to stdout.") + cmd.Flags().StringVar(&localDiagnoser.ProcDir, consts.CmdOptProcDir, defaultProcDir, "Path to the procfs directory used to discover and inspect spdk_tgt.") + cmd.Flags().StringVar(&localDiagnoser.SpdkSocketPath, consts.CmdOptSpdkSocket, defaultSock, "Path to the SPDK JSON-RPC Unix domain socket inside the target mount namespace.") + cmd.Flags().IntVar(&localDiagnoser.TargetPID, consts.CmdOptPid, defaultPID, "Inspect a specific spdk_tgt PID instead of scanning procfs.") + cmd.Flags().BoolVar(&localDiagnoser.IncludeAll, consts.CmdOptAll, false, "Include idle per-blob channels with 0 queued or active requests.") + + return cmd +} diff --git a/cmd/local/subcmd/get.go b/cmd/local/subcmd/get.go index 485d47f72..17a164a8f 100644 --- a/cmd/local/subcmd/get.go +++ b/cmd/local/subcmd/get.go @@ -8,9 +8,10 @@ import ( "github.com/spf13/cobra" "github.com/longhorn/cli/pkg/consts" - local "github.com/longhorn/cli/pkg/local/replica" "github.com/longhorn/cli/pkg/types" "github.com/longhorn/cli/pkg/utils" + + localreplica "github.com/longhorn/cli/pkg/local/replica" ) func NewCmdGet(globalOpts *types.GlobalCmdOptions) *cobra.Command { @@ -27,7 +28,7 @@ func NewCmdGet(globalOpts *types.GlobalCmdOptions) *cobra.Command { } func newCmdGetReplica(globalOpts *types.GlobalCmdOptions) *cobra.Command { - var localGetter = local.Getter{} + var localGetter = localreplica.Getter{} cmd := &cobra.Command{ Use: consts.SubCmdReplica, diff --git a/cmd/remote/subcmd/diagnose.go b/cmd/remote/subcmd/diagnose.go index 7f95deb97..37a67a82e 100644 --- a/cmd/remote/subcmd/diagnose.go +++ b/cmd/remote/subcmd/diagnose.go @@ -11,18 +11,33 @@ import ( "github.com/longhorn/cli/pkg/consts" "github.com/longhorn/cli/pkg/remote/diagnose" + "github.com/longhorn/cli/pkg/remote/spdk" "github.com/longhorn/cli/pkg/types" "github.com/longhorn/cli/pkg/utils" ) func NewCmdDiagnose(globalOpts *types.GlobalCmdOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: consts.SubCmdDiagnose, + Short: "Diagnose the health of the Longhorn system", + } + + utils.SetGlobalOptionsRemote(cmd, globalOpts) + + cmd.AddCommand(newCmdDiagnoseSystem(globalOpts)) + cmd.AddCommand(newCmdDiagnoseV2DataEngine(globalOpts)) + + return cmd +} + +func newCmdDiagnoseSystem(globalOpts *types.GlobalCmdOptions) *cobra.Command { var diagnoser = diagnose.Diagnoser{} var results diagnose.Results var outputFormat string cmd := &cobra.Command{ - Use: consts.SubCmdDiagnose, - Short: "Diagnose the health of the Longhorn system", + Use: consts.SubCmdSystem, + Short: "Diagnose the health of Longhorn system components and volumes", Long: `This command checks whether the Longhorn system is healthy and fully functional. It is useful for verifying a Longhorn deployment, for example in automation. The following checks are performed, ordered so that a component is checked before the components depending on it: @@ -31,6 +46,8 @@ The following checks are performed, ordered so that a component is checked befor - LonghornNodes: The Longhorn nodes are ready. Other node conditions and disks that are not ready are reported as warnings. - EngineImages: The engine images are deployed on all nodes. - InstanceManagers: The instance managers are running. +- ShareManagers: The share managers carry the required discovery labels. +- BackingImageManagers: The backing image managers carry the required discovery labels. - CSIDriver: The Longhorn CSIDriver (driver.longhorn.io) is registered. - CSIPlugin: The longhorn-csi-plugin DaemonSet pods are ready. - CSIAttacher, CSIProvisioner, CSIResizer, CSISnapshotter: The CSI sidecar Deployments have ready pods. @@ -42,8 +59,8 @@ The command only reads the cluster state. It does not create or modify any resou The results are printed to stdout in YAML or JSON, and the logs are printed to stderr. The command exits with a non-zero status if any check fails.`, - Example: `$ longhornctl diagnose -$ longhornctl diagnose -o json 2>/dev/null | jq '.[] | select(.status == "fail")'`, + Example: `$ longhornctl diagnose system +$ longhornctl diagnose system -o json 2>/dev/null | jq '.[] | select(.status == "fail")'`, PreRun: func(cmd *cobra.Command, args []string) { if !slices.Contains(diagnose.OutputFormats, outputFormat) { @@ -94,3 +111,90 @@ $ longhornctl diagnose -o json 2>/dev/null | jq '.[] | select(.status == "fail") return cmd } + +func newCmdDiagnoseV2DataEngine(globalOpts *types.GlobalCmdOptions) *cobra.Command { + var spdkDiagnoser = spdk.Diagnoser{} + + cmd := &cobra.Command{ + Use: consts.SubCmdV2DataEngine, + Short: "Diagnose Longhorn V2 Data Engine SPDK memory, iobuf, blobstore channels, and cross-node deadlocks", + Long: `This command inspects running V2 Data Engine spdk_tgt processes inside instance-manager pods across Longhorn nodes without pausing the SPDK reactor (using read-only /proc//mem pread and SPDK JSON-RPC via a temporary read-only DaemonSet). + +It collects, correlates, and diagnoses: +- Global iobuf pool sizes, live DPDK rte_ring free counts, and per-module allocation/retry counters. +- Per-thread iobuf wait queues and lockless module caches (such as nvmf_TCP large_pool availability). +- Blobstore and per-blob spdk_bs_channel request-set pools (freeReqs, needClusterAlloc queues, and active in-flight CoW / external snapshot reads). +- Bdev shared resources with queued nomem_io requests and active bdev_no_mem_poller counts. +- Cross-node NVMe-oF snapshot read dependencies and distributed deadlock cycles across stalled nodes. + +The results are printed to stdout in YAML, JSON, or human-readable text (-o text), and logs are printed to stderr. +YAML and JSON outputs place the high-level cluster summary, deadlock cycles, and cross-node dependencies first while preserving all raw per-node SPDK data underneath. + +The command exits with a non-zero status if any V2 Data Engine SPDK target is stalled or deadlocked.`, + Example: `$ longhornctl diagnose v2-data-engine +$ longhornctl diagnose v2-data-engine -o text +$ longhornctl diagnose v2-data-engine --node-id=worker-1 -o json 2>/dev/null | jq '.nodes["worker-1"]'`, + + PreRun: func(cmd *cobra.Command, args []string) { + spdkDiagnoser.Image = globalOpts.Image + spdkDiagnoser.ImageRegistry = globalOpts.ImageRegistry + spdkDiagnoser.ImagePullSecret = globalOpts.ImagePullSecret + spdkDiagnoser.KubeConfigPath = globalOpts.KubeConfigPath + spdkDiagnoser.NodeSelector = globalOpts.NodeSelector + spdkDiagnoser.Tolerations = globalOpts.Tolerations + spdkDiagnoser.Namespace = globalOpts.Namespace + spdkDiagnoser.LogLevel = globalOpts.LogLevel + + utils.CheckErr(spdkDiagnoser.Validate()) + + logrus.Info("Initializing V2 Data Engine diagnoser") + if err := spdkDiagnoser.Init(); err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to initialize V2 Data Engine diagnoser")) + } + + logrus.Info("Cleaning up V2 Data Engine diagnoser") + if err := spdkDiagnoser.Cleanup(); err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to cleanup V2 Data Engine diagnoser")) + } + }, + + Run: func(cmd *cobra.Command, args []string) { + logrus.Info("Running V2 Data Engine diagnoser") + output, err := spdkDiagnoser.Run() + if err != nil { + if cleanupErr := spdkDiagnoser.Cleanup(); cleanupErr != nil { + logrus.WithError(cleanupErr).Warn("Failed to cleanup V2 Data Engine diagnoser after error") + } + utils.CheckErr(errors.Wrap(err, "Failed to run V2 Data Engine diagnoser")) + } + + if _, err := cmd.OutOrStdout().Write(output); err != nil { + if cleanupErr := spdkDiagnoser.Cleanup(); cleanupErr != nil { + logrus.WithError(cleanupErr).Warn("Failed to cleanup V2 Data Engine diagnoser after output error") + } + utils.CheckErr(errors.Wrap(err, "Failed to write V2 Data Engine diagnoser output")) + } + }, + + PostRun: func(cmd *cobra.Command, args []string) { + logrus.Info("Cleaning up V2 Data Engine diagnoser") + if err := spdkDiagnoser.Cleanup(); err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to cleanup V2 Data Engine diagnoser")) + } + + if reason := spdkDiagnoser.UnhealthyReason(); reason != "" { + utils.CheckErr(errors.Errorf("Longhorn V2 Data Engine SPDK target is unhealthy (%s)", reason)) + } + + logrus.Info("Completed V2 Data Engine diagnoser") + }, + } + + utils.SetGlobalOptionsRemote(cmd, globalOpts) + + cmd.Flags().StringVar(&spdkDiagnoser.NodeID, consts.CmdOptNodeId, "", "Inspect V2 Data Engine SPDK target processes only on the specified node.") + cmd.Flags().StringVarP(&spdkDiagnoser.OutputFormat, consts.CmdOptOutput, "o", spdk.OutputFormatYAML, fmt.Sprintf("Output format of the results. One of: %v.", strings.Join(spdk.OutputFormats, ", "))) + cmd.Flags().BoolVar(&spdkDiagnoser.IncludeAll, consts.CmdOptAll, false, "Include idle per-blob channels with 0 queued or active requests.") + + return cmd +} diff --git a/cmd/remote/subcmd/diagnose_test.go b/cmd/remote/subcmd/diagnose_test.go new file mode 100644 index 000000000..4e4d21dc5 --- /dev/null +++ b/cmd/remote/subcmd/diagnose_test.go @@ -0,0 +1,90 @@ +package subcmd + +import ( + "bytes" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/types" +) + +func newTestRootWithDiagnose(globalOpts *types.GlobalCmdOptions) *cobra.Command { + root := &cobra.Command{Use: consts.CmdLonghornctlRemote} + root.PersistentFlags().StringVarP(&globalOpts.LogLevel, consts.CmdOptLogLevel, "l", "info", "log level") + root.PersistentFlags().StringVar(&globalOpts.KubeConfigPath, consts.CmdOptKubeConfigPath, "", "kubeconfig") + root.PersistentFlags().StringVar(&globalOpts.Image, consts.CmdOptImage, consts.ImageLonghornCli, "image") + root.PersistentFlags().StringVar(&globalOpts.ImageRegistry, consts.CmdOptImageRegistry, "", "image-registry") + root.PersistentFlags().StringVar(&globalOpts.ImagePullSecret, consts.CmdOptImagePullSecret, "", "image-pull-secret") + root.PersistentFlags().StringVar(&globalOpts.NodeSelector, consts.CmdOptNodeSelector, "", "node-selector") + root.PersistentFlags().StringVar(&globalOpts.Tolerations, consts.CmdOptTolerations, "", "tolerations") + root.PersistentFlags().StringVar(&globalOpts.Namespace, consts.CmdOptNamespace, consts.NamespaceLonghorn, "namespace") + root.AddCommand(NewCmdDiagnose(globalOpts)) + return root +} + +func TestDiagnoseSystemAndV2DataEngineSubcommands(t *testing.T) { + globalOpts := &types.GlobalCmdOptions{} + root := newTestRootWithDiagnose(globalOpts) + + // 1. Bare "diagnose" acts as a pure command group advertising "system" and "v2-data-engine". + var diagHelp bytes.Buffer + root.SetOut(&diagHelp) + root.SetArgs([]string{"diagnose"}) + require.NoError(t, root.Execute()) + assert.Contains(t, diagHelp.String(), "system") + assert.Contains(t, diagHelp.String(), "v2-data-engine") + + // 2. "diagnose system --help" hides DaemonSet pod flags while keeping API check flags. + globalOpts = &types.GlobalCmdOptions{} + root = newTestRootWithDiagnose(globalOpts) + var sysHelp bytes.Buffer + root.SetOut(&sysHelp) + root.SetArgs([]string{"diagnose", "system", "--help"}) + require.NoError(t, root.Execute()) + assert.Contains(t, sysHelp.String(), "--kubeconfig ") + assert.Contains(t, sysHelp.String(), "One of: yaml, json.") + assert.NotContains(t, sysHelp.String(), "--image ") + assert.NotContains(t, sysHelp.String(), "--tolerations") + + // 3. "diagnose v2-data-engine --help" exposes all DaemonSet and V2 Data Engine flags, with NO aliases. + globalOpts = &types.GlobalCmdOptions{} + root = newTestRootWithDiagnose(globalOpts) + var v2Help bytes.Buffer + root.SetOut(&v2Help) + root.SetArgs([]string{"diagnose", "v2-data-engine", "--help"}) + require.NoError(t, root.Execute()) + assert.Contains(t, v2Help.String(), "--image ") + assert.Contains(t, v2Help.String(), "--tolerations ") + assert.Contains(t, v2Help.String(), "--node-id ") + assert.Contains(t, v2Help.String(), "--all") + assert.Contains(t, v2Help.String(), "One of: yaml, json, text.") + assert.NotContains(t, v2Help.String(), "Aliases:") + + // 4. Confirm "v2" and "spdk" are NOT registered aliases under "diagnose". + globalOpts = &types.GlobalCmdOptions{} + root = newTestRootWithDiagnose(globalOpts) + v2Cmd, _, err := root.Find([]string{"diagnose", "v2-data-engine"}) + require.NoError(t, err) + assert.Empty(t, v2Cmd.Aliases) + for _, disallowed := range []string{"v2", "spdk"} { + found, _, _ := root.Find([]string{"diagnose", disallowed}) + assert.NotEqual(t, v2Cmd, found) + } + + // 5. Parsing flags on "diagnose v2-data-engine" populates globalOpts cleanly. + require.NoError(t, v2Cmd.ParseFlags([]string{ + "--image", "derekbit/longhorn-cli:iobuf2", + "--tolerations", "node-role.kubernetes.io/control-plane:NoSchedule", + "--kubeconfig", "/tmp/kubeconfig", + "--node-id", "hp-23-tink-system", + "-o", "text", + "--all", + })) + assert.Equal(t, "derekbit/longhorn-cli:iobuf2", globalOpts.Image) + assert.Equal(t, "node-role.kubernetes.io/control-plane:NoSchedule", globalOpts.Tolerations) + assert.Equal(t, "/tmp/kubeconfig", globalOpts.KubeConfigPath) +} diff --git a/docs/longhornctl.md b/docs/longhornctl.md index c79fb8035..ce7815db2 100644 --- a/docs/longhornctl.md +++ b/docs/longhornctl.md @@ -10,7 +10,7 @@ A CLI tool for troubleshooting and managing Longhorn operations. ``` -h, --help help for longhornctl - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -33,4 +33,4 @@ A CLI tool for troubleshooting and managing Longhorn operations. * [longhornctl trim](longhornctl_trim.md) - Longhorn trimming operations * [longhornctl version](longhornctl_version.md) - Print longhornctl version -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_check.md b/docs/longhornctl_check.md index 3884e0a2a..4992a08ed 100644 --- a/docs/longhornctl_check.md +++ b/docs/longhornctl_check.md @@ -6,7 +6,7 @@ Longhorn checking operations ``` -h, --help help for check - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -26,4 +26,4 @@ Longhorn checking operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl check preflight](longhornctl_check_preflight.md) - Run a preflight check for Longhorn -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_check_preflight.md b/docs/longhornctl_check_preflight.md index df4fecab1..e4e48bbb9 100644 --- a/docs/longhornctl_check_preflight.md +++ b/docs/longhornctl_check_preflight.md @@ -46,7 +46,7 @@ INFO[2024-07-16T17:17:42+08:00] Completed preflight checker --enable-spdk Enable checking of SPDK required packages, modules, and setup. -h, --help help for preflight --huge-page-size int Specify the huge page size in MiB for SPDK. (default 2048) - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -66,4 +66,4 @@ INFO[2024-07-16T17:17:42+08:00] Completed preflight checker * [longhornctl check](longhornctl_check.md) - Longhorn checking operations -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_checksum.md b/docs/longhornctl_checksum.md index 5c68c6a64..2df639a2c 100644 --- a/docs/longhornctl_checksum.md +++ b/docs/longhornctl_checksum.md @@ -6,7 +6,7 @@ Snapshot checksum operations ``` -h, --help help for checksum - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -26,4 +26,4 @@ Snapshot checksum operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl checksum volume](longhornctl_checksum_volume.md) - Trigger on-demand snapshot checksum calculation for a volume -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_checksum_volume.md b/docs/longhornctl_checksum_volume.md index 2ee7465dd..36b4313f9 100644 --- a/docs/longhornctl_checksum_volume.md +++ b/docs/longhornctl_checksum_volume.md @@ -27,7 +27,7 @@ $ longhornctl checksum volume --name=v1 --namespace=longhorn-system ``` --all Apply to all volumes -h, --help help for volume - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -48,4 +48,4 @@ $ longhornctl checksum volume --name=v1 --namespace=longhorn-system * [longhornctl checksum](longhornctl_checksum.md) - Snapshot checksum operations -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_diagnose.md b/docs/longhornctl_diagnose.md index 9f61ad992..ff4191346 100644 --- a/docs/longhornctl_diagnose.md +++ b/docs/longhornctl_diagnose.md @@ -49,4 +49,4 @@ $ longhornctl diagnose -o json 2>/dev/null | jq '.[] | select(.status == "fail") * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. -###### Auto generated by spf13/cobra on 29-Sep-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_doc.md b/docs/longhornctl_doc.md index 85202ec13..955bc1f0b 100644 --- a/docs/longhornctl_doc.md +++ b/docs/longhornctl_doc.md @@ -19,7 +19,7 @@ longhornctl doc [output directory] [flags] ### Options inherited from parent commands ``` - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -33,4 +33,4 @@ longhornctl doc [output directory] [flags] * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_export.md b/docs/longhornctl_export.md index 2f770b83d..e7c0c4ffe 100644 --- a/docs/longhornctl_export.md +++ b/docs/longhornctl_export.md @@ -6,7 +6,7 @@ Export Longhorn resources ``` -h, --help help for export - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -26,4 +26,4 @@ Export Longhorn resources * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl export replica](longhornctl_export_replica.md) - Export data from a Longhorn replica -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_export_replica.md b/docs/longhornctl_export_replica.md index b8970ee9c..c1ceb2809 100644 --- a/docs/longhornctl_export_replica.md +++ b/docs/longhornctl_export_replica.md @@ -48,9 +48,9 @@ lost+found ``` --data-dir string Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg. (default "/var/lib/longhorn") - --engine-image string Engine image to use to create volume from the replica. (default "longhornio/longhorn-engine:v1.13.0-dev") + --engine-image string Engine image to use to create volume from the replica. (default "longhornio/longhorn-engine:v1.14.0-dev") -h, --help help for replica - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -72,4 +72,4 @@ lost+found * [longhornctl export](longhornctl_export.md) - Export Longhorn resources * [longhornctl export replica stop](longhornctl_export_replica_stop.md) - Stop the replica export process -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_export_replica_stop.md b/docs/longhornctl_export_replica_stop.md index 09c6c8625..59e1e4a17 100644 --- a/docs/longhornctl_export_replica_stop.md +++ b/docs/longhornctl_export_replica_stop.md @@ -22,7 +22,7 @@ INFO[2024-07-16T17:29:14+08:00] Successfully stopped exporting replica ``` -h, --help help for stop - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -41,4 +41,4 @@ INFO[2024-07-16T17:29:14+08:00] Successfully stopped exporting replica * [longhornctl export replica](longhornctl_export_replica.md) - Export data from a Longhorn replica -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_get.md b/docs/longhornctl_get.md index 1e10687dd..10850666c 100644 --- a/docs/longhornctl_get.md +++ b/docs/longhornctl_get.md @@ -6,7 +6,7 @@ Longhorn information gathering operations ``` -h, --help help for get - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -26,4 +26,4 @@ Longhorn information gathering operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl get replica](longhornctl_get_replica.md) - Retrieve Longhorn replica information -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_get_replica.md b/docs/longhornctl_get_replica.md index 6f455a802..a6ef71cf1 100644 --- a/docs/longhornctl_get_replica.md +++ b/docs/longhornctl_get_replica.md @@ -49,7 +49,7 @@ INFO[2024-07-16T17:23:51+08:00] Completed replica getter ``` --data-dir string Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg. (default "/var/lib/longhorn") -h, --help help for replica - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -70,4 +70,4 @@ INFO[2024-07-16T17:23:51+08:00] Completed replica getter * [longhornctl get](longhornctl_get.md) - Longhorn information gathering operations -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_global-options.md b/docs/longhornctl_global-options.md index 59fa07502..0f3e58c87 100644 --- a/docs/longhornctl_global-options.md +++ b/docs/longhornctl_global-options.md @@ -19,7 +19,7 @@ longhornctl global-options [flags] ### Options inherited from parent commands ``` - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -33,4 +33,4 @@ longhornctl global-options [flags] * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_install.md b/docs/longhornctl_install.md index 1fbc9ac7e..c7d8d0d21 100644 --- a/docs/longhornctl_install.md +++ b/docs/longhornctl_install.md @@ -6,7 +6,7 @@ Longhorn installation operations ``` -h, --help help for install - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -26,4 +26,4 @@ Longhorn installation operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl install preflight](longhornctl_install_preflight.md) - Install Longhorn preflight -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_install_preflight.md b/docs/longhornctl_install_preflight.md index 8530c50f0..2d7e88467 100644 --- a/docs/longhornctl_install_preflight.md +++ b/docs/longhornctl_install_preflight.md @@ -51,7 +51,7 @@ If a reboot is required, the following message will be displayed: --enable-spdk Enable installation of SPDK required packages, modules, and setup. -h, --help help for preflight --huge-page-size int Specify the huge page size in MiB for SPDK. (default 2048) - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -76,4 +76,4 @@ If a reboot is required, the following message will be displayed: * [longhornctl install](longhornctl_install.md) - Longhorn installation operations * [longhornctl install preflight stop](longhornctl_install_preflight_stop.md) - Stop Longhorn preflight installer -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_install_preflight_stop.md b/docs/longhornctl_install_preflight_stop.md index 552e0e521..0ddff76e1 100644 --- a/docs/longhornctl_install_preflight_stop.md +++ b/docs/longhornctl_install_preflight_stop.md @@ -22,7 +22,7 @@ INFO[2024-07-16T17:21:32+08:00] Successfully stopped preflight installer ``` -h, --help help for stop - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -42,4 +42,4 @@ INFO[2024-07-16T17:21:32+08:00] Successfully stopped preflight installer * [longhornctl install preflight](longhornctl_install_preflight.md) - Install Longhorn preflight -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_trim.md b/docs/longhornctl_trim.md index 07fa7054b..956c2722b 100644 --- a/docs/longhornctl_trim.md +++ b/docs/longhornctl_trim.md @@ -6,7 +6,7 @@ Longhorn trimming operations ``` -h, --help help for trim - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -26,4 +26,4 @@ Longhorn trimming operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl trim volume](longhornctl_trim_volume.md) - Trim a Longhorn volume -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_trim_volume.md b/docs/longhornctl_trim_volume.md index 144729679..5bcfa6a4d 100644 --- a/docs/longhornctl_trim_volume.md +++ b/docs/longhornctl_trim_volume.md @@ -31,7 +31,7 @@ INFO[2024-07-16T17:32:01+08:00] Completed volume trimmer vo ``` -h, --help help for volume - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -51,4 +51,4 @@ INFO[2024-07-16T17:32:01+08:00] Completed volume trimmer vo * [longhornctl trim](longhornctl_trim.md) - Longhorn trimming operations -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/docs/longhornctl_version.md b/docs/longhornctl_version.md index fca9eeff1..47965614f 100644 --- a/docs/longhornctl_version.md +++ b/docs/longhornctl_version.md @@ -15,7 +15,7 @@ longhornctl version [flags] ### Options inherited from parent commands ``` - --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.14.0-dev") --image-pull-secret string Secret with registry credentials for pulling images --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. --kubeconfig string Kubernetes config (kubeconfig) path. If not set, $KUBECONFIG or ~/.kube/config is used @@ -29,4 +29,4 @@ longhornctl version [flags] * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 19-Nov-2026 diff --git a/pkg/consts/cmd.go b/pkg/consts/cmd.go index 94b3175a8..aab695277 100644 --- a/pkg/consts/cmd.go +++ b/pkg/consts/cmd.go @@ -16,9 +16,11 @@ const ( SubCmdTrim = "trim" // The second layer of subcommands (noun) - SubCmdPreflight = "preflight" - SubCmdReplica = "replica" - SubCmdVolume = "volume" + SubCmdPreflight = "preflight" + SubCmdReplica = "replica" + SubCmdSystem = "system" + SubCmdV2DataEngine = "v2-data-engine" + SubCmdVolume = "volume" // The third layer of subcommands (action to the previous layers) SubCmdStop = "stop" @@ -53,7 +55,10 @@ const ( CmdOptDriverOverride = "driver-override" CmdOptEnableSpdk = "enable-spdk" CmdOptHugePageSize = "huge-page-size" + CmdOptPid = "pid" + CmdOptProcDir = "proc-dir" CmdOptSpdkOptions = "spdk-options" + CmdOptSpdkSocket = "spdk-sock" CmdOptUserspaceDriver = "userspace-driver" CmdOptRestartKubelet = "restart-kubelet" CmdOptRestartKubeletWindow = "restart-kubelet-window" diff --git a/pkg/consts/environment.go b/pkg/consts/environment.go index d1809fd4b..8d318eccf 100644 --- a/pkg/consts/environment.go +++ b/pkg/consts/environment.go @@ -23,6 +23,9 @@ const ( EnvEnableSpdk = "ENABLE_SPDK" EnvHugePageSize = "HUGEMEM" EnvPciAllowed = "PCI_ALLOWED" + EnvProcDir = "PROC_DIR" + EnvSpdkPid = "SPDK_PID" + EnvSpdkSocket = "SPDK_SOCKET" EnvUserspaceDriver = "USERSPACE_DRIVER" EnvUpdatePackageList = "UPDATE_PACKAGE_LIST" EnvSpdkOptions = "SPDK_OPTIONS" diff --git a/pkg/consts/spdk.go b/pkg/consts/spdk.go index d7c44cadd..f6a9c938c 100644 --- a/pkg/consts/spdk.go +++ b/pkg/consts/spdk.go @@ -1,3 +1,9 @@ package consts -const SpdkPath = "/tmp/longhorn-spdk" +const ( + SpdkPath = "/tmp/longhorn-spdk" + + AppNameV2DataEngineDiagnoser = "longhorn-v2-data-engine-diagnoser" + DefaultProcDirectory = "/proc" + DefaultSpdkSocketPath = "/var/tmp/spdk.sock" +) diff --git a/pkg/local/spdk/getter.go b/pkg/local/spdk/getter.go new file mode 100644 index 000000000..82b3ab6b5 --- /dev/null +++ b/pkg/local/spdk/getter.go @@ -0,0 +1,197 @@ +package spdk + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/types" + "github.com/longhorn/cli/pkg/utils" +) + +// Diagnoser collects live SPDK memory, iobuf, blobstore channel, and RPC diagnostics on a node. +type Diagnoser struct { + LogLevel string + + CurrentNodeID string + OutputFilePath string + ProcDir string + SpdkSocketPath string + TargetPID int + IncludeAll bool + + logger *logrus.Entry + nodeInfo types.SpdkNodeInfo +} + +// Init initializes the local SPDK Diagnoser. +func (local *Diagnoser) Init() error { + if local.OutputFilePath != "" { + local.logger = logrus.WithField("output", local.OutputFilePath) + } else { + local.logger = logrus.WithField("output", "stdout") + } + + if local.ProcDir == "" { + local.ProcDir = consts.DefaultProcDirectory + } + if local.SpdkSocketPath == "" { + local.SpdkSocketPath = consts.DefaultSpdkSocketPath + } + + local.nodeInfo = types.SpdkNodeInfo{ + Node: local.CurrentNodeID, + Processes: []*types.SpdkProcessInfo{}, + } + return nil +} + +// Run discovers spdk_tgt processes and inspects their RPC state and /proc//mem structures. +func (local *Diagnoser) Run() error { + pids, err := local.discoverSpdkPIDs() + if err != nil { + return err + } + if len(pids) == 0 { + local.nodeInfo.Warnings = append(local.nodeInfo.Warnings, + fmt.Sprintf("no running spdk_tgt process found under %s", local.ProcDir)) + return nil + } + + for _, pid := range pids { + procInfo := local.inspectPID(pid) + local.nodeInfo.Processes = append(local.nodeInfo.Processes, procInfo) + } + return nil +} + +// Output writes the collected SpdkNodeInfo JSON to OutputFilePath or stdout. +func (local *Diagnoser) Output() error { + jsonBytes, err := json.MarshalIndent(local.nodeInfo, "", " ") + if err != nil { + return errors.Wrap(err, "failed to marshal SPDK diagnostics to JSON") + } + return utils.HandleResult(jsonBytes, local.OutputFilePath, local.logger) +} + +func (local *Diagnoser) discoverSpdkPIDs() ([]int, error) { + if local.TargetPID > 0 { + memPath := filepath.Join(local.ProcDir, strconv.Itoa(local.TargetPID), "mem") + if _, err := os.Stat(memPath); err != nil { + return nil, errors.Wrapf(err, "target SPDK PID %d not accessible at %s", local.TargetPID, memPath) + } + return []int{local.TargetPID}, nil + } + + entries, err := os.ReadDir(local.ProcDir) + if err != nil { + return nil, errors.Wrapf(err, "failed to read proc directory %s", local.ProcDir) + } + + var pids []int + for _, entry := range entries { + if !entry.IsDir() { + continue + } + pid, err := strconv.Atoi(entry.Name()) + if err != nil || pid <= 0 { + continue + } + if local.isSpdkTargetProcess(pid) { + pids = append(pids, pid) + } + } + sort.Ints(pids) + return pids, nil +} + +func (local *Diagnoser) isSpdkTargetProcess(pid int) bool { + pidDir := filepath.Join(local.ProcDir, strconv.Itoa(pid)) + if exeTarget, err := os.Readlink(filepath.Join(pidDir, "exe")); err == nil { + cleanExe := strings.TrimSuffix(exeTarget, " (deleted)") + if filepath.Base(cleanExe) == "spdk_tgt" { + return true + } + } + + rawCmdline, err := os.ReadFile(filepath.Join(pidDir, "cmdline")) + if err != nil || len(rawCmdline) == 0 { + return false + } + args := bytes.Split(bytes.TrimRight(rawCmdline, "\x00"), []byte{0}) + if len(args) == 0 { + return false + } + return filepath.Base(string(args[0])) == "spdk_tgt" +} + +func (local *Diagnoser) inspectPID(pid int) *types.SpdkProcessInfo { + pidDir := filepath.Join(local.ProcDir, strconv.Itoa(pid)) + exePath := filepath.Join(pidDir, "exe") + mapsPath := filepath.Join(pidDir, "maps") + memPath := filepath.Join(pidDir, "mem") + + procInfo := &types.SpdkProcessInfo{ + PID: pid, + Cmdline: readFormattedCmdline(filepath.Join(pidDir, "cmdline")), + } + if exeTarget, err := os.Readlink(exePath); err == nil { + procInfo.Executable = exeTarget + } + + sockPath := local.resolveSocketPath(pidDir) + rpcSnap := collectSpdkRPC(sockPath) + procInfo.Version = rpcSnap.version + procInfo.NvmfPollGroups = rpcSnap.nvmfPollGroups + procInfo.Warnings = append(procInfo.Warnings, rpcSnap.warnings...) + + syms, layout, err := loadELFMetadata(exePath, mapsPath) + if err != nil { + procInfo.Errors = append(procInfo.Errors, err.Error()) + return procInfo + } + procInfo.PerBlobChannelSupported = syms.perBlobChannelSupported + procInfo.DWARFLoaded = layout.dwarfLoaded + + memFile, err := os.Open(memPath) + if err != nil { + procInfo.Errors = append(procInfo.Errors, errors.Wrapf(err, "failed to open %s", memPath).Error()) + return procInfo + } + defer func() { + _ = memFile.Close() + }() + + inspectProcessMemory(memFile, syms, layout, rpcSnap, local.IncludeAll, procInfo) + return procInfo +} + +func (local *Diagnoser) resolveSocketPath(pidDir string) string { + nsRootSock := filepath.Join(pidDir, "root", strings.TrimPrefix(local.SpdkSocketPath, "/")) + if _, err := os.Stat(nsRootSock); err == nil { + return nsRootSock + } + return local.SpdkSocketPath +} + +func readFormattedCmdline(path string) string { + raw, err := os.ReadFile(path) + if err != nil || len(raw) == 0 { + return "" + } + parts := bytes.Split(bytes.TrimRight(raw, "\x00"), []byte{0}) + strs := make([]string, 0, len(parts)) + for _, p := range parts { + strs = append(strs, string(p)) + } + return strings.Join(strs, " ") +} diff --git a/pkg/local/spdk/layout.go b/pkg/local/spdk/layout.go new file mode 100644 index 000000000..da8402358 --- /dev/null +++ b/pkg/local/spdk/layout.go @@ -0,0 +1,736 @@ +package spdk + +import ( + "bufio" + "debug/dwarf" + "debug/elf" + "os" + "strconv" + "strings" + + "github.com/pkg/errors" +) + +const ( + bsChannelMaxReqs = 512 + iobufMaxChannels = 64 + + // enum spdk_bs_cpl_type: NONE(0), BS_BASIC(1), BS_HANDLE(2), BLOB_BASIC(3), BLOB_HANDLE(4) + spdkBsCplTypeBlobBasic = 3 + spdkBsCplTypeBlobHandle = 4 + + spdkBdevIOTypeRead = 1 + spdkBdevIOTypeWrite = 2 + spdkBdevIOTypeUnmap = 3 + spdkBdevIOTypeFlush = 4 + spdkBdevIOTypeReset = 5 + spdkBdevIOTypeWriteZero = 7 +) + +type spdkSymbols struct { + gThreads uint64 + gIobuf uint64 + gLvolStores uint64 + blobAllocCopyClusterCpl uint64 + perBlobChannelSupported bool +} + +type spdkStructLayout struct { + dwarfLoaded bool + + // struct spdk_thread + threadID uint64 + threadIOChannels uint64 + threadTailqNext uint64 + threadName uint64 + threadNameSize int + + // struct spdk_io_channel + ioChanDev uint64 + ioChanNode uint64 // RB_ENTRY: left(+0), right(+8) + ioChanCtx uint64 // sizeof(struct spdk_io_channel) + + // struct io_device + ioDevPtr uint64 + ioDevName uint64 + ioDevNameSize int + + // struct iobuf / spdk_iobuf_opts / iobuf_node + iobufSmallPoolCount uint64 + iobufLargePoolCount uint64 + iobufSmallBufSize uint64 + iobufLargeBufSize uint64 + iobufNode uint64 + iobufNodeSmallPool uint64 + iobufNodeLargePool uint64 + + // struct rte_ring + rteRingMask uint64 + rteRingCapacity uint64 + rteRingProdTail uint64 + rteRingConsHead uint64 + + // struct iobuf_channel / spdk_iobuf_entry / spdk_iobuf_channel + iobufChanSmallQueue uint64 + iobufChanLargeQueue uint64 + iobufChanChannels uint64 + iobufEntryStailqNext uint64 + spdkIobufChanModule uint64 + spdkIobufChanParent uint64 + spdkIobufChanCache uint64 + poolCacheSmallCount uint64 + poolCacheSmallSize uint64 + poolCacheSmallRetry uint64 + poolCacheLargeOffset uint64 + poolCacheLargeCount uint64 + poolCacheLargeSize uint64 + poolCacheLargeRetry uint64 + + // struct spdk_bs_channel + bsChanReqMem uint64 + bsChanReqs uint64 + bsChanNeedClusterAlloc uint64 + bsChanQueuedIO uint64 + + // struct spdk_bs_request_set + bsReqSetSize uint64 + bsReqSetCplType uint64 + bsReqSetCplCbFn uint64 + bsReqSetCplCbArg uint64 + bsReqSetCplBlob uint64 + bsReqSetBackChannel uint64 + bsReqSetEsnapBlob uint64 + bsReqSetUserOpBlob uint64 + bsReqSetLinkNext uint64 + + // enum spdk_bs_cpl_type values + cplTypeBlobBasic uint32 + cplTypeBlobHandle uint32 + + // struct spdk_blob_copy_cluster_ctx + copyCtxBlob uint64 + copyCtxIOUnit uint64 + copyCtxNewCluster uint64 + + // struct spdk_blob + blobID uint64 + + // struct spdk_lvol_store & spdk_lvol + lvsLvols uint64 + lvsLinkNext uint64 + lvolBlob uint64 + lvolBlobID uint64 + lvolUUIDStr uint64 + lvolUUIDStrSize int + lvolName uint64 + lvolNameSize int + lvolBdev uint64 + lvolLinkNext uint64 + + // struct spdk_bdev_mgmt_channel / spdk_bdev_shared_resource / spdk_bdev_io / spdk_bdev + bdevMgmtSharedResources uint64 + bdevSharedIOOutstanding uint64 + bdevSharedNoMemIO uint64 + bdevSharedNoMemThresh uint64 + bdevSharedLinkNext uint64 + bdevIOBdev uint64 + bdevIOType uint64 + bdevIONumRetries uint64 + bdevIOLinkNext uint64 + bdevNamePtr uint64 +} + +func defaultSpdkStructLayout(perBlobSupported bool) *spdkStructLayout { + needClusterAllocOff := uint64(56) + queuedIOOff := uint64(72) + if perBlobSupported { + needClusterAllocOff = 96 + queuedIOOff = 112 + } + + return &spdkStructLayout{ + threadID: 176, + threadIOChannels: 208, + threadTailqNext: 216, + threadName: 232, + threadNameSize: 257, + + ioChanDev: 8, + ioChanNode: 24, + ioChanCtx: 96, + + ioDevPtr: 0, + ioDevName: 8, + ioDevNameSize: 257, + + iobufSmallPoolCount: 0, + iobufLargePoolCount: 8, + iobufSmallBufSize: 16, + iobufLargeBufSize: 20, + iobufNode: 72, + iobufNodeSmallPool: 0, + iobufNodeLargePool: 8, + + rteRingMask: 52, + rteRingCapacity: 56, + rteRingProdTail: 132, + rteRingConsHead: 256, + + iobufChanSmallQueue: 0, + iobufChanLargeQueue: 16, + iobufChanChannels: 32, + iobufEntryStailqNext: 16, + spdkIobufChanModule: 0, + spdkIobufChanParent: 8, + spdkIobufChanCache: 16, + poolCacheSmallCount: 24, + poolCacheSmallSize: 28, + poolCacheSmallRetry: 72, + poolCacheLargeOffset: 80, + poolCacheLargeCount: 104, + poolCacheLargeSize: 108, + poolCacheLargeRetry: 152, + + bsChanReqMem: 0, + bsChanReqs: 8, + bsChanNeedClusterAlloc: needClusterAllocOff, + bsChanQueuedIO: queuedIOOff, + + bsReqSetSize: 192, + bsReqSetCplType: 0, + bsReqSetCplCbFn: 8, + bsReqSetCplCbArg: 16, + bsReqSetCplBlob: 24, + bsReqSetBackChannel: 56, + bsReqSetEsnapBlob: 72, + bsReqSetUserOpBlob: 120, + bsReqSetLinkNext: 176, + + cplTypeBlobBasic: spdkBsCplTypeBlobBasic, + cplTypeBlobHandle: spdkBsCplTypeBlobHandle, + + copyCtxBlob: 0, + copyCtxIOUnit: 16, + copyCtxNewCluster: 24, + + blobID: 16, + + lvsLvols: 56, + lvsLinkNext: 112, + lvolBlob: 8, + lvolBlobID: 16, + lvolUUIDStr: 24, + lvolUUIDStrSize: 37, + lvolName: 82, + lvolNameSize: 256, + lvolBdev: 416, + lvolLinkNext: 440, + + bdevMgmtSharedResources: 200, + bdevSharedIOOutstanding: 8, + bdevSharedNoMemIO: 16, + bdevSharedNoMemThresh: 32, + bdevSharedLinkNext: 72, + bdevIOBdev: 0, + bdevIOType: 8, + bdevIONumRetries: 10, + bdevIOLinkNext: 1032, + bdevNamePtr: 8, + } +} + +func loadELFMetadata(exePath, mapsPath string) (*spdkSymbols, *spdkStructLayout, error) { + ef, err := elf.Open(exePath) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to open ELF binary %s", exePath) + } + defer func() { + _ = ef.Close() + }() + + var loadBias uint64 + if ef.Type == elf.ET_DYN && mapsPath != "" { + bias, biasErr := resolvePIELoadBias(ef, mapsPath) + if biasErr == nil { + loadBias = bias + } + } + + syms := &spdkSymbols{} + elfSyms, err := ef.Symbols() + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to read symbol table from %s", exePath) + } + for _, s := range elfSyms { + if s.Value == 0 { + continue + } + switch s.Name { + case "g_threads": + syms.gThreads = loadBias + s.Value + case "g_iobuf": + syms.gIobuf = loadBias + s.Value + case "g_lvol_stores": + syms.gLvolStores = loadBias + s.Value + case "blob_allocate_and_copy_cluster_cpl": + syms.blobAllocCopyClusterCpl = loadBias + s.Value + case "spdk_blob_alloc_io_channel": + syms.perBlobChannelSupported = true + } + } + if syms.gThreads == 0 { + return nil, nil, errors.Errorf("symbol g_threads not found in %s", exePath) + } + + layout := defaultSpdkStructLayout(syms.perBlobChannelSupported) + if dw, dwErr := ef.DWARF(); dwErr == nil && dw != nil { + applyDWARFLayout(dw, layout) + } + + return syms, layout, nil +} + +func resolvePIELoadBias(ef *elf.File, mapsPath string) (uint64, error) { + var minLoadVaddr uint64 + foundLoad := false + for _, prog := range ef.Progs { + if prog.Type == elf.PT_LOAD { + if !foundLoad || prog.Vaddr < minLoadVaddr { + minLoadVaddr = prog.Vaddr + foundLoad = true + } + } + } + if !foundLoad { + return 0, errors.New("no PT_LOAD segment in ELF") + } + + f, err := os.Open(mapsPath) + if err != nil { + return 0, err + } + defer func() { + _ = f.Close() + }() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + fields := strings.Fields(line) + if len(fields) < 6 { + continue + } + if !strings.HasSuffix(fields[5], "/spdk_tgt") && !strings.Contains(fields[5], "/spdk_tgt ") { + continue + } + offset, err := strconv.ParseUint(fields[2], 16, 64) + if err != nil || offset != 0 { + continue + } + addrParts := strings.SplitN(fields[0], "-", 2) + if len(addrParts) != 2 { + continue + } + startAddr, err := strconv.ParseUint(addrParts[0], 16, 64) + if err != nil { + continue + } + return startAddr - minLoadVaddr, nil + } + return 0, errors.New("spdk_tgt base mapping not found") +} + +type dwarfStructInfo struct { + size uint64 + fields map[string]uint64 +} + +var targetDWARFStructs = map[string]struct{}{ + "spdk_thread": {}, + "spdk_io_channel": {}, + "io_device": {}, + "iobuf": {}, + "spdk_iobuf_opts": {}, + "iobuf_node": {}, + "rte_ring": {}, + "rte_ring_headtail": {}, + "iobuf_channel": {}, + "iobuf_channel_node": {}, + "spdk_iobuf_entry": {}, + "spdk_iobuf_channel": {}, + "spdk_iobuf_node_cache": {}, + "spdk_iobuf_pool_cache": {}, + "spdk_iobuf_pool_stats": {}, + "spdk_bs_channel": {}, + "spdk_bs_request_set": {}, + "spdk_bs_user_op_args": {}, + "spdk_blob_copy_cluster_ctx": {}, + "spdk_blob": {}, + "spdk_lvol_store": {}, + "spdk_lvol": {}, + "spdk_bdev_mgmt_channel": {}, + "spdk_bdev_shared_resource": {}, + "spdk_bdev_io": {}, + "spdk_bdev": {}, +} + +func applyDWARFLayout(dw *dwarf.Data, layout *spdkStructLayout) { + found := make(map[string]*dwarfStructInfo, len(targetDWARFStructs)) + cplEnumLoaded := false + reader := dw.Reader() + + for { + entry, err := reader.Next() + if err != nil || entry == nil { + break + } + if entry.Tag == dwarf.TagCompileUnit || entry.Tag == dwarf.TagPartialUnit { + continue + } + if entry.Tag == dwarf.TagEnumerationType { + enumName, _ := entry.Val(dwarf.AttrName).(string) + if enumName == "spdk_bs_cpl_type" && entry.Children && !cplEnumLoaded { + readCplTypeEnum(reader, layout) + cplEnumLoaded = true + continue + } + if entry.Children { + reader.SkipChildren() + } + continue + } + if entry.Tag != dwarf.TagStructType { + if entry.Children { + reader.SkipChildren() + } + continue + } + + name, _ := entry.Val(dwarf.AttrName).(string) + if _, wanted := targetDWARFStructs[name]; !wanted || entry.Val(dwarf.AttrDeclaration) != nil || !entry.Children { + if entry.Children { + reader.SkipChildren() + } + continue + } + if _, already := found[name]; already { + reader.SkipChildren() + continue + } + + byteSize, _ := dwarfInt64(entry.Val(dwarf.AttrByteSize)) + info := &dwarfStructInfo{ + size: uint64(byteSize), + fields: make(map[string]uint64), + } + readStructMembers(dw, reader, 0, info.fields, 0) + found[name] = info + + if len(found) == len(targetDWARFStructs) && cplEnumLoaded { + break + } + } + + if len(found) == 0 { + return + } + layout.dwarfLoaded = true + + setField := func(structName, fieldName string, dst *uint64) { + if st, ok := found[structName]; ok { + if off, ok := st.fields[fieldName]; ok { + *dst = off + } + } + } + + setField("spdk_thread", "id", &layout.threadID) + setField("spdk_thread", "io_channels", &layout.threadIOChannels) + setField("spdk_thread", "tailq", &layout.threadTailqNext) + setField("spdk_thread", "name", &layout.threadName) + + if st, ok := found["spdk_io_channel"]; ok { + if st.size > 0 { + layout.ioChanCtx = st.size + } + setField("spdk_io_channel", "dev", &layout.ioChanDev) + setField("spdk_io_channel", "node", &layout.ioChanNode) + } + + setField("io_device", "io_device", &layout.ioDevPtr) + setField("io_device", "name", &layout.ioDevName) + + var iobufOptsOff uint64 + setField("iobuf", "opts", &iobufOptsOff) + setField("iobuf", "node", &layout.iobufNode) + if optsSt, ok := found["spdk_iobuf_opts"]; ok { + if off, ok := optsSt.fields["small_pool_count"]; ok { + layout.iobufSmallPoolCount = iobufOptsOff + off + } + if off, ok := optsSt.fields["large_pool_count"]; ok { + layout.iobufLargePoolCount = iobufOptsOff + off + } + if off, ok := optsSt.fields["small_bufsize"]; ok { + layout.iobufSmallBufSize = iobufOptsOff + off + } + if off, ok := optsSt.fields["large_bufsize"]; ok { + layout.iobufLargeBufSize = iobufOptsOff + off + } + } + setField("iobuf_node", "small_pool", &layout.iobufNodeSmallPool) + setField("iobuf_node", "large_pool", &layout.iobufNodeLargePool) + + if ringSt, ok := found["rte_ring"]; ok { + setField("rte_ring", "mask", &layout.rteRingMask) + setField("rte_ring", "capacity", &layout.rteRingCapacity) + tailInHT := uint64(4) + headInHT := uint64(0) + if htSt, ok := found["rte_ring_headtail"]; ok { + if off, ok := htSt.fields["tail"]; ok { + tailInHT = off + } + if off, ok := htSt.fields["head"]; ok { + headInHT = off + } + } + if prodOff, ok := ringSt.fields["prod"]; ok { + layout.rteRingProdTail = prodOff + tailInHT + } + if consOff, ok := ringSt.fields["cons"]; ok { + layout.rteRingConsHead = consOff + headInHT + } + } + + var chanNodeOff uint64 + setField("iobuf_channel", "node", &chanNodeOff) + setField("iobuf_channel", "channels", &layout.iobufChanChannels) + if cnSt, ok := found["iobuf_channel_node"]; ok { + if off, ok := cnSt.fields["small_queue"]; ok { + layout.iobufChanSmallQueue = chanNodeOff + off + } + if off, ok := cnSt.fields["large_queue"]; ok { + layout.iobufChanLargeQueue = chanNodeOff + off + } + } + setField("spdk_iobuf_entry", "stailq", &layout.iobufEntryStailqNext) + setField("spdk_iobuf_channel", "module", &layout.spdkIobufChanModule) + setField("spdk_iobuf_channel", "parent", &layout.spdkIobufChanParent) + setField("spdk_iobuf_channel", "cache", &layout.spdkIobufChanCache) + + if ncSt, ok := found["spdk_iobuf_node_cache"]; ok { + smallOff := ncSt.fields["small"] + largeOff, hasLarge := ncSt.fields["large"] + if hasLarge { + layout.poolCacheLargeOffset = largeOff + } + if pcSt, ok := found["spdk_iobuf_pool_cache"]; ok { + if off, ok := pcSt.fields["cache_count"]; ok { + layout.poolCacheSmallCount = smallOff + off + layout.poolCacheLargeCount = layout.poolCacheLargeOffset + off + } + if off, ok := pcSt.fields["cache_size"]; ok { + layout.poolCacheSmallSize = smallOff + off + layout.poolCacheLargeSize = layout.poolCacheLargeOffset + off + } + if statsOff, ok := pcSt.fields["stats"]; ok { + if psSt, ok := found["spdk_iobuf_pool_stats"]; ok { + if retryOff, ok := psSt.fields["retry"]; ok { + layout.poolCacheSmallRetry = smallOff + statsOff + retryOff + layout.poolCacheLargeRetry = layout.poolCacheLargeOffset + statsOff + retryOff + } + } + } + } + } + + setField("spdk_bs_channel", "req_mem", &layout.bsChanReqMem) + setField("spdk_bs_channel", "reqs", &layout.bsChanReqs) + setField("spdk_bs_channel", "need_cluster_alloc", &layout.bsChanNeedClusterAlloc) + setField("spdk_bs_channel", "queued_io", &layout.bsChanQueuedIO) + + if rsSt, ok := found["spdk_bs_request_set"]; ok { + if rsSt.size > 0 { + layout.bsReqSetSize = rsSt.size + } + setField("spdk_bs_request_set", "back_channel", &layout.bsReqSetBackChannel) + setField("spdk_bs_request_set", "esnap_blob", &layout.bsReqSetEsnapBlob) + setField("spdk_bs_request_set", "link", &layout.bsReqSetLinkNext) + if uOff, ok := rsSt.fields["u"]; ok { + if uoSt, ok := found["spdk_bs_user_op_args"]; ok { + if blobOff, ok := uoSt.fields["blob"]; ok { + layout.bsReqSetUserOpBlob = uOff + blobOff + } + } + } + } + + setField("spdk_blob_copy_cluster_ctx", "blob", &layout.copyCtxBlob) + setField("spdk_blob_copy_cluster_ctx", "io_unit", &layout.copyCtxIOUnit) + setField("spdk_blob_copy_cluster_ctx", "new_cluster", &layout.copyCtxNewCluster) + + setField("spdk_blob", "id", &layout.blobID) + + setField("spdk_lvol_store", "lvols", &layout.lvsLvols) + setField("spdk_lvol_store", "link", &layout.lvsLinkNext) + setField("spdk_lvol", "blob", &layout.lvolBlob) + setField("spdk_lvol", "blob_id", &layout.lvolBlobID) + setField("spdk_lvol", "uuid_str", &layout.lvolUUIDStr) + setField("spdk_lvol", "name", &layout.lvolName) + setField("spdk_lvol", "bdev", &layout.lvolBdev) + setField("spdk_lvol", "link", &layout.lvolLinkNext) + + setField("spdk_bdev_mgmt_channel", "shared_resources", &layout.bdevMgmtSharedResources) + setField("spdk_bdev_shared_resource", "io_outstanding", &layout.bdevSharedIOOutstanding) + setField("spdk_bdev_shared_resource", "nomem_io", &layout.bdevSharedNoMemIO) + setField("spdk_bdev_shared_resource", "nomem_threshold", &layout.bdevSharedNoMemThresh) + setField("spdk_bdev_shared_resource", "link", &layout.bdevSharedLinkNext) + + if bioSt, ok := found["spdk_bdev_io"]; ok { + setField("spdk_bdev_io", "bdev", &layout.bdevIOBdev) + setField("spdk_bdev_io", "type", &layout.bdevIOType) + setField("spdk_bdev_io", "num_retries", &layout.bdevIONumRetries) + if internalOff, ok := bioSt.fields["internal.link"]; ok { + layout.bdevIOLinkNext = internalOff + } + } + setField("spdk_bdev", "name", &layout.bdevNamePtr) +} + +func readCplTypeEnum(reader *dwarf.Reader, layout *spdkStructLayout) { + for { + child, err := reader.Next() + if err != nil || child == nil || child.Tag == 0 { + break + } + if child.Tag != dwarf.TagEnumerator { + if child.Children { + reader.SkipChildren() + } + continue + } + name, _ := child.Val(dwarf.AttrName).(string) + val, ok := dwarfInt64(child.Val(dwarf.AttrConstValue)) + if ok && val >= 0 { + switch name { + case "SPDK_BS_CPL_TYPE_BLOB_BASIC": + layout.cplTypeBlobBasic = uint32(val) + case "SPDK_BS_CPL_TYPE_BLOB_HANDLE": + layout.cplTypeBlobHandle = uint32(val) + } + } + if child.Children { + reader.SkipChildren() + } + } +} + +func readStructMembers(dw *dwarf.Data, reader *dwarf.Reader, baseOffset uint64, fields map[string]uint64, depth int) { + if depth > 4 { + return + } + for { + child, err := reader.Next() + if err != nil || child == nil || child.Tag == 0 { + break + } + if child.Tag != dwarf.TagMember { + if child.Children { + reader.SkipChildren() + } + continue + } + + loc, _ := dwarfMemberOffset(child.Val(dwarf.AttrDataMemberLoc)) + memberOff := baseOffset + loc + memberName, _ := child.Val(dwarf.AttrName).(string) + + if child.Children { + reader.SkipChildren() + } + + if memberName != "" { + if _, exists := fields[memberName]; !exists { + fields[memberName] = memberOff + } + // Flatten spdk_bdev_io.internal.link so we can read nomem_io's TAILQ link offset directly. + if memberName == "internal" { + if typeOff, ok := child.Val(dwarf.AttrType).(dwarf.Offset); ok { + subFields := make(map[string]uint64) + flattenTypeAtOffset(dw, typeOff, memberOff, subFields, depth+1) + for k, v := range subFields { + fields["internal."+k] = v + } + } + } + continue + } + + // Anonymous struct or union member: flatten its fields into the parent struct. + if typeOff, ok := child.Val(dwarf.AttrType).(dwarf.Offset); ok { + flattenTypeAtOffset(dw, typeOff, memberOff, fields, depth+1) + } + } +} + +func flattenTypeAtOffset(dw *dwarf.Data, typeOff dwarf.Offset, baseOffset uint64, fields map[string]uint64, depth int) { + if depth > 4 { + return + } + sub := dw.Reader() + sub.Seek(typeOff) + entry, err := sub.Next() + if err != nil || entry == nil { + return + } + for entry != nil && (entry.Tag == dwarf.TagTypedef || entry.Tag == dwarf.TagConstType || entry.Tag == dwarf.TagVolatileType) { + nextOff, ok := entry.Val(dwarf.AttrType).(dwarf.Offset) + if !ok { + return + } + sub.Seek(nextOff) + entry, err = sub.Next() + if err != nil { + return + } + } + if entry == nil || (entry.Tag != dwarf.TagStructType && entry.Tag != dwarf.TagUnionType) || !entry.Children { + return + } + readStructMembers(dw, sub, baseOffset, fields, depth) +} + +func dwarfInt64(v any) (int64, bool) { + switch val := v.(type) { + case int64: + return val, true + case uint64: + return int64(val), true + default: + return 0, false + } +} + +func dwarfMemberOffset(v any) (uint64, bool) { + switch val := v.(type) { + case int64: + return uint64(val), true + case uint64: + return val, true + case []byte: + // DWARF2/3 DW_OP_plus_uconst (0x23) followed by ULEB128 offset. + if len(val) >= 2 && val[0] == 0x23 { + var res uint64 + var shift uint + for _, b := range val[1:] { + res |= uint64(b&0x7f) << shift + if b&0x80 == 0 { + return res, true + } + shift += 7 + } + } + return 0, false + default: + return 0, false + } +} diff --git a/pkg/local/spdk/mem.go b/pkg/local/spdk/mem.go new file mode 100644 index 000000000..8c727d11c --- /dev/null +++ b/pkg/local/spdk/mem.go @@ -0,0 +1,674 @@ +package spdk + +import ( + "encoding/binary" + "fmt" + "io" + "sort" + + "github.com/longhorn/cli/pkg/types" +) + +type memReader struct { + r io.ReaderAt +} + +func (m *memReader) u64(addr uint64) (uint64, bool) { + if addr == 0 { + return 0, false + } + var buf [8]byte + if _, err := m.r.ReadAt(buf[:], int64(addr)); err != nil { + return 0, false + } + return binary.LittleEndian.Uint64(buf[:]), true +} + +func (m *memReader) u32(addr uint64) (uint32, bool) { + if addr == 0 { + return 0, false + } + var buf [4]byte + if _, err := m.r.ReadAt(buf[:], int64(addr)); err != nil { + return 0, false + } + return binary.LittleEndian.Uint32(buf[:]), true +} + +func (m *memReader) u16(addr uint64) (uint16, bool) { + if addr == 0 { + return 0, false + } + var buf [2]byte + if _, err := m.r.ReadAt(buf[:], int64(addr)); err != nil { + return 0, false + } + return binary.LittleEndian.Uint16(buf[:]), true +} + +func (m *memReader) u8(addr uint64) (uint8, bool) { + if addr == 0 { + return 0, false + } + var buf [1]byte + if _, err := m.r.ReadAt(buf[:], int64(addr)); err != nil { + return 0, false + } + return buf[0], true +} + +func (m *memReader) cstr(addr uint64, maxLen int) string { + if addr == 0 || maxLen <= 0 { + return "" + } + buf := make([]byte, maxLen) + n, err := m.r.ReadAt(buf, int64(addr)) + if n == 0 && err != nil { + return "" + } + buf = buf[:n] + for i, b := range buf { + if b == 0 { + return string(buf[:i]) + } + } + return string(buf) +} + +func (m *memReader) walkList(headPtrAddr, nextOffset uint64, maxItems int) []uint64 { + first, ok := m.u64(headPtrAddr) + if !ok || first == 0 { + return nil + } + res := make([]uint64, 0, 16) + seen := make(map[uint64]struct{}, 16) + cur := first + for cur != 0 && len(res) < maxItems { + if _, dup := seen[cur]; dup { + break + } + seen[cur] = struct{}{} + res = append(res, cur) + next, ok := m.u64(cur + nextOffset) + if !ok { + break + } + cur = next + } + return res +} + +func (m *memReader) walkRBTree(rootAddr, nodeOffset uint64, maxItems int) []uint64 { + if rootAddr == 0 { + return nil + } + res := make([]uint64, 0, 32) + seen := make(map[uint64]struct{}, 32) + stack := []uint64{rootAddr} + for len(stack) > 0 && len(res) < maxItems { + cur := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if cur == 0 { + continue + } + if _, dup := seen[cur]; dup { + continue + } + seen[cur] = struct{}{} + res = append(res, cur) + + if right, ok := m.u64(cur + nodeOffset + 8); ok && right != 0 { + stack = append(stack, right) + } + if left, ok := m.u64(cur + nodeOffset); ok && left != 0 { + stack = append(stack, left) + } + } + return res +} + +type lvolMeta struct { + name string + blobID uint64 +} + +type lvolIndex struct { + byBlobPtr map[uint64]lvolMeta + byBdevPtr map[uint64]string + byUUID map[string]string +} + +func inspectProcessMemory( + rawMem io.ReaderAt, + syms *spdkSymbols, + layout *spdkStructLayout, + rpcSnap *rpcSnapshot, + includeIdleChannels bool, + procInfo *types.SpdkProcessInfo, +) { + m := &memReader{r: rawMem} + + if syms.gIobuf != 0 { + procInfo.Iobuf = inspectGlobalIobuf(m, syms.gIobuf, layout, rpcSnap) + } else if rpcSnap != nil && len(rpcSnap.iobufStats) > 0 { + procInfo.Iobuf = &types.SpdkIobufInfo{ + ModuleStats: rpcSnap.iobufStats, + } + } + + lvolIdx := inspectLvolIndex(m, syms.gLvolStores, layout) + + threads := m.walkList(syms.gThreads, layout.threadTailqNext, 256) + for _, th := range threads { + tInfo := inspectThread(m, th, syms, layout, lvolIdx, rpcSnap, procInfo.Iobuf, includeIdleChannels) + procInfo.Threads = append(procInfo.Threads, tInfo) + + if tInfo.BdevNoMemPollerCount > 0 { + procInfo.Warnings = append(procInfo.Warnings, + fmt.Sprintf("thread %s has %d active bdev_no_mem_poller(s)", tInfo.Name, tInfo.BdevNoMemPollerCount)) + } + if tInfo.IobufChannel != nil && tInfo.IobufChannel.LargeWaiters > 0 { + procInfo.Errors = append(procInfo.Errors, + fmt.Sprintf("thread %s has %d request(s) blocked waiting for iobuf large_pool buffers (totalFreeLargeBuffers=%d)", + tInfo.Name, tInfo.IobufChannel.LargeWaiters, tInfo.IobufChannel.TotalFreeLargeBuffers)) + } + for _, bsCh := range tInfo.BlobstoreChannels { + target := bsCh.Device + if bsCh.LvolName != "" { + target = fmt.Sprintf("%s (%s)", bsCh.Device, bsCh.LvolName) + } + if bsCh.FreeReqs == 0 && bsCh.TotalReqs > 0 { + procInfo.Errors = append(procInfo.Errors, + fmt.Sprintf("thread %s %s channel %s exhausted spdk_bs_request_set pool (freeReqs=0/%d, needClusterAlloc=%d)", + tInfo.Name, target, bsCh.Address, bsCh.TotalReqs, bsCh.NeedClusterAlloc)) + } else if bsCh.FreeReqs > 0 && bsCh.NeedClusterAlloc > 0 && len(bsCh.ActiveRequests) > 0 && + (bsCh.NeedClusterAlloc >= bsCh.TotalReqs/2 || len(tInfo.BdevNoMemResources) > 0) { + procInfo.Errors = append(procInfo.Errors, + fmt.Sprintf("thread %s %s channel %s cluster allocation blocked (needClusterAlloc=%d, freeReqs=%d/%d, activeRequests=%d)", + tInfo.Name, target, bsCh.Address, bsCh.NeedClusterAlloc, bsCh.FreeReqs, bsCh.TotalReqs, len(bsCh.ActiveRequests))) + } + } + for _, nm := range tInfo.BdevNoMemResources { + if nm.NoMemIOCount > 0 { + procInfo.Errors = append(procInfo.Errors, + fmt.Sprintf("thread %s has %d bdev I/O(s) queued in nomem_io (ioOutstanding=%d)", + tInfo.Name, nm.NoMemIOCount, nm.IOOutstanding)) + } + } + } +} + +func inspectGlobalIobuf(m *memReader, gIobufAddr uint64, layout *spdkStructLayout, rpcSnap *rpcSnapshot) *types.SpdkIobufInfo { + smallCnt, _ := m.u64(gIobufAddr + layout.iobufSmallPoolCount) + largeCnt, _ := m.u64(gIobufAddr + layout.iobufLargePoolCount) + smallSz, _ := m.u32(gIobufAddr + layout.iobufSmallBufSize) + largeSz, _ := m.u32(gIobufAddr + layout.iobufLargeBufSize) + + nodeBase := gIobufAddr + layout.iobufNode + smallRing, _ := m.u64(nodeBase + layout.iobufNodeSmallPool) + largeRing, _ := m.u64(nodeBase + layout.iobufNodeLargePool) + + info := &types.SpdkIobufInfo{ + SmallPoolCount: smallCnt, + SmallBufSize: smallSz, + SmallRingFree: readRteRingFree(m, smallRing, layout), + LargePoolCount: largeCnt, + LargeBufSize: largeSz, + LargeRingFree: readRteRingFree(m, largeRing, layout), + } + if rpcSnap != nil { + info.ModuleStats = rpcSnap.iobufStats + } + return info +} + +func readRteRingFree(m *memReader, ringAddr uint64, layout *spdkStructLayout) uint32 { + if ringAddr == 0 { + return 0 + } + mask, ok1 := m.u32(ringAddr + layout.rteRingMask) + capVal, _ := m.u32(ringAddr + layout.rteRingCapacity) + prodTail, ok2 := m.u32(ringAddr + layout.rteRingProdTail) + consHead, ok3 := m.u32(ringAddr + layout.rteRingConsHead) + if !ok1 || !ok2 || !ok3 { + return 0 + } + count := (prodTail - consHead) & mask + if capVal > 0 && count > capVal { + count = capVal + } + return count +} + +func inspectLvolIndex(m *memReader, gLvolStoresAddr uint64, layout *spdkStructLayout) *lvolIndex { + idx := &lvolIndex{ + byBlobPtr: make(map[uint64]lvolMeta), + byBdevPtr: make(map[uint64]string), + byUUID: make(map[string]string), + } + if gLvolStoresAddr == 0 { + return idx + } + for _, lvs := range m.walkList(gLvolStoresAddr, layout.lvsLinkNext, 256) { + for _, lvol := range m.walkList(lvs+layout.lvsLvols, layout.lvolLinkNext, 8192) { + name := m.cstr(lvol+layout.lvolName, layout.lvolNameSize) + if blobPtr, ok := m.u64(lvol + layout.lvolBlob); ok && blobPtr != 0 { + blobID, _ := m.u64(lvol + layout.lvolBlobID) + idx.byBlobPtr[blobPtr] = lvolMeta{ + name: name, + blobID: blobID, + } + } + if name != "" { + if layout.lvolUUIDStrSize > 0 { + if uuidStr := m.cstr(lvol+layout.lvolUUIDStr, layout.lvolUUIDStrSize); uuidStr != "" { + idx.byUUID[uuidStr] = name + } + } + if bdevPtr, ok := m.u64(lvol + layout.lvolBdev); ok && bdevPtr != 0 { + idx.byBdevPtr[bdevPtr] = name + } + } + } + } + return idx +} + +func inspectThread( + m *memReader, + threadAddr uint64, + syms *spdkSymbols, + layout *spdkStructLayout, + lvolIdx *lvolIndex, + rpcSnap *rpcSnapshot, + globalIobuf *types.SpdkIobufInfo, + includeIdleChannels bool, +) *types.SpdkThreadInfo { + tName := m.cstr(threadAddr+layout.threadName, layout.threadNameSize) + tID, _ := m.u64(threadAddr + layout.threadID) + + tInfo := &types.SpdkThreadInfo{ + Name: tName, + ID: tID, + } + if rpcSnap != nil { + if ps, ok := rpcSnap.threadPollers[tName]; ok { + tInfo.BdevNoMemPollerCount = ps.bdevNoMemPollerCount + tInfo.ActivePollers = ps.activePollers + tInfo.TimedPollers = ps.timedPollers + if len(ps.timedPollers) > 0 { + counts := make(map[string]int) + for _, tp := range ps.timedPollers { + counts[tp]++ + } + tInfo.TimedPollerCounts = counts + } + } + } + + rbRoot, _ := m.u64(threadAddr + layout.threadIOChannels) + channels := m.walkRBTree(rbRoot, layout.ioChanNode, 16384) + for _, ch := range channels { + dev, ok := m.u64(ch + layout.ioChanDev) + if !ok || dev == 0 { + continue + } + devName := m.cstr(dev+layout.ioDevName, layout.ioDevNameSize) + ctx := ch + layout.ioChanCtx + + switch devName { + case "iobuf": + tInfo.IobufChannel = inspectThreadIobufChannel(m, ch, ctx, layout, globalIobuf) + case "blobstore", "blob": + bsInfo := inspectBsChannel(m, ch, dev, ctx, devName, syms, layout, lvolIdx, rpcSnap) + if includeIdleChannels || devName == "blobstore" || + bsInfo.FreeReqs < bsInfo.TotalReqs || bsInfo.NeedClusterAlloc > 0 || + bsInfo.QueuedIO > 0 || len(bsInfo.ActiveRequests) > 0 { + tInfo.BlobstoreChannels = append(tInfo.BlobstoreChannels, bsInfo) + } + case "bdev_mgr": + tInfo.BdevNoMemResources = append(tInfo.BdevNoMemResources, inspectBdevMgmtChannel(m, ctx, layout, lvolIdx)...) + } + } + + sort.Slice(tInfo.BlobstoreChannels, func(i, j int) bool { + if tInfo.BlobstoreChannels[i].Device != tInfo.BlobstoreChannels[j].Device { + return tInfo.BlobstoreChannels[i].Device == "blobstore" + } + if tInfo.BlobstoreChannels[i].NeedClusterAlloc != tInfo.BlobstoreChannels[j].NeedClusterAlloc { + return tInfo.BlobstoreChannels[i].NeedClusterAlloc > tInfo.BlobstoreChannels[j].NeedClusterAlloc + } + return tInfo.BlobstoreChannels[i].Address < tInfo.BlobstoreChannels[j].Address + }) + + return tInfo +} + +func inspectThreadIobufChannel( + m *memReader, + ioChanAddr, ctxAddr uint64, + layout *spdkStructLayout, + globalIobuf *types.SpdkIobufInfo, +) *types.SpdkThreadIobufInfo { + smallWaiters := len(m.walkList(ctxAddr+layout.iobufChanSmallQueue, layout.iobufEntryStailqNext, 16384)) + largeWaiters := len(m.walkList(ctxAddr+layout.iobufChanLargeQueue, layout.iobufEntryStailqNext, 16384)) + + info := &types.SpdkThreadIobufInfo{ + SmallWaiters: smallWaiters, + LargeWaiters: largeWaiters, + } + + var nvmfLargeCache uint64 + hasNvmfTCP := false + for idx := 0; idx < iobufMaxChannels; idx++ { + ich, ok := m.u64(ctxAddr + layout.iobufChanChannels + uint64(idx)*8) + if !ok || ich == 0 { + continue + } + parent, ok := m.u64(ich + layout.spdkIobufChanParent) + if !ok || parent != ioChanAddr { + continue + } + + modName := "" + if modPtr, ok := m.u64(ich + layout.spdkIobufChanModule); ok && modPtr != 0 { + if namePtr, ok := m.u64(modPtr); ok && namePtr != 0 { + modName = m.cstr(namePtr, 64) + } + } + if modName == "" { + modName = fmt.Sprintf("module_%d", idx) + } + + cacheBase := ich + layout.spdkIobufChanCache + sCount, _ := m.u32(cacheBase + layout.poolCacheSmallCount) + sSize, _ := m.u32(cacheBase + layout.poolCacheSmallSize) + sRetry, _ := m.u64(cacheBase + layout.poolCacheSmallRetry) + lCount, _ := m.u32(cacheBase + layout.poolCacheLargeCount) + lSize, _ := m.u32(cacheBase + layout.poolCacheLargeSize) + lRetry, _ := m.u64(cacheBase + layout.poolCacheLargeRetry) + + if modName == "nvmf_TCP" { + hasNvmfTCP = true + nvmfLargeCache = uint64(lCount) + } + + info.Modules = append(info.Modules, &types.SpdkThreadIobufModuleCache{ + Module: modName, + SmallCacheCount: sCount, + SmallCacheSize: sSize, + SmallRetry: sRetry, + LargeCacheCount: lCount, + LargeCacheSize: lSize, + LargeRetry: lRetry, + }) + } + + if hasNvmfTCP && globalIobuf != nil { + info.TotalFreeLargeBuffers = uint64(globalIobuf.LargeRingFree) + nvmfLargeCache + } + + return info +} + +func inspectBsChannel( + m *memReader, + ioChanAddr, ioDevAddr, ctxAddr uint64, + devName string, + syms *spdkSymbols, + layout *spdkStructLayout, + lvolIdx *lvolIndex, + rpcSnap *rpcSnapshot, +) *types.SpdkBsChannelInfo { + var blobMap map[uint64]lvolMeta + if lvolIdx != nil { + blobMap = lvolIdx.byBlobPtr + } + freeList := m.walkList(ctxAddr+layout.bsChanReqs, layout.bsReqSetLinkNext, bsChannelMaxReqs*2) + needList := m.walkList(ctxAddr+layout.bsChanNeedClusterAlloc, layout.bsReqSetLinkNext, bsChannelMaxReqs*2) + queuedList := m.walkList(ctxAddr+layout.bsChanQueuedIO, layout.bsReqSetLinkNext, bsChannelMaxReqs*2) + + freeSet := make(map[uint64]struct{}, len(freeList)) + for _, r := range freeList { + freeSet[r] = struct{}{} + } + needSet := make(map[uint64]struct{}, len(needList)) + for _, r := range needList { + needSet[r] = struct{}{} + } + queuedSet := make(map[uint64]struct{}, len(queuedList)) + for _, r := range queuedList { + queuedSet[r] = struct{}{} + } + + info := &types.SpdkBsChannelInfo{ + Device: devName, + Address: fmt.Sprintf("0x%x", ioChanAddr), + TotalReqs: bsChannelMaxReqs, + FreeReqs: len(freeSet), + NeedClusterAlloc: len(needSet), + QueuedIO: len(queuedSet), + } + + if devName == "blob" { + if blobPtr, ok := m.u64(ioDevAddr + layout.ioDevPtr); ok && blobPtr != 0 { + info.LvolName, info.BlobID = resolveBlobIdentity(m, blobPtr, layout, blobMap) + } + } + + if len(needList) > 0 { + byLvol := make(map[string]int) + for _, opAddr := range needList { + blobPtr, _ := m.u64(opAddr + layout.bsReqSetUserOpBlob) + lvolName, blobID := resolveBlobIdentity(m, blobPtr, layout, blobMap) + key := lvolName + if key == "" { + key = blobID + } + if key == "" { + key = "unknown" + } + byLvol[key]++ + } + info.NeedClusterAllocByLvol = byLvol + } + + reqMem, ok := m.u64(ctxAddr + layout.bsChanReqMem) + if !ok || reqMem == 0 { + return info + } + + var ctrlrAddrs map[string]string + if rpcSnap != nil { + ctrlrAddrs = rpcSnap.nvmeCtrlrAddrs + } + + for idx := 0; idx < bsChannelMaxReqs; idx++ { + r := reqMem + uint64(idx)*layout.bsReqSetSize + if _, isFree := freeSet[r]; isFree { + continue + } + if _, isNeed := needSet[r]; isNeed { + continue + } + if _, isQueued := queuedSet[r]; isQueued { + continue + } + + active := inspectActiveBsRequest(m, idx, r, syms, layout, blobMap, ctrlrAddrs) + info.ActiveRequests = append(info.ActiveRequests, active) + } + + return info +} + +func inspectActiveBsRequest( + m *memReader, + idx int, + reqAddr uint64, + syms *spdkSymbols, + layout *spdkStructLayout, + blobMap map[uint64]lvolMeta, + ctrlrAddrs map[string]string, +) *types.SpdkBsActiveRequest { + cplType, _ := m.u32(reqAddr + layout.bsReqSetCplType) + cbFn, _ := m.u64(reqAddr + layout.bsReqSetCplCbFn) + cbArg, _ := m.u64(reqAddr + layout.bsReqSetCplCbArg) + backCh, _ := m.u64(reqAddr + layout.bsReqSetBackChannel) + + backingDev := "" + if backCh != 0 { + if bDev, ok := m.u64(backCh + layout.ioChanDev); ok && bDev != 0 { + backingDev = m.cstr(bDev+layout.ioDevName, layout.ioDevNameSize) + } + } + + active := &types.SpdkBsActiveRequest{ + Index: idx, + OpType: "sequence", + BackingDevice: backingDev, + RemoteTargetAddress: resolveBackingDeviceTarget(backingDev, ctrlrAddrs), + } + + expectedBlobBasic := layout.cplTypeBlobBasic + if expectedBlobBasic == 0 { + expectedBlobBasic = spdkBsCplTypeBlobBasic + } + expectedBlobHandle := layout.cplTypeBlobHandle + if expectedBlobHandle == 0 { + expectedBlobHandle = spdkBsCplTypeBlobHandle + } + + if cplType == expectedBlobBasic && cbArg != 0 && + (syms.blobAllocCopyClusterCpl != 0 && cbFn == syms.blobAllocCopyClusterCpl) { + active.OpType = "cluster_cow" + if backingDev != "" && backingDev != "blob" { + active.OpType = "esnap_cow" + } + blobPtr, _ := m.u64(cbArg + layout.copyCtxBlob) + active.LvolName, active.BlobID = resolveBlobIdentity(m, blobPtr, layout, blobMap) + active.IOUnit, _ = m.u64(cbArg + layout.copyCtxIOUnit) + active.NewCluster, _ = m.u64(cbArg + layout.copyCtxNewCluster) + return active + } + + var blobPtr uint64 + if esnapBlob, _ := m.u64(reqAddr + layout.bsReqSetEsnapBlob); esnapBlob != 0 { + blobPtr = esnapBlob + } else if cplType == expectedBlobHandle { + blobPtr, _ = m.u64(reqAddr + layout.bsReqSetCplBlob) + } else { + blobPtr, _ = m.u64(reqAddr + layout.bsReqSetUserOpBlob) + } + if blobPtr != 0 { + active.LvolName, active.BlobID = resolveBlobIdentity(m, blobPtr, layout, blobMap) + } + + return active +} + +func resolveBlobIdentity(m *memReader, blobPtr uint64, layout *spdkStructLayout, blobMap map[uint64]lvolMeta) (string, string) { + if blobPtr == 0 { + return "", "" + } + if meta, ok := blobMap[blobPtr]; ok { + return meta.name, fmt.Sprintf("0x%x", meta.blobID) + } + if id, ok := m.u64(blobPtr + layout.blobID); ok && id != 0 { + return "", fmt.Sprintf("0x%x", id) + } + return "", "" +} + +func inspectBdevMgmtChannel(m *memReader, ctxAddr uint64, layout *spdkStructLayout, lvolIdx *lvolIndex) []*types.SpdkBdevNoMemInfo { + var out []*types.SpdkBdevNoMemInfo + sharedResources := m.walkList(ctxAddr+layout.bdevMgmtSharedResources, layout.bdevSharedLinkNext, 4096) + for _, sr := range sharedResources { + nomemList := m.walkList(sr+layout.bdevSharedNoMemIO, layout.bdevIOLinkNext, 16384) + if len(nomemList) == 0 { + continue + } + ioOut, _ := m.u64(sr + layout.bdevSharedIOOutstanding) + thresh, _ := m.u64(sr + layout.bdevSharedNoMemThresh) + + type bdevKey struct { + name string + ioType string + } + grouped := make(map[bdevKey]*types.SpdkBdevNoMemEntry) + for _, bio := range nomemList { + bdevName := "unknown" + lvolName := "" + bdevPtr, hasBdevPtr := m.u64(bio + layout.bdevIOBdev) + if hasBdevPtr && bdevPtr != 0 { + if namePtr, ok := m.u64(bdevPtr + layout.bdevNamePtr); ok && namePtr != 0 { + if s := m.cstr(namePtr, 256); s != "" { + bdevName = s + } + } + if lvolIdx != nil { + lvolName = lvolIdx.byBdevPtr[bdevPtr] + } + } + if lvolName == "" && lvolIdx != nil && bdevName != "unknown" { + lvolName = lvolIdx.byUUID[bdevName] + } + rawType, _ := m.u8(bio + layout.bdevIOType) + ioType := formatBdevIOType(rawType) + retries, _ := m.u16(bio + layout.bdevIONumRetries) + + k := bdevKey{name: bdevName, ioType: ioType} + entry, exists := grouped[k] + if !exists { + entry = &types.SpdkBdevNoMemEntry{ + BdevName: bdevName, + LvolName: lvolName, + IOType: ioType, + } + grouped[k] = entry + } + entry.Count++ + if retries > entry.MaxRetries { + entry.MaxRetries = retries + } + } + + entries := make([]*types.SpdkBdevNoMemEntry, 0, len(grouped)) + for _, e := range grouped { + entries = append(entries, e) + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].Count != entries[j].Count { + return entries[i].Count > entries[j].Count + } + return entries[i].BdevName < entries[j].BdevName + }) + + out = append(out, &types.SpdkBdevNoMemInfo{ + IOOutstanding: ioOut, + NoMemThreshold: thresh, + NoMemIOCount: len(nomemList), + ByBdev: entries, + }) + } + return out +} + +func formatBdevIOType(t uint8) string { + switch t { + case spdkBdevIOTypeRead: + return "read" + case spdkBdevIOTypeWrite: + return "write" + case spdkBdevIOTypeUnmap: + return "unmap" + case spdkBdevIOTypeFlush: + return "flush" + case spdkBdevIOTypeReset: + return "reset" + case spdkBdevIOTypeWriteZero: + return "write_zeros" + default: + return fmt.Sprintf("type_%d", t) + } +} diff --git a/pkg/local/spdk/mem_test.go b/pkg/local/spdk/mem_test.go new file mode 100644 index 000000000..31d37cd7c --- /dev/null +++ b/pkg/local/spdk/mem_test.go @@ -0,0 +1,471 @@ +package spdk + +import ( + "encoding/binary" + "encoding/json" + "io" + "net" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/longhorn/cli/pkg/types" +) + +type sparseMem struct { + pages map[uint64][]byte +} + +const pageSize = 4096 + +func newSparseMem() *sparseMem { + return &sparseMem{pages: make(map[uint64][]byte)} +} + +func (s *sparseMem) writeBytes(addr uint64, data []byte) { + for i, b := range data { + cur := addr + uint64(i) + pg := cur &^ (pageSize - 1) + off := cur & (pageSize - 1) + buf, ok := s.pages[pg] + if !ok { + buf = make([]byte, pageSize) + s.pages[pg] = buf + } + buf[off] = b + } +} + +func (s *sparseMem) putU64(addr, val uint64) { + var b [8]byte + binary.LittleEndian.PutUint64(b[:], val) + s.writeBytes(addr, b[:]) +} + +func (s *sparseMem) putU32(addr uint64, val uint32) { + var b [4]byte + binary.LittleEndian.PutUint32(b[:], val) + s.writeBytes(addr, b[:]) +} + +func (s *sparseMem) putU16(addr uint64, val uint16) { + var b [2]byte + binary.LittleEndian.PutUint16(b[:], val) + s.writeBytes(addr, b[:]) +} + +func (s *sparseMem) putU8(addr uint64, val uint8) { + s.writeBytes(addr, []byte{val}) +} + +func (s *sparseMem) putCString(addr uint64, str string) { + b := append([]byte(str), 0) + s.writeBytes(addr, b) +} + +func (s *sparseMem) ReadAt(p []byte, off int64) (int, error) { + if off < 0 { + return 0, io.EOF + } + uoff := uint64(off) + for i := range p { + cur := uoff + uint64(i) + pg := cur &^ (pageSize - 1) + poff := cur & (pageSize - 1) + buf, ok := s.pages[pg] + if !ok { + if i == 0 { + return 0, io.ErrUnexpectedEOF + } + return i, io.ErrUnexpectedEOF + } + p[i] = buf[poff] + } + return len(p), nil +} + +func TestInspectProcessMemory_SharedBsChannelDeadlock(t *testing.T) { + mem := newSparseMem() + layout := defaultSpdkStructLayout(false) + syms := &spdkSymbols{ + gThreads: 0x500000, + gIobuf: 0x510000, + gLvolStores: 0x520000, + blobAllocCopyClusterCpl: 0x412340, + perBlobChannelSupported: false, + } + + // 1. Global iobuf + DPDK rings + smallRing := uint64(0x200000100000) + largeRing := uint64(0x200000200000) + mem.putU64(syms.gIobuf+layout.iobufSmallPoolCount, 8192) + mem.putU64(syms.gIobuf+layout.iobufLargePoolCount, 6144) + mem.putU32(syms.gIobuf+layout.iobufSmallBufSize, 8192) + mem.putU32(syms.gIobuf+layout.iobufLargeBufSize, 135168) + mem.putU64(syms.gIobuf+layout.iobufNode+layout.iobufNodeSmallPool, smallRing) + mem.putU64(syms.gIobuf+layout.iobufNode+layout.iobufNodeLargePool, largeRing) + + mem.putU32(smallRing+layout.rteRingMask, 16383) + mem.putU32(smallRing+layout.rteRingCapacity, 8192) + mem.putU32(smallRing+layout.rteRingProdTail, 8192) + mem.putU32(smallRing+layout.rteRingConsHead, 0) + + // Test uint32 wraparound on largeRing: prodTail=500, consHead=0xfffffe0c -> diff = 1000 + mem.putU32(largeRing+layout.rteRingMask, 8191) + mem.putU32(largeRing+layout.rteRingCapacity, 6144) + mem.putU32(largeRing+layout.rteRingProdTail, 500) + mem.putU32(largeRing+layout.rteRingConsHead, ^uint32(499)) + + // 2. g_lvol_stores -> rebuild replica + canary replica + lvsAddr := uint64(0x600000) + lvolRebuild := uint64(0x601000) + lvolCanary := uint64(0x602000) + lvolPeerSnap := uint64(0x602800) + blobRebuild := uint64(0x603000) + blobCanary := uint64(0x604000) + blobPeerSnap := uint64(0x605000) + bdevPtr := uint64(0x901000) + + assert.Equal(t, 3, spdkBsCplTypeBlobBasic) + + mem.putU64(syms.gLvolStores, lvsAddr) + mem.putU64(lvsAddr+layout.lvsLvols, lvolRebuild) + mem.putU64(lvolRebuild+layout.lvolBlob, blobRebuild) + mem.putU64(lvolRebuild+layout.lvolBlobID, 0x100000011) + mem.putCString(lvolRebuild+layout.lvolName, "pvc-rebuild-r-49a7a136") + mem.putU64(lvolRebuild+layout.lvolLinkNext, lvolCanary) + + mem.putU64(lvolCanary+layout.lvolBlob, blobCanary) + mem.putU64(lvolCanary+layout.lvolBlobID, 0x100000022) + mem.putCString(lvolCanary+layout.lvolName, "pvc-canary-r-99999999") + mem.putU64(lvolCanary+layout.lvolLinkNext, lvolPeerSnap) + + mem.putU64(lvolPeerSnap+layout.lvolBlob, blobPeerSnap) + mem.putU64(lvolPeerSnap+layout.lvolBlobID, 0x100000033) + mem.putCString(lvolPeerSnap+layout.lvolUUIDStr, "1af7ab1b-ffe1-4687-95f6-2719672fcda0") + mem.putCString(lvolPeerSnap+layout.lvolName, "pvc-peer-snap-rebuild") + mem.putU64(lvolPeerSnap+layout.lvolBdev, bdevPtr) + mem.putU64(lvolPeerSnap+layout.lvolLinkNext, 0) + + // 3. Thread nvmf_tgt_poll_group_000 with shared blobstore channel + bdev_mgr channel + threadAddr := uint64(0x700000) + mem.putU64(syms.gThreads, threadAddr) + mem.putU64(threadAddr+layout.threadID, 2) + mem.putCString(threadAddr+layout.threadName, "nvmf_tgt_poll_group_000") + + bsChanAddr := uint64(0x7f77442cdfb0) + bsDevAddr := uint64(0x710000) + bdevMgrChanAddr := uint64(0x7f7744300000) + bdevMgrDevAddr := uint64(0x711000) + + mem.putU64(threadAddr+layout.threadIOChannels, bsChanAddr) + mem.putU64(bsChanAddr+layout.ioChanDev, bsDevAddr) + mem.putCString(bsDevAddr+layout.ioDevName, "blobstore") + mem.putU64(bsChanAddr+layout.ioChanNode+8, bdevMgrChanAddr) + + // 512 request sets: idx 7 is active esnap_cow, remaining 511 are queued in need_cluster_alloc (freeReqs = 0) + reqMem := uint64(0x800000) + bsCtx := bsChanAddr + layout.ioChanCtx + mem.putU64(bsCtx+layout.bsChanReqMem, reqMem) + mem.putU64(bsCtx+layout.bsChanReqs, 0) + + var prevNeed uint64 + for idx := 0; idx < bsChannelMaxReqs; idx++ { + r := reqMem + uint64(idx)*layout.bsReqSetSize + if idx == 7 { + copyCtx := uint64(0x880000) + backChan := uint64(0x890000) + backDev := uint64(0x891000) + + mem.putU32(r+layout.bsReqSetCplType, spdkBsCplTypeBlobBasic) + mem.putU64(r+layout.bsReqSetCplCbFn, syms.blobAllocCopyClusterCpl) + mem.putU64(r+layout.bsReqSetCplCbArg, copyCtx) + mem.putU64(r+layout.bsReqSetBackChannel, backChan) + + mem.putU64(copyCtx+layout.copyCtxBlob, blobRebuild) + mem.putU64(copyCtx+layout.copyCtxIOUnit, 4636672) + mem.putU64(copyCtx+layout.copyCtxNewCluster, 102) + + mem.putU64(backChan+layout.ioChanDev, backDev) + mem.putCString(backDev+layout.ioDevName, "bdev_pvc-rebuild-r-b5b5cfdc-snap-rebuild-464c20a6n1") + continue + } + + targetBlob := blobRebuild + if idx >= 400 { + targetBlob = blobCanary + } + mem.putU64(r+layout.bsReqSetUserOpBlob, targetBlob) + if prevNeed == 0 { + mem.putU64(bsCtx+layout.bsChanNeedClusterAlloc, r) + } else { + mem.putU64(prevNeed+layout.bsReqSetLinkNext, r) + } + prevNeed = r + } + mem.putU64(prevNeed+layout.bsReqSetLinkNext, 0) + + // 4. bdev_mgr channel with 2 queued nomem_io snapshot reads + mem.putU64(bdevMgrChanAddr+layout.ioChanDev, bdevMgrDevAddr) + mem.putCString(bdevMgrDevAddr+layout.ioDevName, "bdev_mgr") + sharedRes := uint64(0x900000) + bdevNameStr := uint64(0x901500) + bio1 := uint64(0x910000) + bio2 := uint64(0x911000) + + mem.putU64(bdevMgrChanAddr+layout.ioChanCtx+layout.bdevMgmtSharedResources, sharedRes) + mem.putU64(sharedRes+layout.bdevSharedIOOutstanding, 512) + mem.putU64(sharedRes+layout.bdevSharedNoMemThresh, 64) + mem.putU64(sharedRes+layout.bdevSharedNoMemIO, bio1) + + mem.putCString(bdevNameStr, "1af7ab1b-ffe1-4687-95f6-2719672fcda0") + mem.putU64(bdevPtr+layout.bdevNamePtr, bdevNameStr) + + mem.putU64(bio1+layout.bdevIOBdev, bdevPtr) + mem.putU8(bio1+layout.bdevIOType, spdkBdevIOTypeRead) + mem.putU16(bio1+layout.bdevIONumRetries, 36254) + mem.putU64(bio1+layout.bdevIOLinkNext, bio2) + + mem.putU64(bio2+layout.bdevIOBdev, bdevPtr) + mem.putU8(bio2+layout.bdevIOType, spdkBdevIOTypeRead) + mem.putU16(bio2+layout.bdevIONumRetries, 36250) + mem.putU64(bio2+layout.bdevIOLinkNext, 0) + + rpcSnap := &rpcSnapshot{ + threadPollers: map[string]*threadPollerSnapshot{ + "nvmf_tgt_poll_group_000": { + bdevNoMemPollerCount: 1, + timedPollers: []string{"bdev_no_mem_poller", "nvmf_ctrlr_keep_alive_poll", "nvmf_ctrlr_keep_alive_poll"}, + }, + }, + nvmeCtrlrAddrs: map[string]string{ + "pvc-rebuild-r-b5b5cfdc-snap-rebuild-464c20a6": "10.42.2.10:20015", + }, + } + + procInfo := &types.SpdkProcessInfo{PID: 23} + inspectProcessMemory(mem, syms, layout, rpcSnap, false, procInfo) + + require.NotNil(t, procInfo.Iobuf) + assert.Equal(t, uint32(1000), procInfo.Iobuf.LargeRingFree) + assert.Equal(t, uint32(8192), procInfo.Iobuf.SmallRingFree) + + require.Len(t, procInfo.Threads, 1) + th := procInfo.Threads[0] + assert.Equal(t, 1, th.BdevNoMemPollerCount) + assert.Len(t, th.TimedPollers, 3) + assert.Equal(t, 2, th.TimedPollerCounts["nvmf_ctrlr_keep_alive_poll"]) + assert.Equal(t, 1, th.TimedPollerCounts["bdev_no_mem_poller"]) + require.Len(t, th.BlobstoreChannels, 1) + + bsCh := th.BlobstoreChannels[0] + assert.Equal(t, 0, bsCh.FreeReqs) + assert.Equal(t, 511, bsCh.NeedClusterAlloc) + assert.Equal(t, 399, bsCh.NeedClusterAllocByLvol["pvc-rebuild-r-49a7a136"]) + assert.Equal(t, 112, bsCh.NeedClusterAllocByLvol["pvc-canary-r-99999999"]) + + require.Len(t, bsCh.ActiveRequests, 1) + act := bsCh.ActiveRequests[0] + assert.Equal(t, 7, act.Index) + assert.Equal(t, "esnap_cow", act.OpType) + assert.Equal(t, "pvc-rebuild-r-49a7a136", act.LvolName) + assert.Equal(t, "0x100000011", act.BlobID) + assert.Equal(t, uint64(4636672), act.IOUnit) + assert.Equal(t, "10.42.2.10:20015", act.RemoteTargetAddress) + + require.Len(t, th.BdevNoMemResources, 1) + assert.Equal(t, 2, th.BdevNoMemResources[0].NoMemIOCount) + require.Len(t, th.BdevNoMemResources[0].ByBdev, 1) + assert.Equal(t, "1af7ab1b-ffe1-4687-95f6-2719672fcda0", th.BdevNoMemResources[0].ByBdev[0].BdevName) + assert.Equal(t, "pvc-peer-snap-rebuild", th.BdevNoMemResources[0].ByBdev[0].LvolName) + assert.Equal(t, "read", th.BdevNoMemResources[0].ByBdev[0].IOType) + assert.Equal(t, uint16(36254), th.BdevNoMemResources[0].ByBdev[0].MaxRetries) + + assert.Len(t, procInfo.Warnings, 1) + assert.Len(t, procInfo.Errors, 2) +} + +func TestInspectProcessMemory_PerBlobIobufExhaustion(t *testing.T) { + mem := newSparseMem() + layout := defaultSpdkStructLayout(true) + syms := &spdkSymbols{ + gThreads: 0x500000, + gIobuf: 0x510000, + gLvolStores: 0x520000, + perBlobChannelSupported: true, + } + + largeRing := uint64(0x200000200000) + mem.putU64(syms.gIobuf+layout.iobufLargePoolCount, 1024) + mem.putU32(syms.gIobuf+layout.iobufLargeBufSize, 135168) + mem.putU64(syms.gIobuf+layout.iobufNode+layout.iobufNodeLargePool, largeRing) + mem.putU32(largeRing+layout.rteRingMask, 2047) + mem.putU32(largeRing+layout.rteRingCapacity, 1024) + mem.putU32(largeRing+layout.rteRingProdTail, 2784) + mem.putU32(largeRing+layout.rteRingConsHead, 2784) // ring_free = 0 + + // Thread with exhausted iobuf large_pool + 2 waiters, and 1 idle per-blob channel + threadAddr := uint64(0x700000) + mem.putU64(syms.gThreads, threadAddr) + mem.putCString(threadAddr+layout.threadName, "nvmf_tgt_poll_group_000") + + iobufChan := uint64(0x720000) + iobufDev := uint64(0x721000) + idleBlobChan := uint64(0x730000) + idleBlobDev := uint64(0x731000) + idleBlobPtr := uint64(0x732000) + + mem.putU64(threadAddr+layout.threadIOChannels, iobufChan) + mem.putU64(iobufChan+layout.ioChanDev, iobufDev) + mem.putCString(iobufDev+layout.ioDevName, "iobuf") + mem.putU64(iobufChan+layout.ioChanNode, idleBlobChan) + + // 2 waiters in large_queue + w1 := uint64(0x740000) + w2 := uint64(0x740100) + iobufCtx := iobufChan + layout.ioChanCtx + mem.putU64(iobufCtx+layout.iobufChanLargeQueue, w1) + mem.putU64(w1+layout.iobufEntryStailqNext, w2) + mem.putU64(w2+layout.iobufEntryStailqNext, 0) + + // nvmf_TCP spdk_iobuf_channel with large_cache = 0/512 + nvmfIch := uint64(0x750000) + modStruct := uint64(0x751000) + modNameStr := uint64(0x751100) + mem.putCString(modNameStr, "nvmf_TCP") + mem.putU64(modStruct, modNameStr) + mem.putU64(iobufCtx+layout.iobufChanChannels+8, nvmfIch) + mem.putU64(nvmfIch+layout.spdkIobufChanModule, modStruct) + mem.putU64(nvmfIch+layout.spdkIobufChanParent, iobufChan) + mem.putU32(nvmfIch+layout.spdkIobufChanCache+layout.poolCacheLargeCount, 0) + mem.putU32(nvmfIch+layout.spdkIobufChanCache+layout.poolCacheLargeSize, 512) + mem.putU64(nvmfIch+layout.spdkIobufChanCache+layout.poolCacheLargeRetry, 5409293) + + // Idle per-blob channel (all 512 reqs in free list) + mem.putU64(idleBlobChan+layout.ioChanDev, idleBlobDev) + mem.putU64(idleBlobDev+layout.ioDevPtr, idleBlobPtr) + mem.putCString(idleBlobDev+layout.ioDevName, "blob") + mem.putU64(idleBlobPtr+layout.blobID, 0x1000000aa) + reqMem := uint64(0x800000) + blobCtx := idleBlobChan + layout.ioChanCtx + mem.putU64(blobCtx+layout.bsChanReqMem, reqMem) + mem.putU64(blobCtx+layout.bsChanReqs, reqMem) + for i := 0; i < bsChannelMaxReqs-1; i++ { + cur := reqMem + uint64(i)*layout.bsReqSetSize + nxt := reqMem + uint64(i+1)*layout.bsReqSetSize + mem.putU64(cur+layout.bsReqSetLinkNext, nxt) + } + + procFiltered := &types.SpdkProcessInfo{PID: 908} + inspectProcessMemory(mem, syms, layout, nil, false, procFiltered) + require.Len(t, procFiltered.Threads, 1) + require.NotNil(t, procFiltered.Threads[0].IobufChannel) + assert.Equal(t, 2, procFiltered.Threads[0].IobufChannel.LargeWaiters) + assert.Equal(t, uint64(0), procFiltered.Threads[0].IobufChannel.TotalFreeLargeBuffers) + assert.Empty(t, procFiltered.Threads[0].BlobstoreChannels) + require.Len(t, procFiltered.Errors, 1) + assert.Contains(t, procFiltered.Errors[0], "waiting for iobuf large_pool buffers") + + procAll := &types.SpdkProcessInfo{PID: 908} + inspectProcessMemory(mem, syms, layout, nil, true, procAll) + require.Len(t, procAll.Threads[0].BlobstoreChannels, 1) + assert.Equal(t, "0x1000000aa", procAll.Threads[0].BlobstoreChannels[0].BlobID) + assert.Equal(t, 512, procAll.Threads[0].BlobstoreChannels[0].FreeReqs) +} + +func TestCollectSpdkRPC_UnixSocket(t *testing.T) { + tmpDir, err := os.MkdirTemp("/tmp", "spdk-rpc-*") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(tmpDir) + }() + + sockPath := filepath.Join(tmpDir, "spdk.sock") + ln, err := net.Listen("unix", sockPath) + require.NoError(t, err) + defer func() { + _ = ln.Close() + }() + + done := make(chan struct{}) + go func() { + defer close(done) + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func(c net.Conn) { + defer func() { _ = c.Close() }() + var req jsonRPCRequest + if err := json.NewDecoder(c).Decode(&req); err != nil { + return + } + var payload any + switch req.Method { + case "spdk_get_version": + payload = map[string]any{"version": "SPDK v26.05"} + case "iobuf_get_stats": + payload = []map[string]any{ + { + "module": "nvmf_TCP", + "small_pool": map[string]any{"cache_size": 4096, "cache": 10, "main": 0, "retry": 0}, + "large_pool": map[string]any{"cache_size": 3072, "cache": 999, "main": 0, "retry": 0}, + }, + } + case "nvmf_get_stats": + payload = map[string]any{ + "poll_groups": []map[string]any{ + {"name": "nvmf_tgt_poll_group_000", "current_admin_qpairs": 4, "current_io_qpairs": 12, "pending_bdev_io": 0, "completed_nvme_io": 500}, + }, + } + case "thread_get_pollers": + payload = map[string]any{ + "threads": []map[string]any{ + { + "name": "nvmf_tgt_poll_group_000", + "active_pollers": []map[string]any{{"name": "nvmf_TCP"}}, + "timed_pollers": []map[string]any{{"name": "bdev_no_mem_poller"}}, + }, + }, + } + case "bdev_nvme_get_controllers": + payload = []map[string]any{ + { + "name": "pvc-123-snap-rebuild", + "ctrlrs": []map[string]any{ + {"trid": map[string]any{"traddr": "10.42.1.5", "trsvcid": "20010"}}, + }, + }, + } + } + raw, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": req.ID, "result": payload}) + _, _ = c.Write(append(raw, '\n')) + }(conn) + } + }() + + snap := collectSpdkRPC(sockPath) + assert.Empty(t, snap.warnings) + assert.Equal(t, "SPDK v26.05", snap.version) + require.Len(t, snap.iobufStats, 1) + assert.Equal(t, uint64(999), snap.iobufStats[0].LargePool.Cache) + require.Len(t, snap.nvmfPollGroups, 1) + assert.Equal(t, uint32(12), snap.nvmfPollGroups[0].CurrentIOQpairs) + assert.Equal(t, 1, snap.threadPollers["nvmf_tgt_poll_group_000"].bdevNoMemPollerCount) + assert.Equal(t, "10.42.1.5:20010", resolveBackingDeviceTarget("bdev_pvc-123-snap-rebuildn1", snap.nvmeCtrlrAddrs)) +} + +func TestDWARFMemberOffset(t *testing.T) { + off, ok := dwarfMemberOffset(int64(96)) + assert.True(t, ok) + assert.Equal(t, uint64(96), off) + + // DW_OP_plus_uconst (0x23) with ULEB128 for 1032 (0x88, 0x08) + off, ok = dwarfMemberOffset([]byte{0x23, 0x88, 0x08}) + assert.True(t, ok) + assert.Equal(t, uint64(1032), off) +} diff --git a/pkg/local/spdk/rpc.go b/pkg/local/spdk/rpc.go new file mode 100644 index 000000000..d2db57366 --- /dev/null +++ b/pkg/local/spdk/rpc.go @@ -0,0 +1,241 @@ +package spdk + +import ( + "encoding/json" + "fmt" + "net" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/longhorn/cli/pkg/types" +) + +const ( + rpcDialTimeout = 2 * time.Second + rpcCallTimeout = 3 * time.Second +) + +type threadPollerSnapshot struct { + bdevNoMemPollerCount int + activePollers []string + timedPollers []string +} + +type rpcSnapshot struct { + version string + iobufStats []*types.SpdkIobufModuleStat + nvmfPollGroups []*types.SpdkNvmfPollGroupStat + threadPollers map[string]*threadPollerSnapshot + nvmeCtrlrAddrs map[string]string + warnings []string +} + +type jsonRPCRequest struct { + JSONRPC string `json:"jsonrpc"` + ID int `json:"id"` + Method string `json:"method"` +} + +type jsonRPCResponse struct { + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error,omitempty"` +} + +func callSpdkRPC(sockPath, method string, out any) error { + conn, err := net.DialTimeout("unix", sockPath, rpcDialTimeout) + if err != nil { + return err + } + defer func() { + _ = conn.Close() + }() + + if err := conn.SetDeadline(time.Now().Add(rpcCallTimeout)); err != nil { + return err + } + + reqBytes, err := json.Marshal(jsonRPCRequest{ + JSONRPC: "2.0", + ID: 1, + Method: method, + }) + if err != nil { + return err + } + reqBytes = append(reqBytes, '\n') + if _, err := conn.Write(reqBytes); err != nil { + return err + } + + dec := json.NewDecoder(conn) + var resp jsonRPCResponse + if err := dec.Decode(&resp); err != nil { + return err + } + if resp.Error != nil { + return errors.Errorf("RPC %s error %d: %s", method, resp.Error.Code, resp.Error.Message) + } + return json.Unmarshal(resp.Result, out) +} + +func collectSpdkRPC(sockPath string) *rpcSnapshot { + snap := &rpcSnapshot{ + threadPollers: make(map[string]*threadPollerSnapshot), + nvmeCtrlrAddrs: make(map[string]string), + } + + var ver struct { + Version string `json:"version"` + } + if err := callSpdkRPC(sockPath, "spdk_get_version", &ver); err != nil { + snap.warnings = append(snap.warnings, fmt.Sprintf("SPDK RPC unavailable at %s: %v", sockPath, err)) + return snap + } + snap.version = ver.Version + + var rawIobuf []struct { + Module string `json:"module"` + SmallPool struct { + CacheSize uint32 `json:"cache_size"` + Cache uint64 `json:"cache"` + Main uint64 `json:"main"` + Retry uint64 `json:"retry"` + } `json:"small_pool"` + LargePool struct { + CacheSize uint32 `json:"cache_size"` + Cache uint64 `json:"cache"` + Main uint64 `json:"main"` + Retry uint64 `json:"retry"` + } `json:"large_pool"` + } + if err := callSpdkRPC(sockPath, "iobuf_get_stats", &rawIobuf); err != nil { + snap.warnings = append(snap.warnings, fmt.Sprintf("RPC iobuf_get_stats failed: %v", err)) + } else { + for _, item := range rawIobuf { + snap.iobufStats = append(snap.iobufStats, &types.SpdkIobufModuleStat{ + Module: item.Module, + SmallPool: types.SpdkIobufPoolStat{ + CacheSize: item.SmallPool.CacheSize, + Cache: item.SmallPool.Cache, + Main: item.SmallPool.Main, + Retry: item.SmallPool.Retry, + }, + LargePool: types.SpdkIobufPoolStat{ + CacheSize: item.LargePool.CacheSize, + Cache: item.LargePool.Cache, + Main: item.LargePool.Main, + Retry: item.LargePool.Retry, + }, + }) + } + } + + var rawNvmf struct { + PollGroups []struct { + Name string `json:"name"` + CurrentAdminQpairs uint32 `json:"current_admin_qpairs"` + CurrentIOQpairs uint32 `json:"current_io_qpairs"` + PendingBdevIO uint64 `json:"pending_bdev_io"` + CompletedNvmeIO uint64 `json:"completed_nvme_io"` + } `json:"poll_groups"` + } + if err := callSpdkRPC(sockPath, "nvmf_get_stats", &rawNvmf); err != nil { + snap.warnings = append(snap.warnings, fmt.Sprintf("RPC nvmf_get_stats failed: %v", err)) + } else { + for _, pg := range rawNvmf.PollGroups { + snap.nvmfPollGroups = append(snap.nvmfPollGroups, &types.SpdkNvmfPollGroupStat{ + Name: pg.Name, + CurrentAdminQpairs: pg.CurrentAdminQpairs, + CurrentIOQpairs: pg.CurrentIOQpairs, + PendingBdevIO: pg.PendingBdevIO, + CompletedNvmeIO: pg.CompletedNvmeIO, + }) + } + } + + var rawPollers struct { + Threads []struct { + Name string `json:"name"` + ActivePollers []struct { + Name string `json:"name"` + } `json:"active_pollers"` + TimedPollers []struct { + Name string `json:"name"` + } `json:"timed_pollers"` + } `json:"threads"` + } + if err := callSpdkRPC(sockPath, "thread_get_pollers", &rawPollers); err != nil { + snap.warnings = append(snap.warnings, fmt.Sprintf("RPC thread_get_pollers failed: %v", err)) + } else { + for _, th := range rawPollers.Threads { + ps := &threadPollerSnapshot{} + for _, ap := range th.ActivePollers { + ps.activePollers = append(ps.activePollers, ap.Name) + } + for _, tp := range th.TimedPollers { + ps.timedPollers = append(ps.timedPollers, tp.Name) + if tp.Name == "bdev_no_mem_poller" { + ps.bdevNoMemPollerCount++ + } + } + snap.threadPollers[th.Name] = ps + } + } + + var rawCtrlrs []struct { + Name string `json:"name"` + Ctrlrs []struct { + Trid struct { + Traddr string `json:"traddr"` + Trsvcid string `json:"trsvcid"` + } `json:"trid"` + } `json:"ctrlrs"` + } + if err := callSpdkRPC(sockPath, "bdev_nvme_get_controllers", &rawCtrlrs); err == nil { + for _, c := range rawCtrlrs { + for _, sub := range c.Ctrlrs { + if sub.Trid.Traddr != "" { + addr := sub.Trid.Traddr + if sub.Trid.Trsvcid != "" { + addr = fmt.Sprintf("%s:%s", sub.Trid.Traddr, sub.Trid.Trsvcid) + } + snap.nvmeCtrlrAddrs[c.Name] = addr + break + } + } + } + } + + return snap +} + +func resolveBackingDeviceTarget(backingDev string, ctrlrAddrs map[string]string) string { + if backingDev == "" || len(ctrlrAddrs) == 0 { + return "" + } + trimmed := strings.TrimPrefix(backingDev, "bdev_") + if addr, ok := ctrlrAddrs[trimmed]; ok { + return addr + } + // Strip NVMe namespace suffix such as "n1" in "bdev_n1". + if idx := strings.LastIndexByte(trimmed, 'n'); idx > 0 { + allDigits := true + for _, r := range trimmed[idx+1:] { + if r < '0' || r > '9' { + allDigits = false + break + } + } + if allDigits { + if addr, ok := ctrlrAddrs[trimmed[:idx]]; ok { + return addr + } + } + } + return "" +} diff --git a/pkg/remote/spdk/getter.go b/pkg/remote/spdk/getter.go new file mode 100644 index 000000000..134d377b6 --- /dev/null +++ b/pkg/remote/spdk/getter.go @@ -0,0 +1,1003 @@ +package spdk + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net" + "path/filepath" + "slices" + "sort" + "strings" + "text/tabwriter" + + "github.com/pkg/errors" + + "k8s.io/utils/ptr" + "sigs.k8s.io/kustomize/kyaml/yaml" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeclient "k8s.io/client-go/kubernetes" + + commonkube "github.com/longhorn/go-common-libs/kubernetes" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/types" + "github.com/longhorn/cli/pkg/utils" + + kubeutils "github.com/longhorn/cli/pkg/utils/kubernetes" +) + +const ( + OutputFormatYAML = "yaml" + OutputFormatJSON = "json" + OutputFormatText = "text" +) + +// OutputFormats are the supported output formats for longhornctl diagnose v2-data-engine. +var OutputFormats = []string{OutputFormatYAML, OutputFormatJSON, OutputFormatText} + +// Diagnoser provides cluster-wide SPDK target memory, iobuf, and deadlock diagnosis. +type Diagnoser struct { + DiagnoserCmdOptions + + kubeClient kubeclient.Interface + rawClient *kubeclient.Clientset + + appName string + collection *types.SpdkCollection +} + +// DiagnoserCmdOptions holds the CLI options for longhornctl diagnose v2-data-engine. +type DiagnoserCmdOptions struct { + types.GlobalCmdOptions + + NodeID string + OutputFormat string + IncludeAll bool +} + +// Validate validates the command options. +func (remote *Diagnoser) Validate() error { + if remote.OutputFormat == "" { + remote.OutputFormat = OutputFormatYAML + } + if !slices.Contains(OutputFormats, remote.OutputFormat) { + return errors.Errorf("invalid --%s %q, must be one of %v", consts.CmdOptOutput, remote.OutputFormat, OutputFormats) + } + return nil +} + +// Init initializes the remote SPDK Diagnoser. +func (remote *Diagnoser) Init() error { + kubeClient, err := kubeutils.NewKubeClient("", remote.KubeConfigPath) + if err != nil { + return err + } + remote.rawClient = kubeClient + remote.kubeClient = kubeClient + remote.appName = consts.AppNameV2DataEngineDiagnoser + return nil +} + +// Run deploys the SPDK diagnoser DaemonSet, collects per-node SPDK diagnostics, +// analyzes cross-node dependencies and deadlock cycles, and marshals the output. +func (remote *Diagnoser) Run() ([]byte, error) { + newDaemonSet, err := kubeutils.PrepareDaemonSet(remote.newDaemonSet(), remote.rawClient, remote.NodeSelector, remote.ImagePullSecret, remote.Tolerations) + if err != nil { + return nil, err + } + if remote.NodeID != "" { + if newDaemonSet.Spec.Template.Spec.NodeSelector == nil { + newDaemonSet.Spec.Template.Spec.NodeSelector = make(map[string]string) + } + newDaemonSet.Spec.Template.Spec.NodeSelector[corev1.LabelHostname] = remote.NodeID + } + + daemonSet, err := commonkube.CreateDaemonSet(remote.rawClient, newDaemonSet) + if err != nil { + return nil, err + } + + err = kubeutils.MonitorDaemonSetContainer(remote.rawClient, daemonSet, consts.ContainerNameInit, kubeutils.WaitForDaemonSetContainersExit, ptr.To(consts.ContainerConditionMaxTolerationMedium)) + if err != nil { + return nil, err + } + + err = kubeutils.MonitorDaemonSetContainer(remote.rawClient, daemonSet, consts.ContainerNameOutput, kubeutils.WaitForDaemonSetContainersExit, ptr.To(consts.ContainerConditionMaxTolerationShort)) + if err != nil { + return nil, err + } + + podCollections, err := kubeutils.GetDaemonSetPodCollections(remote.rawClient, daemonSet, consts.ContainerNameOutput, false, false, nil) + if err != nil { + return nil, err + } + + collection := &types.SpdkCollection{ + Nodes: make(map[string]*types.SpdkNodeInfo), + } + for _, podLog := range podCollections.Pods { + var nodeInfo types.SpdkNodeInfo + if err := json.Unmarshal([]byte(podLog.Log), &nodeInfo); err != nil { + return nil, errors.Wrapf(err, "failed to parse SPDK output from node %s", podLog.Node) + } + nodeName := podLog.Node + if nodeName == "" { + nodeName = nodeInfo.Node + } + nodeInfo.Node = nodeName + collection.Nodes[nodeName] = &nodeInfo + } + + podIPToNode := remote.listInstanceManagerIPMap(context.Background()) + AnalyzeCrossNodeDependencies(collection, podIPToNode) + remote.collection = collection + + return MarshalCollection(collection, remote.OutputFormat) +} + +// UnhealthyReason returns a non-empty summary string when the diagnosed SPDK targets are stalled or deadlocked. +func (remote *Diagnoser) UnhealthyReason() string { + return UnhealthyReason(remote.collection) +} + +// UnhealthyReason returns a non-empty summary string when collection indicates Stalled or Deadlocked SPDK state. +func UnhealthyReason(collection *types.SpdkCollection) string { + if collection == nil || collection.Summary == nil { + return "" + } + status := collection.Summary.Status + if status != types.SpdkStatusDeadlocked && status != types.SpdkStatusStalled { + return "" + } + + var unhealthyNodes []string + for _, ns := range collection.Summary.Nodes { + if ns.Status == types.SpdkStatusDeadlocked || ns.Status == types.SpdkStatusStalled { + unhealthyNodes = append(unhealthyNodes, fmt.Sprintf("%s (%s)", ns.Node, ns.Status)) + } + } + if len(unhealthyNodes) > 0 { + return fmt.Sprintf("status=%s, unhealthy nodes: %s", status, strings.Join(unhealthyNodes, ", ")) + } + return fmt.Sprintf("status=%s", status) +} + +// Cleanup deletes the DaemonSet created for the SPDK diagnoser. +func (remote *Diagnoser) Cleanup() error { + return commonkube.DeleteDaemonSet(remote.rawClient, remote.Namespace, remote.appName) +} + +func (remote *Diagnoser) listInstanceManagerIPMap(ctx context.Context) map[string]string { + ipToNode := make(map[string]string) + if remote.kubeClient == nil { + return ipToNode + } + pods, err := remote.kubeClient.CoreV1().Pods(remote.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "longhorn.io/component=instance-manager", + }) + if err != nil { + return ipToNode + } + for i := range pods.Items { + pod := &pods.Items[i] + if pod.Status.PodIP != "" && pod.Spec.NodeName != "" { + ipToNode[pod.Status.PodIP] = pod.Spec.NodeName + } + } + return ipToNode +} + +func (remote *Diagnoser) newDaemonSet() *appsv1.DaemonSet { + outputFilePath := filepath.Join(consts.VolumeMountSharedDirectory, consts.FileNameOutputJSON) + cmd := []string{consts.CmdLonghornctlLocal, consts.SubCmdDiagnose, consts.SubCmdV2DataEngine} + if remote.IncludeAll { + cmd = append(cmd, "--"+consts.CmdOptAll) + } + + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: remote.appName, + Namespace: remote.Namespace, + Labels: map[string]string{ + "app": remote.appName, + }, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": remote.appName, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": remote.appName, + }, + }, + Spec: corev1.PodSpec{ + HostPID: true, + InitContainers: []corev1.Container{ + { + Name: consts.ContainerNameInit, + Image: utils.BuildImageName(remote.Image, remote.ImageRegistry), + Command: cmd, + Env: []corev1.EnvVar{ + { + Name: consts.EnvCurrentNodeID, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: consts.EnvLogLevel, + Value: remote.LogLevel, + }, + { + Name: consts.EnvOutputFilePath, + Value: outputFilePath, + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: consts.VolumeMountHostName, + MountPath: consts.VolumeMountHostDirectory, + ReadOnly: true, + }, + { + Name: consts.VolumeMountSharedName, + MountPath: consts.VolumeMountSharedDirectory, + }, + }, + }, + { + Name: consts.ContainerNameOutput, + Image: utils.BuildImageName(remote.Image, remote.ImageRegistry), + Command: []string{"cat", outputFilePath}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: consts.VolumeMountSharedName, + MountPath: consts.VolumeMountSharedDirectory, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: consts.ContainerNamePause, + Image: utils.BuildImageName(consts.ImagePause, remote.ImageRegistry), + }, + }, + Volumes: []corev1.Volume{ + { + Name: consts.VolumeMountHostName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/", + }, + }, + }, + { + Name: consts.VolumeMountSharedName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + Type: appsv1.RollingUpdateDaemonSetStrategyType, + }, + }, + } +} + +// MarshalCollection encodes a SpdkCollection in YAML, JSON, or human-readable text format. +func MarshalCollection(collection *types.SpdkCollection, format string) ([]byte, error) { + switch format { + case OutputFormatYAML: + return yaml.Marshal(collection) + case OutputFormatJSON: + out, err := json.MarshalIndent(collection, "", " ") + if err != nil { + return nil, err + } + return append(out, '\n'), nil + case OutputFormatText: + return []byte(FormatCollectionText(collection)), nil + default: + return nil, errors.Errorf("unsupported output format %q, must be one of %v", format, OutputFormats) + } +} + +// AnalyzeCrossNodeDependencies enriches active requests with target node names, +// populates CrossNodeDependencies, detects DeadlockCycles, and synthesizes Summary. +// All raw per-node fields under collection.Nodes are preserved intact. +func AnalyzeCrossNodeDependencies(collection *types.SpdkCollection, podIPToNode map[string]string) { + if collection == nil { + return + } + collection.CrossNodeDependencies = nil + collection.DeadlockCycles = nil + + if len(collection.Nodes) == 0 { + collection.Summary = &types.SpdkSummary{ + Status: types.SpdkStatusHealthy, + } + return + } + + nodeNames := make([]string, 0, len(collection.Nodes)) + stallReasons := make(map[string]string, len(collection.Nodes)) + for nodeName, nodeInfo := range collection.Nodes { + nodeNames = append(nodeNames, nodeName) + stallReasons[nodeName] = summarizeNodeStallReason(nodeInfo) + } + sort.Strings(nodeNames) + + // Resolve RemoteTargetNode on every active request first. + for _, fromNode := range nodeNames { + nodeInfo := collection.Nodes[fromNode] + if nodeInfo == nil { + continue + } + for _, proc := range nodeInfo.Processes { + for _, th := range proc.Threads { + for _, bsCh := range th.BlobstoreChannels { + for _, act := range bsCh.ActiveRequests { + if act.RemoteTargetAddress == "" { + continue + } + host := act.RemoteTargetAddress + if h, _, err := net.SplitHostPort(act.RemoteTargetAddress); err == nil { + host = h + } + if toNode := podIPToNode[host]; toNode != "" { + act.RemoteTargetNode = toNode + } + } + } + } + } + } + + // Propagate transitive stalls when a channel with queued cluster allocations waits on a stalled remote node. + changed := true + for changed { + changed = false + for _, fromNode := range nodeNames { + if stallReasons[fromNode] != "" { + continue + } + nodeInfo := collection.Nodes[fromNode] + if nodeInfo == nil { + continue + } + for _, proc := range nodeInfo.Processes { + for _, th := range proc.Threads { + for _, bsCh := range th.BlobstoreChannels { + if bsCh.NeedClusterAlloc == 0 && bsCh.FreeReqs > 0 { + continue + } + for _, act := range bsCh.ActiveRequests { + if act.RemoteTargetNode != "" && stallReasons[act.RemoteTargetNode] != "" { + stallReasons[fromNode] = fmt.Sprintf( + "%s bs_channel %s cluster_alloc blocked (need_cluster_alloc=%d, free_reqs=%d/%d) waiting on stalled node %s", + bsCh.Device, bsCh.Address, bsCh.NeedClusterAlloc, bsCh.FreeReqs, bsCh.TotalReqs, act.RemoteTargetNode) + changed = true + break + } + } + if changed { + break + } + } + if changed { + break + } + } + if changed { + break + } + } + } + } + + adj := make(map[string]map[string]struct{}) + for _, fromNode := range nodeNames { + nodeInfo := collection.Nodes[fromNode] + if nodeInfo == nil { + continue + } + for _, proc := range nodeInfo.Processes { + for _, th := range proc.Threads { + for _, bsCh := range th.BlobstoreChannels { + for _, act := range bsCh.ActiveRequests { + if act.RemoteTargetAddress == "" { + continue + } + toNode := act.RemoteTargetNode + reason := stallReasons[toNode] + lvolName := act.LvolName + if lvolName == "" { + lvolName = bsCh.LvolName + } + remoteSnap := extractRemoteSnapshotLvol(act.BackingDevice) + readCnt, maxRetries := findNodeNoMemReads(collection.Nodes[toNode], remoteSnap) + + dep := &types.SpdkCrossNodeDependency{ + FromNode: fromNode, + ToNode: toNode, + OpType: act.OpType, + TargetAddress: act.RemoteTargetAddress, + LocalLvol: lvolName, + RemoteSnapshotLvol: remoteSnap, + BackingDevice: act.BackingDevice, + IOUnit: act.IOUnit, + TargetStalled: reason != "", + TargetReason: reason, + RemoteNoMemReadCount: readCnt, + RemoteNoMemMaxRetries: maxRetries, + } + collection.CrossNodeDependencies = append(collection.CrossNodeDependencies, dep) + + if toNode != "" && toNode != fromNode && reason != "" && stallReasons[fromNode] != "" { + if adj[fromNode] == nil { + adj[fromNode] = make(map[string]struct{}) + } + adj[fromNode][toNode] = struct{}{} + } + } + } + } + } + } + + collection.DeadlockCycles = findDirectedCycles(adj) + collection.Summary = buildCollectionSummary(collection, nodeNames, stallReasons) +} + +func summarizeNodeStallReason(nodeInfo *types.SpdkNodeInfo) string { + if nodeInfo == nil { + return "" + } + var reasons []string + for _, proc := range nodeInfo.Processes { + for _, th := range proc.Threads { + if th.IobufChannel != nil && (th.IobufChannel.LargeWaiters > 0 || th.IobufChannel.SmallWaiters > 0) { + reasons = append(reasons, + fmt.Sprintf("iobuf pool exhausted (free=%d, waiters=%d)", + th.IobufChannel.TotalFreeLargeBuffers, th.IobufChannel.LargeWaiters+th.IobufChannel.SmallWaiters)) + } + for _, bsCh := range th.BlobstoreChannels { + if bsCh.FreeReqs == 0 && bsCh.TotalReqs > 0 { + if len(bsCh.NeedClusterAllocByLvol) > 0 { + reasons = append(reasons, + fmt.Sprintf("%s bs_channel %s free_reqs=0/%d (need_cluster_alloc=%d across %d lvols)", + bsCh.Device, bsCh.Address, bsCh.TotalReqs, bsCh.NeedClusterAlloc, len(bsCh.NeedClusterAllocByLvol))) + } else { + reasons = append(reasons, + fmt.Sprintf("%s bs_channel %s free_reqs=0/%d (need_cluster_alloc=%d)", + bsCh.Device, bsCh.Address, bsCh.TotalReqs, bsCh.NeedClusterAlloc)) + } + } else if bsCh.FreeReqs > 0 && bsCh.NeedClusterAlloc > 0 && len(bsCh.ActiveRequests) > 0 && + (bsCh.NeedClusterAlloc >= bsCh.TotalReqs/2 || len(th.BdevNoMemResources) > 0) { + reasons = append(reasons, + fmt.Sprintf("%s bs_channel %s cluster_alloc blocked (need_cluster_alloc=%d, free_reqs=%d/%d)", + bsCh.Device, bsCh.Address, bsCh.NeedClusterAlloc, bsCh.FreeReqs, bsCh.TotalReqs)) + } + } + + totalNoMemIO := 0 + noMemResCount := 0 + for _, nm := range th.BdevNoMemResources { + if nm.NoMemIOCount > 0 { + totalNoMemIO += nm.NoMemIOCount + noMemResCount++ + } + } + if totalNoMemIO > 0 { + reasons = append(reasons, fmt.Sprintf("bdev nomem_io=%d across %d bdevs", totalNoMemIO, noMemResCount)) + } + } + } + return strings.Join(reasons, "; ") +} + +func extractRemoteSnapshotLvol(backingDev string) string { + if backingDev == "" || backingDev == "blob" { + return "" + } + trimmed := strings.TrimPrefix(backingDev, "bdev_") + if idx := strings.LastIndexByte(trimmed, 'n'); idx > 0 && idx < len(trimmed)-1 { + allDigits := true + for _, r := range trimmed[idx+1:] { + if r < '0' || r > '9' { + allDigits = false + break + } + } + if allDigits { + return trimmed[:idx] + } + } + return trimmed +} + +func findNodeNoMemReads(nodeInfo *types.SpdkNodeInfo, targetLvol string) (int, uint16) { + if nodeInfo == nil || targetLvol == "" { + return 0, 0 + } + totalReads := 0 + var maxRetries uint16 + for _, proc := range nodeInfo.Processes { + for _, th := range proc.Threads { + for _, nm := range th.BdevNoMemResources { + for _, entry := range nm.ByBdev { + if entry.IOType != "read" { + continue + } + if entry.LvolName == targetLvol || entry.BdevName == targetLvol { + totalReads += entry.Count + if entry.MaxRetries > maxRetries { + maxRetries = entry.MaxRetries + } + } + } + } + } + } + return totalReads, maxRetries +} + +func buildCollectionSummary( + collection *types.SpdkCollection, + nodeNames []string, + stallReasons map[string]string, +) *types.SpdkSummary { + deadlockedSet := make(map[string]bool) + for _, cycle := range collection.DeadlockCycles { + for _, n := range cycle { + deadlockedSet[n] = true + } + } + + summary := &types.SpdkSummary{ + Status: types.SpdkStatusHealthy, + Nodes: make([]*types.SpdkNodeSummary, 0, len(nodeNames)), + } + + totalIobufWaiters := 0 + hasIobufTelemetry := false + hasBsExhaustionOrBlock := false + hasBdevNoMem := false + + for _, nodeName := range nodeNames { + nodeInfo := collection.Nodes[nodeName] + ns := buildNodeSummary(nodeName, nodeInfo, deadlockedSet[nodeName], stallReasons[nodeName], collection) + summary.Nodes = append(summary.Nodes, ns) + + totalIobufWaiters += ns.IobufWaiters + if ns.IobufLargeFree > 0 || ns.IobufWaiters > 0 { + hasIobufTelemetry = true + } + if ns.BsChannelTotalReqs > 0 && (ns.BsChannelFreeReqs == 0 || ns.BsChannelNeedClusterAlloc > 0) { + hasBsExhaustionOrBlock = true + } + if ns.BdevNoMemIOCount > 0 { + hasBdevNoMem = true + } + + switch ns.Status { + case types.SpdkStatusDeadlocked: + summary.Status = types.SpdkStatusDeadlocked + case types.SpdkStatusStalled: + if summary.Status != types.SpdkStatusDeadlocked { + summary.Status = types.SpdkStatusStalled + } + case types.SpdkStatusWarning: + if summary.Status == types.SpdkStatusHealthy { + summary.Status = types.SpdkStatusWarning + } + } + } + + for _, cycle := range collection.DeadlockCycles { + summary.Findings = append(summary.Findings, + fmt.Sprintf("Cross-node SPDK deadlock cycle detected: %s", strings.Join(cycle, " -> "))) + } + + for _, dep := range collection.CrossNodeDependencies { + if !dep.TargetStalled { + continue + } + targetLabel := dep.ToNode + if targetLabel == "" { + targetLabel = dep.TargetAddress + } + snapLabel := dep.RemoteSnapshotLvol + if snapLabel == "" { + snapLabel = dep.BackingDevice + } + if !deadlockedSet[dep.FromNode] { + summary.Findings = append(summary.Findings, + fmt.Sprintf("Node %s is blocked on local lvol %s waiting for stalled node %s (%s, snapshot %s)", + dep.FromNode, dep.LocalLvol, targetLabel, dep.TargetAddress, snapLabel)) + } + if dep.RemoteNoMemReadCount > 0 { + summary.Findings = append(summary.Findings, + fmt.Sprintf("Cross-node snapshot read %s -> %s (%s) has %d read I/O(s) stuck in %s nomem_io (maxRetries=%d)", + dep.FromNode, targetLabel, snapLabel, dep.RemoteNoMemReadCount, targetLabel, dep.RemoteNoMemMaxRetries)) + } + } + + for _, ns := range summary.Nodes { + if ns.IobufWaiters > 0 { + summary.Findings = append(summary.Findings, + fmt.Sprintf("Node %s has %d request(s) blocked on SPDK iobuf pool exhaustion (iobufLargeFree=%d)", + ns.Node, ns.IobufWaiters, ns.IobufLargeFree)) + } else if len(collection.DeadlockCycles) == 0 && ns.Status == types.SpdkStatusStalled && ns.Reason != "" { + alreadyCovered := false + for _, dep := range collection.CrossNodeDependencies { + if dep.FromNode == ns.Node && dep.TargetStalled { + alreadyCovered = true + break + } + } + if !alreadyCovered { + summary.Findings = append(summary.Findings, fmt.Sprintf("Node %s is stalled: %s", ns.Node, ns.Reason)) + } + } + } + + if (summary.Status == types.SpdkStatusDeadlocked || summary.Status == types.SpdkStatusStalled) && + totalIobufWaiters == 0 && hasIobufTelemetry && hasBsExhaustionOrBlock { + msg := "SPDK iobuf pools are healthy on all inspected nodes (0 waiters)" + if hasBdevNoMem { + msg += "; bdev nomem_io queues are a downstream symptom of blobstore channel exhaustion/blocking" + } + summary.Findings = append(summary.Findings, msg) + } + + return summary +} + +func buildNodeSummary( + nodeName string, + nodeInfo *types.SpdkNodeInfo, + inDeadlock bool, + stallReason string, + collection *types.SpdkCollection, +) *types.SpdkNodeSummary { + ns := &types.SpdkNodeSummary{ + Node: nodeName, + Status: types.SpdkStatusHealthy, + Reason: stallReason, + } + if nodeInfo == nil { + return ns + } + + hasErrors := len(nodeInfo.Errors) > 0 + hasWarnings := len(nodeInfo.Warnings) > 0 + affectedLvols := make(map[string]struct{}) + + var bestBsCh *types.SpdkBsChannelInfo + bestBsIsDataPlane := false + + for _, proc := range nodeInfo.Processes { + if ns.PID == 0 { + ns.PID = proc.PID + ns.PerBlobChannelSupported = proc.PerBlobChannelSupported + } + if len(proc.Errors) > 0 { + hasErrors = true + } + if len(proc.Warnings) > 0 { + hasWarnings = true + } + if proc.Iobuf != nil && uint64(proc.Iobuf.LargeRingFree) > ns.IobufLargeFree { + ns.IobufLargeFree = uint64(proc.Iobuf.LargeRingFree) + } + + for _, th := range proc.Threads { + isDataPlane := th.Name != "app_thread" + if th.IobufChannel != nil { + ns.IobufWaiters += th.IobufChannel.SmallWaiters + th.IobufChannel.LargeWaiters + if th.IobufChannel.TotalFreeLargeBuffers > ns.IobufLargeFree { + ns.IobufLargeFree = th.IobufChannel.TotalFreeLargeBuffers + } + } + + for _, bsCh := range th.BlobstoreChannels { + if isMoreConstrainedBsChannel(bsCh, isDataPlane, bestBsCh, bestBsIsDataPlane) { + bestBsCh = bsCh + bestBsIsDataPlane = isDataPlane + } + if bsCh.NeedClusterAlloc > 0 || bsCh.FreeReqs == 0 { + for lvol := range bsCh.NeedClusterAllocByLvol { + affectedLvols[lvol] = struct{}{} + } + if bsCh.LvolName != "" { + affectedLvols[bsCh.LvolName] = struct{}{} + } + for _, act := range bsCh.ActiveRequests { + ns.BlockingRequests = append(ns.BlockingRequests, formatBlockingRequest(act, bsCh, collection)) + } + } + } + + for _, nm := range th.BdevNoMemResources { + if nm.NoMemIOCount > 0 { + ns.BdevNoMemIOCount += nm.NoMemIOCount + ns.BdevNoMemBdevCount++ + } + } + } + } + + if bestBsCh != nil { + ns.BsChannelFreeReqs = bestBsCh.FreeReqs + ns.BsChannelTotalReqs = bestBsCh.TotalReqs + ns.BsChannelNeedClusterAlloc = bestBsCh.NeedClusterAlloc + } + ns.BsChannelAffectedLvols = len(affectedLvols) + + switch { + case inDeadlock: + ns.Status = types.SpdkStatusDeadlocked + case stallReason != "" || hasErrors: + ns.Status = types.SpdkStatusStalled + if ns.Reason == "" { + ns.Reason = firstNodeError(nodeInfo) + } + case hasWarnings: + ns.Status = types.SpdkStatusWarning + default: + ns.Status = types.SpdkStatusHealthy + } + + return ns +} + +func isMoreConstrainedBsChannel( + cand *types.SpdkBsChannelInfo, candDataPlane bool, + cur *types.SpdkBsChannelInfo, curDataPlane bool, +) bool { + if cur == nil { + return true + } + candBusy := cand.NeedClusterAlloc > 0 || cand.FreeReqs < cand.TotalReqs || len(cand.ActiveRequests) > 0 + curBusy := cur.NeedClusterAlloc > 0 || cur.FreeReqs < cur.TotalReqs || len(cur.ActiveRequests) > 0 + if candBusy != curBusy { + return candBusy + } + if candDataPlane != curDataPlane { + return candDataPlane + } + if cand.FreeReqs != cur.FreeReqs { + return cand.FreeReqs < cur.FreeReqs + } + return cand.NeedClusterAlloc > cur.NeedClusterAlloc +} + +func formatBlockingRequest( + act *types.SpdkBsActiveRequest, + bsCh *types.SpdkBsChannelInfo, + collection *types.SpdkCollection, +) string { + lvol := act.LvolName + if lvol == "" { + lvol = bsCh.LvolName + } + if lvol == "" { + lvol = act.BlobID + } + + desc := fmt.Sprintf("req #%d (%s)", act.Index, act.OpType) + if lvol != "" { + desc += fmt.Sprintf(" on %s", lvol) + } + if act.IOUnit > 0 { + desc += fmt.Sprintf(" [ioUnit=%d]", act.IOUnit) + } + if act.RemoteTargetAddress != "" { + target := act.RemoteTargetAddress + if act.RemoteTargetNode != "" { + target = fmt.Sprintf("%s (%s)", act.RemoteTargetNode, act.RemoteTargetAddress) + } + snap := extractRemoteSnapshotLvol(act.BackingDevice) + if snap != "" { + desc += fmt.Sprintf(" -> %s snap=%s", target, snap) + if collection != nil && act.RemoteTargetNode != "" { + if readCnt, retries := findNodeNoMemReads(collection.Nodes[act.RemoteTargetNode], snap); readCnt > 0 { + desc += fmt.Sprintf(" [%d read I/O(s) stuck in target nomem_io, maxRetries=%d]", readCnt, retries) + } + } + } else { + desc += fmt.Sprintf(" -> %s", target) + } + } + return desc +} + +func firstNodeError(nodeInfo *types.SpdkNodeInfo) string { + if len(nodeInfo.Errors) > 0 { + return nodeInfo.Errors[0] + } + for _, proc := range nodeInfo.Processes { + if len(proc.Errors) > 0 { + return proc.Errors[0] + } + } + return "" +} + +// FormatCollectionText renders a concise, human-readable diagnostic report for quick troubleshooting. +func FormatCollectionText(collection *types.SpdkCollection) string { + if collection == nil { + return "" + } + if collection.Summary == nil { + nodeNames := make([]string, 0, len(collection.Nodes)) + stallReasons := make(map[string]string, len(collection.Nodes)) + for n, info := range collection.Nodes { + nodeNames = append(nodeNames, n) + stallReasons[n] = summarizeNodeStallReason(info) + } + sort.Strings(nodeNames) + collection.Summary = buildCollectionSummary(collection, nodeNames, stallReasons) + } + + var buf bytes.Buffer + sum := collection.Summary + fmt.Fprintf(&buf, "CLUSTER SPDK STATUS: %s\n", strings.ToUpper(sum.Status)) + + if len(sum.Findings) > 0 { + fmt.Fprintf(&buf, "\nFINDINGS:\n") + for _, f := range sum.Findings { + fmt.Fprintf(&buf, " - %s\n", f) + } + } + + if len(collection.DeadlockCycles) > 0 { + fmt.Fprintf(&buf, "\nDEADLOCK CYCLES:\n") + for i, cycle := range collection.DeadlockCycles { + fmt.Fprintf(&buf, " [%d] %s\n", i+1, strings.Join(cycle, " -> ")) + } + } + + if len(collection.CrossNodeDependencies) > 0 { + fmt.Fprintf(&buf, "\nCROSS-NODE DEPENDENCIES:\n") + for _, dep := range collection.CrossNodeDependencies { + toLabel := dep.ToNode + if toLabel == "" { + toLabel = dep.TargetAddress + } else { + toLabel = fmt.Sprintf("%s (%s)", dep.ToNode, dep.TargetAddress) + } + snap := dep.RemoteSnapshotLvol + if snap == "" { + snap = dep.BackingDevice + } + state := "target=ok" + if dep.TargetStalled { + state = "target=STALLED" + } + line := fmt.Sprintf(" - %s (%s) --[%s]--> %s snap=%s [%s]", + dep.FromNode, dep.LocalLvol, dep.OpType, toLabel, snap, state) + if dep.RemoteNoMemReadCount > 0 { + line += fmt.Sprintf(" (%d read I/Os stuck in target nomem_io, maxRetries=%d)", + dep.RemoteNoMemReadCount, dep.RemoteNoMemMaxRetries) + } + fmt.Fprintln(&buf, line) + } + } + + if len(sum.Nodes) > 0 { + fmt.Fprintf(&buf, "\nNODE SUMMARY:\n") + tw := tabwriter.NewWriter(&buf, 0, 0, 2, ' ', 0) + _, _ = fmt.Fprintln(tw, " NODE\tSTATUS\tPID\tPER-BLOB-CH\tIOBUF-LARGE-FREE\tBS-REQ-FREE\tCLUSTER-ALLOC-Q\tBDEV-NOMEM-IO") + for _, ns := range sum.Nodes { + iobufCol := fmt.Sprintf("%d (%d waiters)", ns.IobufLargeFree, ns.IobufWaiters) + bsFreeCol := "-" + allocCol := "-" + if ns.BsChannelTotalReqs > 0 { + bsFreeCol = fmt.Sprintf("%d/%d", ns.BsChannelFreeReqs, ns.BsChannelTotalReqs) + if ns.BsChannelAffectedLvols > 0 { + allocCol = fmt.Sprintf("%d (%d lvols)", ns.BsChannelNeedClusterAlloc, ns.BsChannelAffectedLvols) + } else { + allocCol = fmt.Sprintf("%d", ns.BsChannelNeedClusterAlloc) + } + } + nomemCol := "0" + if ns.BdevNoMemIOCount > 0 { + nomemCol = fmt.Sprintf("%d (%d bdevs)", ns.BdevNoMemIOCount, ns.BdevNoMemBdevCount) + } + _, _ = fmt.Fprintf(tw, " %s\t%s\t%d\t%t\t%s\t%s\t%s\t%s\n", + ns.Node, ns.Status, ns.PID, ns.PerBlobChannelSupported, iobufCol, bsFreeCol, allocCol, nomemCol) + } + _ = tw.Flush() + } + + hasDetails := false + for _, ns := range sum.Nodes { + if len(ns.BlockingRequests) > 0 || (ns.Status != types.SpdkStatusHealthy && ns.Reason != "") { + if !hasDetails { + fmt.Fprintf(&buf, "\nNODE DETAILS:\n") + hasDetails = true + } + fmt.Fprintf(&buf, " %s (%s):\n", ns.Node, ns.Status) + if ns.Reason != "" { + fmt.Fprintf(&buf, " reason: %s\n", ns.Reason) + } + for _, br := range ns.BlockingRequests { + fmt.Fprintf(&buf, " blocker: %s\n", br) + } + } + } + + fmt.Fprintf(&buf, "\nNote: Use \"-o yaml\" or \"-o json\" for full raw thread, poller, iobuf cache, and per-lvol queue forensic data.\n") + return buf.String() +} + +func findDirectedCycles(adj map[string]map[string]struct{}) [][]string { + if len(adj) == 0 { + return nil + } + nodes := make([]string, 0, len(adj)) + for n := range adj { + nodes = append(nodes, n) + } + sort.Strings(nodes) + + seenCycles := make(map[string]struct{}) + var cycles [][]string + + for _, start := range nodes { + var dfs func(cur string, path []string, inPath map[string]bool) + dfs = func(cur string, path []string, inPath map[string]bool) { + nexts := make([]string, 0, len(adj[cur])) + for nxt := range adj[cur] { + nexts = append(nexts, nxt) + } + sort.Strings(nexts) + + for _, nxt := range nexts { + if nxt < start { + continue + } + if nxt == start && len(path) >= 2 { + cycle := make([]string, len(path)+1) + copy(cycle, path) + cycle[len(path)] = start + key := strings.Join(cycle, "->") + if _, dup := seenCycles[key]; !dup { + seenCycles[key] = struct{}{} + cycles = append(cycles, cycle) + } + continue + } + if inPath[nxt] { + continue + } + inPath[nxt] = true + dfs(nxt, append(path, nxt), inPath) + delete(inPath, nxt) + } + } + dfs(start, []string{start}, map[string]bool{start: true}) + } + + return cycles +} diff --git a/pkg/remote/spdk/getter_test.go b/pkg/remote/spdk/getter_test.go new file mode 100644 index 000000000..c4732ac2f --- /dev/null +++ b/pkg/remote/spdk/getter_test.go @@ -0,0 +1,272 @@ +package spdk + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/longhorn/cli/pkg/types" +) + +func TestAnalyzeCrossNodeDependencies_ThreeNodeDeadlock(t *testing.T) { + collection := &types.SpdkCollection{ + Nodes: map[string]*types.SpdkNodeInfo{ + "hp-23": { + Node: "hp-23", + Processes: []*types.SpdkProcessInfo{ + { + PID: 23, + Iobuf: &types.SpdkIobufInfo{ + LargePoolCount: 6144, + LargeRingFree: 3008, + }, + Threads: []*types.SpdkThreadInfo{ + { + Name: "app_thread", + BlobstoreChannels: []*types.SpdkBsChannelInfo{ + { + Device: "blobstore", + Address: "0x4b4cf90", + TotalReqs: 512, + FreeReqs: 512, + }, + }, + }, + { + Name: "nvmf_tgt_poll_group_000", + IobufChannel: &types.SpdkThreadIobufInfo{ + TotalFreeLargeBuffers: 4409, + }, + BlobstoreChannels: []*types.SpdkBsChannelInfo{ + { + Device: "blobstore", + Address: "0x7f77442cdfb0", + TotalReqs: 512, + FreeReqs: 0, + NeedClusterAlloc: 511, + NeedClusterAllocByLvol: map[string]int{ + "pvc-a-r-1": 300, + "pvc-x-r-9": 211, + }, + ActiveRequests: []*types.SpdkBsActiveRequest{ + { + Index: 4, + OpType: "esnap_cow", + LvolName: "pvc-a-r-1", + IOUnit: 4636672, + BackingDevice: "bdev_pvc-a-snap-on-123n1", + RemoteTargetAddress: "10.42.1.123:20010", + }, + }, + }, + }, + BdevNoMemResources: []*types.SpdkBdevNoMemInfo{ + { + NoMemIOCount: 8, + ByBdev: []*types.SpdkBdevNoMemEntry{ + { + BdevName: "1af7ab1b-ffe1-4687-95f6-2719672fcda0", + LvolName: "pvc-b-snap-on-23", + IOType: "read", + Count: 8, + MaxRetries: 38719, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "hp-123": { + Node: "hp-123", + Processes: []*types.SpdkProcessInfo{ + { + PID: 23, + Iobuf: &types.SpdkIobufInfo{ + LargePoolCount: 6144, + LargeRingFree: 3008, + }, + Threads: []*types.SpdkThreadInfo{ + { + Name: "nvmf_tgt_poll_group_000", + IobufChannel: &types.SpdkThreadIobufInfo{ + TotalFreeLargeBuffers: 4906, + }, + BlobstoreChannels: []*types.SpdkBsChannelInfo{ + { + Device: "blobstore", + Address: "0x7fb7a43949a0", + TotalReqs: 512, + FreeReqs: 0, + NeedClusterAlloc: 511, + ActiveRequests: []*types.SpdkBsActiveRequest{ + { + Index: 12, + OpType: "esnap_cow", + LvolName: "pvc-b-r-2", + IOUnit: 7585792, + BackingDevice: "bdev_pvc-b-snap-on-23n1", + RemoteTargetAddress: "10.42.0.23:20020", + }, + }, + }, + }, + BdevNoMemResources: []*types.SpdkBdevNoMemInfo{ + { + NoMemIOCount: 8, + ByBdev: []*types.SpdkBdevNoMemEntry{ + { + BdevName: "cd5bc02d-cad2-4e81-bbad-8a4a27d23230", + LvolName: "pvc-a-snap-on-123", + IOType: "read", + Count: 8, + MaxRetries: 38746, + }, + }, + }, + }, + }, + }, + }, + }, + }, + "hp-161": { + Node: "hp-161", + Processes: []*types.SpdkProcessInfo{ + { + PID: 23, + Iobuf: &types.SpdkIobufInfo{ + LargePoolCount: 6144, + LargeRingFree: 2944, + }, + Threads: []*types.SpdkThreadInfo{ + { + Name: "nvmf_tgt_poll_group_000", + IobufChannel: &types.SpdkThreadIobufInfo{ + TotalFreeLargeBuffers: 4171, + }, + BlobstoreChannels: []*types.SpdkBsChannelInfo{ + { + Device: "blobstore", + Address: "0x7f1100200000", + TotalReqs: 512, + FreeReqs: 47, + NeedClusterAlloc: 464, + ActiveRequests: []*types.SpdkBsActiveRequest{ + { + Index: 1, + OpType: "esnap_cow", + LvolName: "pvc-c-r-3", + BackingDevice: "bdev_pvc-c-snap-on-23n1", + RemoteTargetAddress: "10.42.0.23:20025", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + podIPToNode := map[string]string{ + "10.42.0.23": "hp-23", + "10.42.1.123": "hp-123", + "10.42.2.161": "hp-161", + } + + AnalyzeCrossNodeDependencies(collection, podIPToNode) + + require.Len(t, collection.CrossNodeDependencies, 3) + for _, dep := range collection.CrossNodeDependencies { + assert.True(t, dep.TargetStalled) + assert.Contains(t, dep.TargetReason, "free_reqs=0/512") + } + // Verify cross-node snapshot read correlation against remote nomem_io + assert.Equal(t, "pvc-b-snap-on-23", collection.CrossNodeDependencies[0].RemoteSnapshotLvol) + assert.Equal(t, 8, collection.CrossNodeDependencies[0].RemoteNoMemReadCount) + assert.Equal(t, uint16(38719), collection.CrossNodeDependencies[0].RemoteNoMemMaxRetries) + + require.Len(t, collection.DeadlockCycles, 1) + assert.Equal(t, []string{"hp-123", "hp-23", "hp-123"}, collection.DeadlockCycles[0]) + + require.NotNil(t, collection.Summary) + assert.Equal(t, types.SpdkStatusDeadlocked, collection.Summary.Status) + require.Len(t, collection.Summary.Nodes, 3) + // Sorted node order: hp-123, hp-161, hp-23 + assert.Equal(t, types.SpdkStatusDeadlocked, collection.Summary.Nodes[0].Status) + assert.Equal(t, types.SpdkStatusStalled, collection.Summary.Nodes[1].Status) + assert.Equal(t, 47, collection.Summary.Nodes[1].BsChannelFreeReqs) + assert.Equal(t, 464, collection.Summary.Nodes[1].BsChannelNeedClusterAlloc) + assert.Equal(t, types.SpdkStatusDeadlocked, collection.Summary.Nodes[2].Status) + // Ensure data-plane channel (0/512) was chosen over idle app_thread (512/512) + assert.Equal(t, 0, collection.Summary.Nodes[2].BsChannelFreeReqs) + assert.Equal(t, 2, collection.Summary.Nodes[2].BsChannelAffectedLvols) + + // Verify YAML places summary, deadlockCycles, and crossNodeDependencies BEFORE raw nodes while preserving raw data + yamlBytes, err := MarshalCollection(collection, OutputFormatYAML) + require.NoError(t, err) + yamlStr := string(yamlBytes) + summaryIdx := strings.Index(yamlStr, "summary:") + cycleIdx := strings.Index(yamlStr, "deadlockCycles:") + depIdx := strings.Index(yamlStr, "crossNodeDependencies:") + nodesIdx := strings.Index(yamlStr, "\nnodes:\n") + require.True(t, summaryIdx == 0 && summaryIdx < cycleIdx && cycleIdx < depIdx && depIdx < nodesIdx) + // Raw data (such as app_thread and raw UUID BdevName) remains preserved under nodes + assert.Contains(t, yamlStr[nodesIdx:], "app_thread") + assert.Contains(t, yamlStr[nodesIdx:], "1af7ab1b-ffe1-4687-95f6-2719672fcda0") + + textBytes, err := MarshalCollection(collection, OutputFormatText) + require.NoError(t, err) + textStr := string(textBytes) + assert.Contains(t, textStr, "CLUSTER SPDK STATUS: DEADLOCKED") + assert.Contains(t, textStr, "hp-123 -> hp-23 -> hp-123") + assert.Contains(t, textStr, "8 read I/Os stuck in target nomem_io") + assert.Contains(t, textStr, "NODE SUMMARY:") + + reason := UnhealthyReason(collection) + assert.Equal(t, "status=Deadlocked, unhealthy nodes: hp-123 (Deadlocked), hp-161 (Stalled), hp-23 (Deadlocked)", reason) +} + +func TestDiagnoserValidateMarshalAndDaemonSet(t *testing.T) { + d := &Diagnoser{} + require.NoError(t, d.Validate()) + assert.Equal(t, OutputFormatYAML, d.OutputFormat) + + d.OutputFormat = OutputFormatText + require.NoError(t, d.Validate()) + + d.OutputFormat = "invalid" + assert.EqualError(t, d.Validate(), `invalid --output "invalid", must be one of [yaml json text]`) + + d.IncludeAll = true + ds := d.newDaemonSet() + require.Len(t, ds.Spec.Template.Spec.InitContainers, 2) + assert.Equal(t, []string{"longhornctl-local", "diagnose", "v2-data-engine", "--all"}, ds.Spec.Template.Spec.InitContainers[0].Command) + + col := &types.SpdkCollection{ + Nodes: map[string]*types.SpdkNodeInfo{ + "node-1": {Node: "node-1", Processes: []*types.SpdkProcessInfo{{PID: 100}}}, + }, + } + AnalyzeCrossNodeDependencies(col, nil) + assert.Empty(t, UnhealthyReason(col)) + + yamlBytes, err := MarshalCollection(col, OutputFormatYAML) + require.NoError(t, err) + assert.Contains(t, string(yamlBytes), "node-1:") + + jsonBytes, err := MarshalCollection(col, OutputFormatJSON) + require.NoError(t, err) + assert.Contains(t, string(jsonBytes), `"pid": 100`) + + textBytes, err := MarshalCollection(col, OutputFormatText) + require.NoError(t, err) + assert.Contains(t, string(textBytes), "CLUSTER SPDK STATUS: HEALTHY") +} diff --git a/pkg/types/spdk.go b/pkg/types/spdk.go new file mode 100644 index 000000000..d58fb2f23 --- /dev/null +++ b/pkg/types/spdk.go @@ -0,0 +1,193 @@ +package types + +const ( + SpdkStatusHealthy = "Healthy" + SpdkStatusWarning = "Warning" + SpdkStatusStalled = "Stalled" + SpdkStatusDeadlocked = "Deadlocked" +) + +// SpdkCollection holds cluster-wide or single-node SPDK target memory and iobuf diagnostics. +// Summary, DeadlockCycles, and CrossNodeDependencies are placed ahead of Nodes so YAML/JSON +// outputs surface the high-level diagnosis first while preserving all raw node data below. +type SpdkCollection struct { + Summary *SpdkSummary `json:"summary,omitempty" yaml:"summary,omitempty"` + DeadlockCycles [][]string `json:"deadlockCycles,omitempty" yaml:"deadlockCycles,omitempty"` + CrossNodeDependencies []*SpdkCrossNodeDependency `json:"crossNodeDependencies,omitempty" yaml:"crossNodeDependencies,omitempty"` + Nodes map[string]*SpdkNodeInfo `json:"nodes" yaml:"nodes"` +} + +// SpdkSummary provides a high-level cluster and per-node diagnosis synthesized from raw SPDK data. +type SpdkSummary struct { + Status string `json:"status" yaml:"status"` + Findings []string `json:"findings,omitempty" yaml:"findings,omitempty"` + Nodes []*SpdkNodeSummary `json:"nodes,omitempty" yaml:"nodes,omitempty"` +} + +// SpdkNodeSummary summarizes the data-plane SPDK state on one node for quick triage. +type SpdkNodeSummary struct { + Node string `json:"node" yaml:"node"` + Status string `json:"status" yaml:"status"` + PID int `json:"pid,omitempty" yaml:"pid,omitempty"` + PerBlobChannelSupported bool `json:"perBlobChannelSupported" yaml:"perBlobChannelSupported"` + IobufLargeFree uint64 `json:"iobufLargeFree" yaml:"iobufLargeFree"` + IobufWaiters int `json:"iobufWaiters" yaml:"iobufWaiters"` + BsChannelFreeReqs int `json:"bsChannelFreeReqs" yaml:"bsChannelFreeReqs"` + BsChannelTotalReqs int `json:"bsChannelTotalReqs" yaml:"bsChannelTotalReqs"` + BsChannelNeedClusterAlloc int `json:"bsChannelNeedClusterAlloc" yaml:"bsChannelNeedClusterAlloc"` + BsChannelAffectedLvols int `json:"bsChannelAffectedLvols" yaml:"bsChannelAffectedLvols"` + BdevNoMemIOCount int `json:"bdevNoMemIOCount" yaml:"bdevNoMemIOCount"` + BdevNoMemBdevCount int `json:"bdevNoMemBdevCount" yaml:"bdevNoMemBdevCount"` + BlockingRequests []string `json:"blockingRequests,omitempty" yaml:"blockingRequests,omitempty"` + Reason string `json:"reason,omitempty" yaml:"reason,omitempty"` +} + +// SpdkCrossNodeDependency describes an in-flight SPDK request on FromNode waiting on a remote NVMe-oF target on ToNode. +type SpdkCrossNodeDependency struct { + FromNode string `json:"fromNode" yaml:"fromNode"` + ToNode string `json:"toNode,omitempty" yaml:"toNode,omitempty"` + OpType string `json:"opType,omitempty" yaml:"opType,omitempty"` + TargetAddress string `json:"targetAddress" yaml:"targetAddress"` + LocalLvol string `json:"localLvol,omitempty" yaml:"localLvol,omitempty"` + RemoteSnapshotLvol string `json:"remoteSnapshotLvol,omitempty" yaml:"remoteSnapshotLvol,omitempty"` + BackingDevice string `json:"backingDevice" yaml:"backingDevice"` + IOUnit uint64 `json:"ioUnit,omitempty" yaml:"ioUnit,omitempty"` + TargetStalled bool `json:"targetStalled,omitempty" yaml:"targetStalled,omitempty"` + TargetReason string `json:"targetReason,omitempty" yaml:"targetReason,omitempty"` + RemoteNoMemReadCount int `json:"remoteNoMemReadCount,omitempty" yaml:"remoteNoMemReadCount,omitempty"` + RemoteNoMemMaxRetries uint16 `json:"remoteNoMemMaxRetries,omitempty" yaml:"remoteNoMemMaxRetries,omitempty"` +} + +// SpdkNodeInfo holds SPDK diagnostics collected on a single node. +type SpdkNodeInfo struct { + Node string `json:"node,omitempty" yaml:"node,omitempty"` + Processes []*SpdkProcessInfo `json:"processes" yaml:"processes"` + Warnings []string `json:"warnings,omitempty" yaml:"warnings,omitempty"` + Errors []string `json:"errors,omitempty" yaml:"errors,omitempty"` +} + +// SpdkProcessInfo holds memory, iobuf, blobstore channel, and RPC diagnostics for one spdk_tgt process. +type SpdkProcessInfo struct { + PID int `json:"pid" yaml:"pid"` + Cmdline string `json:"cmdline,omitempty" yaml:"cmdline,omitempty"` + Executable string `json:"executable,omitempty" yaml:"executable,omitempty"` + Version string `json:"version,omitempty" yaml:"version,omitempty"` + PerBlobChannelSupported bool `json:"perBlobChannelSupported" yaml:"perBlobChannelSupported"` + DWARFLoaded bool `json:"dwarfLoaded" yaml:"dwarfLoaded"` + Iobuf *SpdkIobufInfo `json:"iobuf,omitempty" yaml:"iobuf,omitempty"` + NvmfPollGroups []*SpdkNvmfPollGroupStat `json:"nvmfPollGroups,omitempty" yaml:"nvmfPollGroups,omitempty"` + Threads []*SpdkThreadInfo `json:"threads,omitempty" yaml:"threads,omitempty"` + Warnings []string `json:"warnings,omitempty" yaml:"warnings,omitempty"` + Errors []string `json:"errors,omitempty" yaml:"errors,omitempty"` +} + +// SpdkIobufInfo holds global g_iobuf pool configuration, live DPDK ring free counts, and RPC module statistics. +type SpdkIobufInfo struct { + SmallPoolCount uint64 `json:"smallPoolCount" yaml:"smallPoolCount"` + SmallBufSize uint32 `json:"smallBufSize" yaml:"smallBufSize"` + SmallRingFree uint32 `json:"smallRingFree" yaml:"smallRingFree"` + LargePoolCount uint64 `json:"largePoolCount" yaml:"largePoolCount"` + LargeBufSize uint32 `json:"largeBufSize" yaml:"largeBufSize"` + LargeRingFree uint32 `json:"largeRingFree" yaml:"largeRingFree"` + ModuleStats []*SpdkIobufModuleStat `json:"moduleStats,omitempty" yaml:"moduleStats,omitempty"` +} + +// SpdkIobufModuleStat holds cumulative per-module iobuf allocation statistics from iobuf_get_stats RPC. +type SpdkIobufModuleStat struct { + Module string `json:"module" yaml:"module"` + SmallPool SpdkIobufPoolStat `json:"smallPool" yaml:"smallPool"` + LargePool SpdkIobufPoolStat `json:"largePool" yaml:"largePool"` +} + +// SpdkIobufPoolStat holds cumulative allocation counters for an iobuf pool. +type SpdkIobufPoolStat struct { + CacheSize uint32 `json:"cacheSize" yaml:"cacheSize"` + Cache uint64 `json:"cache" yaml:"cache"` + Main uint64 `json:"main" yaml:"main"` + Retry uint64 `json:"retry" yaml:"retry"` +} + +// SpdkNvmfPollGroupStat holds NVMe-oF target poll group counters from nvmf_get_stats RPC. +type SpdkNvmfPollGroupStat struct { + Name string `json:"name" yaml:"name"` + CurrentAdminQpairs uint32 `json:"currentAdminQpairs" yaml:"currentAdminQpairs"` + CurrentIOQpairs uint32 `json:"currentIOQpairs" yaml:"currentIOQpairs"` + PendingBdevIO uint64 `json:"pendingBdevIO" yaml:"pendingBdevIO"` + CompletedNvmeIO uint64 `json:"completedNvmeIO" yaml:"completedNvmeIO"` +} + +// SpdkThreadInfo holds memory and poller state for an SPDK thread (such as nvmf_tgt_poll_group_000 or app_thread). +type SpdkThreadInfo struct { + Name string `json:"name" yaml:"name"` + ID uint64 `json:"id" yaml:"id"` + BdevNoMemPollerCount int `json:"bdevNoMemPollerCount" yaml:"bdevNoMemPollerCount"` + ActivePollers []string `json:"activePollers,omitempty" yaml:"activePollers,omitempty"` + TimedPollerCounts map[string]int `json:"timedPollerCounts,omitempty" yaml:"timedPollerCounts,omitempty"` + TimedPollers []string `json:"timedPollers,omitempty" yaml:"timedPollers,omitempty"` + IobufChannel *SpdkThreadIobufInfo `json:"iobufChannel,omitempty" yaml:"iobufChannel,omitempty"` + BlobstoreChannels []*SpdkBsChannelInfo `json:"blobstoreChannels,omitempty" yaml:"blobstoreChannels,omitempty"` + BdevNoMemResources []*SpdkBdevNoMemInfo `json:"bdevNoMemResources,omitempty" yaml:"bdevNoMemResources,omitempty"` +} + +// SpdkThreadIobufInfo holds live per-thread iobuf wait queues and per-module lockless cache occupancy. +type SpdkThreadIobufInfo struct { + SmallWaiters int `json:"smallWaiters" yaml:"smallWaiters"` + LargeWaiters int `json:"largeWaiters" yaml:"largeWaiters"` + TotalFreeLargeBuffers uint64 `json:"totalFreeLargeBuffers,omitempty" yaml:"totalFreeLargeBuffers,omitempty"` + Modules []*SpdkThreadIobufModuleCache `json:"modules,omitempty" yaml:"modules,omitempty"` +} + +// SpdkThreadIobufModuleCache holds live cache counts and retry counters for one iobuf module on a thread. +type SpdkThreadIobufModuleCache struct { + Module string `json:"module" yaml:"module"` + SmallCacheCount uint32 `json:"smallCacheCount" yaml:"smallCacheCount"` + SmallCacheSize uint32 `json:"smallCacheSize" yaml:"smallCacheSize"` + SmallRetry uint64 `json:"smallRetry,omitempty" yaml:"smallRetry,omitempty"` + LargeCacheCount uint32 `json:"largeCacheCount" yaml:"largeCacheCount"` + LargeCacheSize uint32 `json:"largeCacheSize" yaml:"largeCacheSize"` + LargeRetry uint64 `json:"largeRetry,omitempty" yaml:"largeRetry,omitempty"` +} + +// SpdkBsChannelInfo holds live spdk_bs_channel pool and cluster-allocation serialization state. +type SpdkBsChannelInfo struct { + Device string `json:"device" yaml:"device"` + Address string `json:"address" yaml:"address"` + LvolName string `json:"lvolName,omitempty" yaml:"lvolName,omitempty"` + BlobID string `json:"blobID,omitempty" yaml:"blobID,omitempty"` + TotalReqs int `json:"totalReqs" yaml:"totalReqs"` + FreeReqs int `json:"freeReqs" yaml:"freeReqs"` + NeedClusterAlloc int `json:"needClusterAlloc" yaml:"needClusterAlloc"` + QueuedIO int `json:"queuedIO" yaml:"queuedIO"` + NeedClusterAllocByLvol map[string]int `json:"needClusterAllocByLvol,omitempty" yaml:"needClusterAllocByLvol,omitempty"` + ActiveRequests []*SpdkBsActiveRequest `json:"activeRequests,omitempty" yaml:"activeRequests,omitempty"` +} + +// SpdkBsActiveRequest describes an in-flight spdk_bs_request_set that is neither free nor waiting in need_cluster_alloc. +type SpdkBsActiveRequest struct { + Index int `json:"index" yaml:"index"` + OpType string `json:"opType" yaml:"opType"` + LvolName string `json:"lvolName,omitempty" yaml:"lvolName,omitempty"` + BlobID string `json:"blobID,omitempty" yaml:"blobID,omitempty"` + IOUnit uint64 `json:"ioUnit,omitempty" yaml:"ioUnit,omitempty"` + NewCluster uint64 `json:"newCluster,omitempty" yaml:"newCluster,omitempty"` + BackingDevice string `json:"backingDevice,omitempty" yaml:"backingDevice,omitempty"` + RemoteTargetAddress string `json:"remoteTargetAddress,omitempty" yaml:"remoteTargetAddress,omitempty"` + RemoteTargetNode string `json:"remoteTargetNode,omitempty" yaml:"remoteTargetNode,omitempty"` +} + +// SpdkBdevNoMemInfo describes a spdk_bdev_shared_resource with queued nomem_io requests. +type SpdkBdevNoMemInfo struct { + IOOutstanding uint64 `json:"ioOutstanding" yaml:"ioOutstanding"` + NoMemThreshold uint64 `json:"noMemThreshold" yaml:"noMemThreshold"` + NoMemIOCount int `json:"noMemIOCount" yaml:"noMemIOCount"` + ByBdev []*SpdkBdevNoMemEntry `json:"byBdev,omitempty" yaml:"byBdev,omitempty"` +} + +// SpdkBdevNoMemEntry summarizes queued nomem_io requests for one bdev. +type SpdkBdevNoMemEntry struct { + BdevName string `json:"bdevName" yaml:"bdevName"` + LvolName string `json:"lvolName,omitempty" yaml:"lvolName,omitempty"` + IOType string `json:"ioType,omitempty" yaml:"ioType,omitempty"` + Count int `json:"count" yaml:"count"` + MaxRetries uint16 `json:"maxRetries" yaml:"maxRetries"` +}