From b0271365ee598e26d8a044e4a5cb35040fd28835 Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 6 Aug 2026 19:19:47 +0200 Subject: [PATCH 1/7] docs: add failure domains, multipathing, CLI migration/replication, and limits pages - Failure domains: concept page (placement contract, balance rules, recovery) and CLI operations page (create, tag, expand, remove, immutable membership) - Storage network multipathing: setup as the alternative to a bonded HA network (separate VLANs, --data-nics, client connect, verification) - Volume migration: bare-metal CLI workflow (migrate / connect / continue, 5-minute continue window, batch migration, preconditions) - Asynchronous replication: CLI workflows incl. cross-cluster volume migration via replication-commit, failover and failback - Limits: hard per-node object limits (6000/75/50) and vCPU-dependent limits Co-Authored-By: Claude Fable 5 --- docs/architecture/concepts/failure-domains.md | 105 +++++++++++ .../storage-network-multipathing.md | 109 +++++++++++ .../operations/asynchronous-replication.md | 170 ++++++++++++++++++ .../operations/failure-domains.md | 142 +++++++++++++++ .../operations/volume-migration.md | 133 ++++++++++++++ docs/reference/limits.md | 97 ++++++++++ 6 files changed, 756 insertions(+) create mode 100644 docs/architecture/concepts/failure-domains.md create mode 100644 docs/non-kubernetes/installation/storage-network-multipathing.md create mode 100644 docs/non-kubernetes/operations/asynchronous-replication.md create mode 100644 docs/non-kubernetes/operations/failure-domains.md create mode 100644 docs/non-kubernetes/operations/volume-migration.md create mode 100644 docs/reference/limits.md diff --git a/docs/architecture/concepts/failure-domains.md b/docs/architecture/concepts/failure-domains.md new file mode 100644 index 00000000..6551fad9 --- /dev/null +++ b/docs/architecture/concepts/failure-domains.md @@ -0,0 +1,105 @@ +--- +title: "Failure Domains" +description: "How simplyblock failure domains group storage nodes by rack, cabinet, or availability zone and constrain data, journal, and failover-path placement." +weight: 30750 +--- + +A failure domain groups storage nodes that share a common infrastructure dependency — a rack, a cabinet, a power +distribution unit, or an availability zone. When failure domains are enabled, simplyblock spreads data chunks, +journal copies, and failover paths across the domains so that the loss of one entire domain does not interrupt +the availability of the cluster. + +Failure domains are identified by a non-negative integer chosen by the operator. Simplyblock does not detect the +physical topology itself: every storage node is explicitly tagged with the id of the domain it belongs to when it +is added to the cluster. + +!!! important + Failure-domain support is a deploy-time decision. It is enabled when the storage cluster is created and cannot + be switched on or off for an existing cluster. To gain the feature, a cluster must be redeployed. + +## What Failure Domains Protect + +With failure domains enabled, placement decisions consider the domain tag in four independent dimensions: + +1. **Data and parity chunks**: The distributed erasure coding spreads the chunks of each stripe across distinct + failure domains, so that a full domain outage leaves enough chunks to reconstruct all data within the configured + erasure coding scheme. +2. **Journal copies**: The copies of the high-availability write journal are balanced across domains with a + per-domain cap, so that losing a whole domain always leaves enough journal copies to maintain the journal quorum. +3. **Failover paths**: The secondary (and, with two parity chunks, tertiary) failover nodes of each logical volume + are placed in different failure domains than the primary node wherever possible. +4. **Cluster status**: The health assessment of the cluster understands domains. Any combination of node and + device outages confined to a single failure domain keeps the cluster serving I/O in a degraded state instead + of suspending it. + +## The Placement Contract + +Placement constraints are applied in a fixed priority order: distinct hosts are a hard requirement, distinct +failure domains are enforced next, and distinct physical labels are considered last. + +For the failover paths, the guaranteed invariant is: + +> Every logical volume store keeps **at least one failover path in a different failure domain** than its +> primary node. + +With two failure domains and three paths (primary, secondary, tertiary), it is mathematically impossible to place +all three paths in distinct domains. Simplyblock therefore guarantees at least one cross-domain failover path per +volume store — enough to survive a full domain outage — and places the remaining paths cross-domain wherever the +topology allows it. + +At cluster activation, simplyblock arranges the hosts in a round-robin order across the failure domains and derives +all secondary and tertiary assignments from this interleaved rotation. On a cluster with equally sized domains, this +construction makes every secondary path cross-domain by design. + +## Balance Rules + +Failure-domain placement only works if the domains stay comparable in size. Simplyblock enforces this: + +- At **activation**, all failure domains must contain an **equal number of hosts**, and at least two domains must + exist. +- During **operation**, the host count per domain may never diverge by more than **one host** (±1 rule). Adding or + removing a node is refused if it would unbalance the domains further. +- Every domain must keep at least **two hosts** once the cluster holds data. + +A cluster with a one-host imbalance stays fully within the availability contract: exactly one volume store then has +a same-domain secondary path, and its tertiary path is still guaranteed to be cross-domain. + +!!! note + Balance is counted in physical hosts, not storage nodes. On multi-socket hosts running two storage nodes, both + nodes count as one host and must carry the same failure-domain id. Dedicated secondary nodes are not counted + towards the balance. + +## Failure Domains and Erasure Coding Schemes + +The number of failure domains should match the data protection goal: + +| Goal | Recommendation | +|------|----------------| +| Survive one full domain outage | At least `parity chunks + 1` distinct failure domains | +| Survive one full domain outage plus one further node or drive failure elsewhere | Erasure coding scheme with two parity chunks (e.g., `1+2`, `2+2`) and at least as many domains as data chunks | + +The high-availability journal requires at least **four** journal copies on failure-domain clusters (instead of +three), even with a single parity chunk. With three copies and two domains, one domain would hold two copies and +its loss would break the journal quorum. + +## Domain Membership Is Immutable + +A host's failure domain cannot be changed while the host is part of the cluster. Moving a host between domains +requires removing the node, restoring the domain balance, and re-adding it with the new failure-domain id. This +prevents accidental topology changes that would silently invalidate the placement of existing data. + +## Recovery Behavior + +Failure domains also change how the cluster recovers from large outages: + +- An outage confined to one domain — up to and including every node of the domain — keeps the cluster **degraded + but serving**. The cluster is not suspended. +- With two parity chunks, the cluster additionally tolerates the loss of one entire domain **plus** one further + node or device outage in exactly one other domain. +- When a whole domain returns from an outage (for example, after a rack power loss), its nodes are restarted **in + parallel** instead of strictly one-by-one, substantially shortening the recovery of large domains. + +For operating instructions — creating a failure-domain cluster, adding and removing nodes, and expansion rules — +see [Managing Failure Domains](../../non-kubernetes/operations/failure-domains.md). For Kubernetes-based +deployments, failure domains are assigned through the Simplyblock Operator; see the +[Operator Reference](../../reference/operator/index.md). diff --git a/docs/non-kubernetes/installation/storage-network-multipathing.md b/docs/non-kubernetes/installation/storage-network-multipathing.md new file mode 100644 index 00000000..85b5ebef --- /dev/null +++ b/docs/non-kubernetes/installation/storage-network-multipathing.md @@ -0,0 +1,109 @@ +--- +title: "Storage Network Multipathing" +description: "Configure NVMe-oF multipathing over two independent storage networks as an alternative to a bonded, highly available network." +weight: 36000 +--- + +Simplyblock supports two ways to make the storage network redundant: + +- A **redundant network** below a single interface, built with link aggregation (LACP), stacked switches, MLAG, or + active/passive bonding. Simplyblock sees one data interface; the redundancy is handled entirely in the network + layer. +- **NVMe-oF multipathing** over two (or more) independent storage networks. Each storage node is attached with + multiple data interfaces in separate VLANs or subnets, routed over separate NIC ports and switches. Simplyblock + exposes every NVMe-oF subsystem on all data interfaces, and the NVMe hosts and cluster-internal connections use + native NVMe multipathing across them. + +Multipathing places the redundancy in the NVMe layer instead of in the network layer (L2). It requires no switch +support for link aggregation, keeps the two paths physically independent end-to-end, and also spreads the I/O load +across both networks. + +## Network Requirements + +Each data interface of a host must be in its **own VLAN or subnet**, connected through separate NIC ports and +switch paths. + +!!! warning + Placing two data interfaces into the same subnet does not provide independent paths. Linux routes all outbound + traffic of a subnet through one interface (the one with the lowest metric), so both "paths" collapse onto a + single NIC. Use separate subnets per data interface, or set up policy routing. + +A typical layout separates management and storage traffic completely: + +| Network interface | Purpose | Subnet (example) | +|-------------------|---------------------|------------------| +| eth0 | Management / control plane | 192.168.10.0/24 | +| eth1 | Storage path A | 10.10.10.0/24 | +| eth2 | Storage path B | 10.10.20.0/24 | + +The management network should still be highly available (a simple bond is sufficient), but it does not carry +storage traffic. + +## Configuring Storage Nodes + +The data interfaces of a storage node are declared when the node is attached to the cluster, using the +`--data-nics` parameter of `storage-node add-node`. Multiple interfaces are given as a comma-separated list: + +```bash title="Attach a storage node with two data interfaces" +{{ cliname }} storage-node add-node \ + --data-nics eth1,eth2 +``` + +!!! note + The interface list is comma-separated without spaces (`eth1,eth2`). If `--data-nics` is omitted, the + management interface carries the storage traffic and no multipathing is available. + +There is no separate switch to enable multipathing: as soon as a node has more than one usable data interface, +all of its NVMe-oF subsystems — logical volumes as well as cluster-internal device and journal subsystems — listen +on every data interface, and all connections to the node are established once per interface. + +Multipathing applies per node, but a consistent configuration across all nodes is strongly recommended: use the +same number of data interfaces, in the same set of VLANs, on every storage node. + +## Client Connections + +With multipathing, `{{ cliname }} volume connect` returns one `nvme connect` command per combination of node and +data interface. A volume with one failover path (erasure coding with one parity chunk) on nodes with two data +interfaces yields **four** connection strings; with two failover paths (two parity chunks), **six**: + +```bash title="Retrieve all connection strings for a volume" +{{ cliname }} volume connect +``` + +Run **all** returned `nvme connect` commands on the host. The commands connect the same NVMe subsystem (the same +NQN) over the different paths; the Linux kernel's native NVMe multipathing merges them into a single block device +and steers I/O based on the ANA (Asymmetric Namespace Access) state that simplyblock manages per path. No +`dm-multipath` configuration is required or supported. + +If a path fails — a NIC, a switch, or an entire network — the kernel transparently continues on the remaining +paths. When a primary node fails over to a secondary node, simplyblock switches the ANA states, and the host +follows without a reconnect. + +## Verifying the Configuration + +After attaching the nodes, verify that all paths exist: + +1. `{{ cliname }} storage-node list --json` — every node reports all of its data interfaces (`data_nics`). +2. `{{ cliname }} storage-node port-list ` — lists the data interfaces of a node. +3. `{{ cliname }} volume connect ` — returns one connection string per node and interface (for + example, four entries for a volume with one failover path on dual-interface nodes). +4. `{{ cliname }} storage-node check ` — verifies all NVMe-oF connections to and from the node, + including all paths of the cluster-internal connections. + +Per-interface I/O statistics are available with `{{ cliname }} storage-node port-io-stats `. + +## Kubernetes Deployments + +In Kubernetes-based deployments, the data interfaces are declared in the `StorageNodeSet` resource: the +`dataIfname` field takes a list of interface names, equivalent to `--data-nics`. Volume connections made by the +CSI driver automatically use all paths; no storage-class parameter is required. See the +[Operator Reference](../../reference/operator/index.md) for details. + +## Interaction with Failure Domains and Migration + +- Multipathing and [failure domains](../../architecture/concepts/failure-domains.md) are independent features + that combine naturally: failure domains protect against the loss of a rack or site, multipathing against the + loss of a network path. +- During a [volume migration](../operations/volume-migration.md), the target subsystem is exposed on all data + interfaces of the target node. The client must connect all returned target paths before continuing the + migration, so that the cutover is seamless on every path. diff --git a/docs/non-kubernetes/operations/asynchronous-replication.md b/docs/non-kubernetes/operations/asynchronous-replication.md new file mode 100644 index 00000000..be041f2c --- /dev/null +++ b/docs/non-kubernetes/operations/asynchronous-replication.md @@ -0,0 +1,170 @@ +--- +title: "Asynchronous Replication" +description: "Asynchronous replication between simplyblock clusters: disaster recovery, cross-cluster volume migration, failover and failback." +weight: 20045 +--- + +Simplyblock replicates volumes between two storage clusters by taking copy-on-write snapshots at regular intervals +and transferring them to the target cluster. On top of this mechanism, two workflows are available per volume: + +- **Disaster recovery** (`failover` mode): the target cluster holds a continuously updated copy. If the source + cluster is lost, the volume is materialized on the target and clients reconnect there. +- **Cross-cluster volume migration** (`migration` mode): a planned, online move of a volume to another cluster. + The remaining delta is transferred under a brief I/O freeze, and the client fails over to the target paths + without a disconnect. + +For the architecture background, see [Replication Concepts](../../architecture/concepts/replication.md). For +Kubernetes environments, where replication is managed through the `SnapshotReplication` resource, see +[Asynchronous Replication on Kubernetes](../../kubernetes/operations/asynchronous-replication.md). + +## Prerequisites + +- **Both clusters are managed by the same control plane.** The first cluster is created with + `{{ cliname }} cluster create`; the second is attached to the same control plane with `{{ cliname }} cluster add`. +- The storage nodes of the source cluster can reach the storage nodes of the target cluster over the storage + network: replication transfers data directly between the nodes over NVMe-oF. +- Both clusters are activated, and both have an active storage pool. + +!!! note + For multi-site setups, distribute the management nodes of the control plane across the sites so that the + control plane survives a site failure. + +## Configuring the Replication Target + +Replication is configured per source cluster and points at one target cluster and pool: + +```bash title="Assign the replication target cluster" +{{ cliname }} cluster add-replication \ + --target-pool [--timeout ] +``` + +If `--target-pool` is omitted, the first active pool of the target cluster is used. For bidirectional protection, +run the command once in each direction. + +## Enabling Replication on a Volume + +```bash title="Start replication for a volume" +{{ cliname }} volume replication-start \ + [--replication-cluster-id ] \ + [--mode failover|migration] \ + [--interval-min ] +``` + +- `--mode failover` (default): asynchronous disaster recovery. The target volume is only materialized on + failover. +- `--mode migration`: planned cutover. The target subsystem is pre-created up front (inaccessible), and the + volume is cut over on an explicit commit. +- `--interval-min `: take an internal replication snapshot every `N` minutes (the first one immediately). + `0` disables interval snapshots; then only user-created snapshots replicate. + +Every snapshot of a replicated volume — interval-based or user-created — is queued for transfer to the target +cluster. Snapshots that were taken before replication was enabled are transferred as well. The achievable recovery +point (RPO) is roughly the snapshot interval plus the transfer time. + +A volume can also opt into replication at creation time with `{{ cliname }} volume add ... --replicate`, using +the cluster's configured replication target. Mode and interval are then set with a subsequent +`replication-start`. + +To take an immediate replication snapshot outside the interval: + +```bash title="Trigger an immediate replication snapshot" +{{ cliname }} volume replication-trigger +``` + +## Monitoring + +```bash title="Replication progress of one volume" +{{ cliname }} volume replication-info +``` + +The output shows the last snapshot, the last completed replication and its duration, the number of replicated +snapshots, the **time lag** (the age of the newest point-in-time that exists on the target — the actual RPO), and +the outstanding backlog (count and bytes of not-yet-replicated snapshots). A volume is caught up when the +outstanding count is zero. + +```bash title="All replication tasks of a cluster" +{{ cliname }} volume replication-status +``` + +## Cross-Cluster Volume Migration + +A planned, online migration of a volume to another cluster combines `migration` mode with an explicit commit: + +```bash title="Step 1: Replicate the volume in migration mode" +{{ cliname }} volume replication-start \ + --replication-cluster-id \ + --mode migration --interval-min 1 +``` + +Wait until `{{ cliname }} volume replication-info ` reports an outstanding count of zero, then: + +```bash title="Step 2: Commit the cutover" +{{ cliname }} volume replication-commit +``` + +The commit takes a final snapshot to minimize the delta, builds the target volume on the last replicated +snapshot — with the **same NQN and namespace ID** as the source — exposes it as inaccessible, and queues the final +cutover task. The cutover freezes source I/O, transfers the residual delta, and flips the ANA states so that the +client fails over to the target paths without a disconnect. + +!!! important + For an interruption-free cutover, the client must already hold the target paths when the ANA states flip. + Retrieve the connection strings of the target volume right after `replication-commit` and run the + `nvme connect` commands on the client before the cutover task completes. Because source and target expose the + same NQN and namespace ID, the new paths join the existing multipath device. + +Since continuous replication keeps the backlog small, the final freeze only covers the residual delta — typically +a fraction of a second to a few seconds. + +## Failover (Disaster Recovery) + +If the source cluster is lost, a replicated volume is materialized on the target cluster from the last +successfully replicated snapshot. This is currently exposed through the management API: + +```bash title="Fail a volume over to the target cluster" +curl -X POST -H "Authorization: " \ + https:///api/v2/clusters//storage-pools//volumes//replicate_lvol +``` + +The response contains the volume's NQN, namespace ID, and the connection strings on the target cluster. The NQN +and namespace ID are identical to the source volume, so clients reconnect to the returned addresses and continue +with the same device identity. + +!!! warning + Data written on the source after the last successfully replicated snapshot is not available on the target. + The data gap is at most the replication interval plus the replication lag; check + `volume replication-info` to see the effective lag. Unlike the planned cutover, a failover interrupts I/O: + workloads must reconnect (and typically restart) against the target paths. + +## Failback + +After a failover, the volume can be moved back to a source cluster: + +```bash title="Configure the failback" +{{ cliname }} volume replication-failback [--source-cluster-id ] +``` + +- **Recovered original source** (omit `--source-cluster-id`): snapshots that already exist on the original source + are recognized, and only the delta written since the failover is replicated back. +- **Fresh source cluster** (pass a different cluster id): the full volume is replicated to the new cluster. + +Failback uses the same mechanism as cross-cluster migration: once the backlog is caught up, complete the failback +with `{{ cliname }} volume replication-commit `. The same client-connect rule applies for an +interruption-free cutback. + +## Stopping Replication and Cleaning Up + +```bash title="Stop replication for a volume" +{{ cliname }} volume replication-stop +``` + +Stopping cancels the pending replication tasks of the volume and disables further snapshots from replicating. The +already replicated snapshots on the target are kept; they can be removed individually without touching the source +snapshot: + +```bash title="Delete only the replicated copy of a snapshot" +{{ cliname }} snapshot delete-replication-only +``` + +The replication status of individual snapshots is available with +`{{ cliname }} snapshot replication-status ` and `{{ cliname }} snapshot list --with-details`. diff --git a/docs/non-kubernetes/operations/failure-domains.md b/docs/non-kubernetes/operations/failure-domains.md new file mode 100644 index 00000000..0352f433 --- /dev/null +++ b/docs/non-kubernetes/operations/failure-domains.md @@ -0,0 +1,142 @@ +--- +title: "Managing Failure Domains" +description: "Deploy and operate a simplyblock storage cluster with failure domains: cluster creation, node tagging, balance rules, expansion, and node removal." +weight: 20055 +--- + +Failure domains group storage nodes by shared infrastructure (rack, cabinet, power unit, availability zone) so that +simplyblock can spread data, journal copies, and failover paths across independent fault groups. The concept and +the placement guarantees are described in +[Failure Domains](../../architecture/concepts/failure-domains.md). + +This page describes how to deploy and operate a failure-domain cluster with the CLI. In Kubernetes environments, +failure domains are assigned declaratively through the Simplyblock Operator +(`enableFailureDomains` on the `StorageCluster` and `failureDomain` per node); see the +[Operator Reference](../../reference/operator/index.md). + +## Enabling Failure Domains + +Failure-domain support is enabled when the storage cluster is created and is immutable afterwards: + +```bash title="Create a cluster with failure-domain support" +{{ cliname }} cluster create --enable-failure-domain +``` + +The same flag exists on `{{ cliname }} cluster add` when attaching an additional cluster to an existing control +plane. + +!!! warning + A cluster cannot be upgraded into failure-domain mode. If `--enable-failure-domain` was not given at creation + time, the cluster must be redeployed to use failure domains. + +## Tagging Storage Nodes + +On a failure-domain cluster, every storage node must be added with a failure-domain id — a non-negative integer +identifying the rack, cabinet, or availability zone. All nodes in the same physical fault group share the same id. + +```bash title="Add storage nodes with failure-domain tags" +# Rack A (domain 0) +{{ cliname }} storage-node add-node --failure-domain 0 + +# Rack B (domain 1) +{{ cliname }} storage-node add-node --failure-domain 1 +``` + +The tag is mandatory on failure-domain clusters and must be omitted on clusters without the feature. Both +mismatches are rejected with an explanatory error. + +All storage nodes on the same physical host must carry the same failure-domain id. On multi-socket hosts with two +storage nodes, both nodes belong to the host's domain. + +The assigned domains are shown in the node list once at least one node carries a tag: + +```bash title="List storage nodes with their failure domains" +{{ cliname }} storage-node list +``` + +## Activation Requirements + +Activating a freshly assembled failure-domain cluster enforces the following rules: + +| Rule | Enforcement | +|------|-------------| +| Every node carries a failure-domain id | Hard — activation fails | +| A host does not span two domains | Hard — activation fails | +| At least two distinct domains exist | Hard — activation fails | +| All domains hold an equal number of hosts | Hard — activation fails | +| At least `parity chunks + 1` distinct domains | Recommendation — a warning is logged, activation proceeds with best-effort placement | + +During activation, simplyblock computes the interleaved host rotation across the domains and assigns all secondary +and tertiary failover paths from it. Re-activation of an existing cluster (for example, during disaster recovery) +deliberately skips these gates: recovery always takes precedence over topology policy. + +## Journal Copies + +Failure-domain clusters require at least four copies of the high-availability journal, even with a single parity +chunk. The default of `--ha-jm-count` is 3 for single-parity clusters, so it must be raised explicitly: + +```bash title="Add a node with four journal copies" +{{ cliname }} storage-node add-node \ + --failure-domain 0 --ha-jm-count 4 +``` + +With three copies and two domains, one domain would hold two copies, and losing that domain would break the +journal quorum. + +## Balance Rules During Operation + +Once the cluster holds data, topology changes are admitted only if the failure domains stay balanced: + +- The host count per domain may never diverge by more than one (±1 rule). On a balanced cluster, one host can be + added to any domain; the next host must then go to a different domain. +- No domain may drop below two hosts. +- Adding another storage node slot on an already-member host (multi-socket systems) is balance-neutral and always + admitted, as long as the host keeps its original domain id. + +Violating additions and removals are refused up front, before any data is moved. + +## Expanding a Failure-Domain Cluster + +Single-node expansion integrates a new node into the cluster by re-homing existing secondary and tertiary +failover paths: + +```bash title="Expand the cluster by one node" +{{ cliname }} storage-node add-node \ + --failure-domain --expansion +``` + +On failure-domain clusters, the expansion planner inserts the newcomer into the existing host rotation at a +position that preserves the cross-domain failover invariant. If no valid position exists, the expansion is refused +before any change is made. + +!!! important + On clusters with a single parity chunk (FTT 1), an odd total host count cannot satisfy the cross-domain + invariant, because there is no tertiary path to fall back on. Grow such clusters in pairs — one host per + domain at a time. + +## Removing a Storage Node + +Node removal applies the same balance rules (±1, minimum two hosts per domain). In addition, the failover paths +hosted by the node being removed are relocated to other nodes. If the path being relocated is the only +cross-domain path of its volume store, the replacement node **must** be in a different failure domain than the +primary; if no such node exists, the removal is refused. + +## Moving a Host Between Domains + +A host's failure domain is immutable. Re-adding a host or one of its node slots with a different domain id is +rejected. To move a host: + +1. Remove the node with `{{ cliname }} storage-node remove`. +2. Restore the domain balance if necessary. +3. Re-add the node with the new `--failure-domain` id. + +## Behavior During Outages + +- Node and device outages confined to one failure domain keep the cluster **degraded but serving**, regardless of + how many nodes of that domain are down. +- With two parity chunks, the cluster also tolerates a full domain outage plus one additional node or device + outage in exactly one other domain. +- Anything broader suspends the cluster until enough nodes return. +- When an entire domain returns (for example, after a rack power cycle), its nodes are restarted in parallel. + Parallel recovery stops automatically as soon as any node of that domain is online again or a node in another + domain starts restarting. diff --git a/docs/non-kubernetes/operations/volume-migration.md b/docs/non-kubernetes/operations/volume-migration.md new file mode 100644 index 00000000..09f32dd0 --- /dev/null +++ b/docs/non-kubernetes/operations/volume-migration.md @@ -0,0 +1,133 @@ +--- +title: "Volume Migration" +description: "Migrate a logical volume between storage nodes with the simplyblock CLI: pre-create, client connect, continue, monitor, and cancel." +weight: 20040 +--- + +Simplyblock can move a logical volume — including its snapshots — from one storage node to another while the +volume stays online. I/O is only frozen for the brief moment needed to transfer the final delta at the end of the +migration. + +This page describes the CLI-driven migration between nodes of the **same cluster**. For moving volumes between +**clusters**, see [Asynchronous Replication](asynchronous-replication.md), which provides a +replication-based cross-cluster migration. In Kubernetes environments, migrations are managed declaratively +through the `VolumeMigration` resource; see +[Volume Migration on Kubernetes](../../kubernetes/operations/volume-migration.md). + +## How a Migration Works + +A migration is a two-step operation with a client action in between: + +1. `volume migrate` **pre-creates** the target: the NVMe-oF subsystem for the volume is created on the target + node with the same NQN as on the source, with all listeners in the ANA state `inaccessible`. The command + returns a migration ID and the NVMe connect strings for the new target paths. +2. The operator runs the returned `nvme connect` commands **on the client**. The new paths join the client's + native NVMe multipath for the volume; because they are `inaccessible`, they carry no I/O yet. +3. `volume migrate-continue` starts the data transfer. The snapshot chain is copied oldest-first, the live delta + is progressively shrunk with intermediate snapshots, and the final delta is transferred under a short I/O + freeze. At cutover, the ANA states flip: the target paths become active and the source paths become + inaccessible. The client follows automatically, without a disconnect. + +!!! warning + A pre-created migration must be continued within **five minutes**. If `migrate-continue` is not run in time, + the migration is automatically cancelled and the target resources are released. + +## Starting a Migration + +```bash title="Step 1: Pre-create the migration" +{{ cliname }} volume migrate +``` + +The output contains the migration ID and one connect command per target path (one per data interface of the +target node): + +```bash title="Step 2: Connect the client to the target paths" +# Run the connect commands returned by 'volume migrate' on the client host. +sudo nvme connect --transport=tcp --traddr= --trsvcid= --nqn= ... +``` + +!!! important + On nodes with multiple data interfaces, connect **all** returned target paths. A path that is not connected + is simply unused after the cutover, reducing the redundancy of the volume. + +```bash title="Step 3: Start the data transfer" +{{ cliname }} volume migrate-continue +``` + +`migrate-continue` accepts `--max-retries ` (default 10) and `--deadline ` (default 14400; `0` +disables the deadline). + +If the volume has host authentication configured (DH-HMAC-CHAP), pass the client's host NQN to the pre-create +step with `--host-nqn `. + +## Monitoring + +```bash title="List migrations" +{{ cliname }} volume migrate-list [--cluster-id ] [--json] +``` + +The list shows source and target node, the current phase, status, snapshot progress (migrated/planned), the retry +counter, and the last error, if any. + +| Phase | Meaning | +|-------|---------| +| `pre_created` | Target subsystem exists, waiting for `migrate-continue`. | +| `snap_copy` | The snapshot chain is being copied to the target. | +| `lvol_migrate` | The final delta is being transferred. This is the only phase with a (short) I/O freeze. | +| `cleanup_source` | Data has moved; source-side objects are being removed. | +| `cleanup_target` | Rollback after a failure or cancellation: target-side objects are being removed. | +| `completed` | The migration has finished. | + +A volume that is part of an active migration shows the migration ID in the `migrating` field of +`{{ cliname }} volume get `. + +## Cancelling a Migration + +```bash title="Cancel a migration" +{{ cliname }} volume migrate-cancel +``` + +A migration cancelled in the `pre_created` phase is torn down immediately. In later phases, the cancellation is +picked up asynchronously by the migration runner, which rolls the target back (`cleanup_target`); it may take a +few seconds to reflect in `migrate-list`. Data on the source remains intact and authoritative until the final +cutover, so a migration can be cancelled at any phase before `cleanup_source`. + +## Migrating Shared Subsystems (Batch Migration) + +Volumes that share one NVMe-oF subsystem (namespaced volumes) can only be migrated together. Pass `--batch` with +any member volume; simplyblock migrates all volumes of the subsystem as one coordinated group and returns a +migration group ID, which is then used with `--batch` on the other commands: + +```bash title="Migrate all volumes of a shared subsystem" +{{ cliname }} volume migrate --batch +{{ cliname }} volume migrate-continue --batch +{{ cliname }} volume migrate-group-list [--cluster-id ] +{{ cliname }} volume migrate-cancel --batch +``` + +Attempting to migrate a single member of a shared subsystem without `--batch` is rejected. + +## Preconditions and Constraints + +A migration is admitted only if: + +- The cluster is active and not currently rebalancing (no device migration or post-restart rebalancing tasks are + running). +- The volume is online; the target node is online and different from the source node; the source node is online + or suspended. +- The volume has no other active migration. Re-running `volume migrate` with the same volume and target returns + the existing migration ID; a different target requires cancelling the existing migration first. + +Additional operational constraints while a migration is active: + +- **Snapshots of volumes on the source node cannot be created** until the migration completes. +- New volumes cannot be attached to a subsystem that has an active migration. +- The erasure coding scheme of the volume is preserved; it is not re-negotiated on the target. +- Simplyblock does not pre-check the free capacity of the target node. Ensure the target has enough capacity for + the volume and its snapshots before starting the migration. + +## Draining a Node + +Volume migration is the building block for emptying a node before removing it from the cluster +(`{{ cliname }} storage-node remove` requires the node to host no volumes or snapshots). Migrate all volumes off +the node first, then remove it. diff --git a/docs/reference/limits.md b/docs/reference/limits.md new file mode 100644 index 00000000..20da8cf1 --- /dev/null +++ b/docs/reference/limits.md @@ -0,0 +1,97 @@ +--- +title: "Limits" +description: "Hard object limits and vCPU-dependent resource limits of simplyblock storage nodes and clusters: subsystems, namespaces, objects per node, and sizing rules." +weight: 20140 +--- + +Simplyblock enforces a set of limits per storage node and per cluster. Some are hard limits built into the +control plane; others depend on the node's vCPU count and memory configuration. + +## Hard Per-Node Object Limits + +| Limit | Value | What it counts | +|-------|------:|----------------| +| Objects per node | 6000 | Logical volumes, clones, and snapshots owned by the node (its logical volume store) | +| NVMe-oF subsystems per node | 75 | Subsystems for which the node is the primary; namespaced volumes sharing one subsystem count as one | +| Namespaces per subsystem | 50 | Volumes (namespaces) sharing one NVMe-oF subsystem | + +These limits are enforced on every create path (volume create, snapshot create, clone). When a limit is reached, +the operation fails with an explanatory error, for example: + +```plain title="Limit errors" +Object limit reached on lvstore of node : 6000 objects (lvols/clones: 4100, snapshots: 1900); the hard limit is 6000 per lvstore +Too many subsystems on node: , max subsystems reached: 75 +max_namespace_per_subsys=64 exceeds the hard limit of 50 namespaces per subsystem +``` + +Notes on what counts against the limits: + +- Only the **primary** node of a volume is charged. Failover copies on secondary and tertiary nodes do not count + against those nodes' limits — their resource reservation already provisions for them. +- Deleted objects do not count; objects in creation or deletion still do. +- When volume placement finds no node below its subsystem limit, volume creation fails with + `No nodes found with enough resources to create the LVol`. + +## Configured Subsystem Limit per Node + +The 75-subsystem ceiling applies on top of the per-node configured maximum, set at host configuration time: + +```bash title="Configure the maximum number of subsystems per node" +{{ cliname }} storage-node configure --max-subsys +``` + +The effective subsystem limit of a node is the **smaller** of `--max-subsys` and 75. The configured value also +drives the node's memory reservation (huge pages), so it should reflect the actually planned number of volumes. +It can be changed later via `{{ cliname }} storage-node restart --max-subsys `. + +## Namespaces per Subsystem + +By default, simplyblock places each volume in its own NVMe-oF subsystem. Namespaced volumes share a subsystem; +the default maximum is **32 namespaces per subsystem**, configurable per volume at creation time up to the hard +ceiling of 50: + +```bash title="Create a namespaced volume with a custom namespace limit" +{{ cliname }} volume add ... --max-namespace-per-subsys # N ≤ 50 +``` + +When a shared subsystem is full, the next volume automatically starts a new subsystem (which then counts against +the node's subsystem limit). + +## vCPU-Dependent Limits + +On top of the hard object limits, several resource limits scale with the vCPU count of the storage node: + +| Limit | Rule | +|-------|------| +| CPU cores per storage node | At most 64 cores can be assigned to one storage node (SPDK instance). | +| Distribution services per node | Scales with the assigned cores, capped at 12. | +| NVMe-oF buffer pools | Scale with core count and `--max-subsys`; they determine part of the huge-page demand. | +| Huge-page memory | The minimum huge-page memory grows with the core count and the configured maximum number of subsystems. Nodes refuse to start with insufficient huge pages. | +| Storage nodes per host | 1 or 2 (`--nodes-per-socket`), aligned to NUMA sockets. | + +In practice, the **memory** derived from vCPU count and `--max-subsys` is the sizing driver: see +[Hardware Requirements](../deployment-preparation/hardware-requirements.md) for the RAM formula per subsystem. + +## Cluster-Level Limits and Gates + +| Limit | Default | Description | +|-------|--------:|-------------| +| Fault tolerance (FTT) | 1 | 1 or 2, derived from the parity chunks of the erasure coding scheme. | +| Minimum online devices at activation | — | Data chunks + parity chunks + 1. | +| Minimum online nodes for volume creation | — | At least data chunks + parity chunks online nodes. | +| Journal copies (`--ha-jm-count`) | 3 (FTT 1) / 4 (FTT 2) | Failure-domain clusters require 4 even at FTT 1. | +| Minimum volume size | 100 MiB | Smaller volumes are rejected. | +| Provisioning warning (`--prov-cap-warn`) | 250 % | Warning when total provisioned capacity exceeds this ratio of the cluster capacity. | +| Provisioning limit (`--prov-cap-crit`) | 500 % | Volume creation fails beyond this over-provisioning ratio. | +| Utilization warning / critical (`--cap-warn` / `--cap-crit`) | 89 % / 99 % | Alerts on used physical capacity. | +| Storage pool caps (`--pool-max`, `--lvol-max`) | unlimited | Optional per-pool caps for total provisioned size and per-volume size. | +| NVMe/TCP qpairs per volume (`--qpair-count`) | 32 | Cluster-internal queue pair count per volume connection. | +| Client qpairs (`--client-qpair-count`) | 3 | Queue pairs per client connection. | + +There is no built-in limit on the number of storage nodes per cluster, clusters per control plane, or storage +pools per cluster. + +!!! note + The hard per-node object limits protect the storage node from memory and metadata overload. They are not + configurable at runtime. If a workload legitimately needs more objects, distribute it across more storage + nodes or clusters. From 614064c820f6363e974c1bafff70a4fc2ea9508d Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 6 Aug 2026 19:20:06 +0200 Subject: [PATCH 2/7] docs: fix stale CLI syntax, defaults, and procedures against current sbcli main - --max-lvol renamed to --max-subsys (storage-node configure/restart) - --journal-partition deprecated: document --enable-journal-device instead - --data-nics: multiple comma-separated data NICs supported (multipathing); drop the 'R25.10, zero or one NICs' claim - --crypto-key1/2 removed (KMS-based since 26.2): rewrite encrypting page, fix provisioning parameter table (defaults: --max-size 1000T, --ha-type default; host ACLs moved to storage-pool add-host) - QoS examples: remove copied metavars (invalid syntax) - Node removal: reflect online node removal semantics (no volumes/snapshots, node must be ONLINE/SUSPENDED, async task; --force-remove only cancels tasks) - Cluster expansion: single-node expansion supported, drop 'pairs only' - find-secondary-node: mention tertiary path at FTT 2 - reconnect-nvme-device: lvol listener ports start at 4420, not 9100 - network-port-table: port 5000 is the storage node API, not spdk-http-proxy - cluster-deployment-options: --qpair-count default is 32 (client 3); add --enable-failure-domain - parallel-node-addition: maxParallelNodeAdds lives on StorageNodeSet - nvmf-security: storage-pool add requires cluster id positional - 26-2 release notes: failure domains are flat integer domains, not hierarchical - HA page: add failure domains and storage-network multipathing sections Co-Authored-By: Claude Fable 5 --- docs/architecture/concepts/nvmf-security.md | 2 +- .../high-availability-fault-tolerance.md | 11 +++++ .../cluster-deployment-options.md | 19 ++++++--- .../hardware-requirements.md | 5 ++- docs/kubernetes/installation/talos.md | 4 +- .../operations/parallel-node-addition.md | 2 +- .../scaling/expanding-storage-cluster.md | 8 ++-- docs/non-kubernetes/installation/index.md | 3 +- .../non-kubernetes/installation/install-sp.md | 34 ++++++++++------ .../operations/backup-recovery.md | 40 +++++++++++++------ .../operations/find-secondary-node.md | 8 ++-- .../operations/migrating-storage-node.md | 2 +- .../operations/reconnect-nvme-device.md | 4 +- .../operations/replacing-storage-node.md | 10 +++-- docs/non-kubernetes/usage/encrypting.md | 33 ++++++--------- docs/non-kubernetes/usage/provisioning.md | 30 ++++++++------ .../usage/quality-of-service.md | 20 +++++----- .../troubleshooting/storage-plane.md | 13 +++--- docs/release-notes/26-2.md | 2 +- snippets/network-port-table.md | 2 +- 20 files changed, 149 insertions(+), 103 deletions(-) diff --git a/docs/architecture/concepts/nvmf-security.md b/docs/architecture/concepts/nvmf-security.md index 19835346..5c76196b 100755 --- a/docs/architecture/concepts/nvmf-security.md +++ b/docs/architecture/concepts/nvmf-security.md @@ -68,7 +68,7 @@ enabled: - Host access control (`allow_any_host` is set to `false` for all volumes in the pool) ```bash title="Create Pool with DH-HMAC-CHAP Authentication" -{{ cliname }} storage-pool add --dhchap +{{ cliname }} storage-pool add --dhchap ``` ## Host Management diff --git a/docs/architecture/high-availability-fault-tolerance.md b/docs/architecture/high-availability-fault-tolerance.md index fb213a11..c6f98a07 100644 --- a/docs/architecture/high-availability-fault-tolerance.md +++ b/docs/architecture/high-availability-fault-tolerance.md @@ -37,6 +37,11 @@ storage nodes along with parity fragments. This provides: outage and a concurrent node outage on another rack or drive failures on any other node with an storage overhead of just 50%. +Clusters can be deployed with [failure domains](concepts/failure-domains.md): storage nodes are tagged with the +rack, cabinet, or availability zone they belong to, and simplyblock spreads data chunks, journal copies, and +failover paths across the domains. A failure-domain cluster keeps serving I/O through the outage of one entire +domain (and, with two parity chunks, one additional node or drive failure in another domain). + ### 2. Multipathing with Primary and Secondary Nodes Simplyblock supports NVMe over Fabrics (NVMe-oF) multipathing to provide path redundancy between clients and @@ -56,6 +61,12 @@ with a parity level of _2_ (`1+2`, `2+2`, `4+2`) have two secondary paths. The number of secondary paths defines how many storage nodes can be lost at the same time without impacting the availability of the logical volume. +In addition to the node-level path redundancy, simplyblock supports multipathing across independent **storage +networks**: storage nodes attached with multiple data interfaces in separate VLANs expose every path on every +network, multiplying the number of client connections (for example, four connections with one failover path and +two networks, or six with two failover paths). This provides an alternative to link aggregation (LACP, MLAG) — +see [Storage Network Multipathing](../non-kubernetes/installation/storage-network-multipathing.md). + ### 3. Redundant Control Plane and Storage Plane To ensure cluster-wide availability, simplyblock operates with full redundancy in both its control plane and diff --git a/docs/deployment-preparation/cluster-deployment-options.md b/docs/deployment-preparation/cluster-deployment-options.md index 9b5a145a..ee2757f6 100644 --- a/docs/deployment-preparation/cluster-deployment-options.md +++ b/docs/deployment-preparation/cluster-deployment-options.md @@ -56,12 +56,19 @@ using either option (defined per volume or storage class), but the cluster inter ### ```--qpair-count``` -The default number of queue pairs (sockets) per volume for an initiator (host) to connect to the -target (server). More queue pairs per volume increase concurrency and volume performance, but require more -server resources (RAM, CPU) and thus limit the total number of volumes per storage node. The default is 3. -For a few very performant volumes, increase the amount. For a large number of less performant -volumes, decrease it. More than 12 parallel connections have limited impact on overall performance. Also, the -host requires at least one core per queue pair. +The default number of queue pairs (sockets) per volume used within the storage cluster. More queue pairs per +volume increase concurrency and volume performance, but require more server resources (RAM, CPU) and thus limit +the total number of volumes per storage node. The default is 32. +For a few very performant volumes, increase the amount. For a large number of less performant volumes, decrease +it. The number of queue pairs per client (initiator) connection is controlled separately via +`--client-qpair-count` (default 3). The host requires at least one core per queue pair. + +### ```--enable-failure-domain``` + +Enables failure-domain anti-affinity for the cluster. Each storage node must then be added with a +`--failure-domain` tag (rack, cabinet, or availability zone), and data, journal copies, and failover paths are +spread across distinct failure domains. This option is deploy-time only: a cluster cannot be upgraded into the +feature, it must be redeployed. See [Failure Domains](../architecture/concepts/failure-domains.md). ### ```--use-backup``` diff --git a/docs/deployment-preparation/hardware-requirements.md b/docs/deployment-preparation/hardware-requirements.md index a53266df..63421ffb 100644 --- a/docs/deployment-preparation/hardware-requirements.md +++ b/docs/deployment-preparation/hardware-requirements.md @@ -77,7 +77,8 @@ As hyper-converged deployments have to share vCPUs, it is recommended to dedicat For RAM, it is required to define the maximum number of NVMe-oF subsystems per node. This depends on the assigned vCPUs and networking performance of the node. For each 10 GBit/s of dedicated network bandwidth it is recommended to use at least 3 subsystems. For each vCPU exceeding 8, it is recommended to use one additional -subsystem. Use the lower of both values (dedicated network bandwidth, vCPUs). +subsystem. Use the lower of both values (dedicated network bandwidth, vCPUs). A hard limit of 75 subsystems per +node applies. See [Limits](../reference/limits.md). For storage nodes, simplyblock highly recommends DDR5 memory for optimal performance. @@ -263,7 +264,7 @@ step. Low-level formating can also be executed manually. In production, simplyblock works with one of two options: - A **redundant network** for storage traffic (e.g., via LACP, Stacked Switches, MLAG, active/active or active/passive NICs, STP, or MSTP). -- Two separate VLANs per node for storage traffic, connected via two separate NIC ports and switch paths, as well as configured as ***NVMe Multipathing***. +- Two separate VLANs per node for storage traffic, connected via two separate NIC ports and switch paths, as well as configured as ***NVMe Multipathing*** (see [Storage Network Multipathing](../non-kubernetes/installation/storage-network-multipathing.md)). In such a setup simplyblock still recommend to provide a **redundant network for management traffic**, but it is not obligatory. For production, software-defined switches such as Linux Bridge or OVS cannot be used. An interface on top of a Linux diff --git a/docs/kubernetes/installation/talos.md b/docs/kubernetes/installation/talos.md index 91ab94d1..abbbeae3 100644 --- a/docs/kubernetes/installation/talos.md +++ b/docs/kubernetes/installation/talos.md @@ -36,7 +36,7 @@ Run the following command on the admin control pod to calculate the huge pages r ```bash title="Run the huge memory calculator" {{ cliname }} storage-node configure \ --calculate-hp-only \ - --max-lvol \ + --max-subsys \ --number-of-devices ``` The following flags also affect the huge page calculation: @@ -47,7 +47,7 @@ The following flags also affect the huge page calculation: ```plain title="Example output of huge pages calculator" [demo@demo ~]# {{ cliname }} storage-node configure \ - --calculate-hp-only --max-lvol 10 --number-of-devices 4 + --calculate-hp-only --max-subsys 10 --number-of-devices 4 2026-02-22 22:27:47,017: 140705369632256: INFO: The required number of huge pages on this host is: 5776 (11552 MB) True ``` diff --git a/docs/kubernetes/operations/parallel-node-addition.md b/docs/kubernetes/operations/parallel-node-addition.md index 6994a85a..c30fe4aa 100644 --- a/docs/kubernetes/operations/parallel-node-addition.md +++ b/docs/kubernetes/operations/parallel-node-addition.md @@ -26,7 +26,7 @@ threshold, which would cause cluster unavailability. ## Configuration -Parallelism for non-FDB workers is controlled by `StorageNode.spec.maxParallelNodeAdds`. +Parallelism for non-FDB workers is controlled by `StorageNodeSet.spec.maxParallelNodeAdds`. | Value | Behavior | |---------------|------------------------------------------------------------------| diff --git a/docs/kubernetes/operations/scaling/expanding-storage-cluster.md b/docs/kubernetes/operations/scaling/expanding-storage-cluster.md index b2ef5bbb..f50f4527 100644 --- a/docs/kubernetes/operations/scaling/expanding-storage-cluster.md +++ b/docs/kubernetes/operations/scaling/expanding-storage-cluster.md @@ -13,8 +13,10 @@ designed to be minimal, it is still recommended to expand the cluster at times w full utilization. !!! info - Add storage nodes in **pairs** (i.e., 2, 4, 6, … nodes at a time). - Expansions with an odd number of nodes are **not supported**. + Storage nodes can be added **individually** (expansion mode, which integrates the new node by re-homing + existing failover paths) or **in groups**: at least two new nodes for clusters with one parity chunk, at + least three for clusters with two parity chunks (FTT 2). On clusters with failure domains, additional + balance rules apply. See [Failure Domains](../../../architecture/concepts/failure-domains.md). To add a new storage node, follow the installation steps for the chosen deployment method up to the point where nodes are added to the cluster, then continue here: @@ -22,7 +24,7 @@ To add a new storage node, follow the installation steps for the chosen deployme - [Storage nodes on Linux](../../../non-kubernetes/installation/install-sp.md) After adding the **first** new storage node, the cluster transitions to **IN_EXPANSION** and starts background rebalancing. -Add the remaining node(s) required for the expansion (storage nodes must be added in **pairs**). +Add the remaining node(s) required for the expansion. Once all newly added nodes are healthy/ready, finalize the expansion: ```bash title="Finalize cluster expansion" diff --git a/docs/non-kubernetes/installation/index.md b/docs/non-kubernetes/installation/index.md index 73bff726..dcd40378 100644 --- a/docs/non-kubernetes/installation/index.md +++ b/docs/non-kubernetes/installation/index.md @@ -50,9 +50,10 @@ On storage nodes, simplyblock can use either one network interface for both stor or separate interfaces (VLANs or subnets). !!! Important - It is possible to configure a storage cluster with NVMe-oF Multipathing. This requires two storage + It is possible to configure a storage cluster with NVMe-oF multipathing. This requires two storage VLANs per host, routed via separate NIC ports and switches for fault tolerance. This configuration can be used as an alternative to a HA networking setup with link aggregation (such as LACP, MLAG, and similar). + See [Storage Network Multipathing](storage-network-multipathing.md) for the setup instructions. To install simplyblock in a specific environment, these commands may have to be adopted to match its configuration. diff --git a/docs/non-kubernetes/installation/install-sp.md b/docs/non-kubernetes/installation/install-sp.md index 82771711..b2470958 100644 --- a/docs/non-kubernetes/installation/install-sp.md +++ b/docs/non-kubernetes/installation/install-sp.md @@ -76,11 +76,15 @@ will configure one storage node per NUMA node. ```bash title="Configure the storage node" sudo {{ cliname }} storage-node configure \ - --max-lvol + --max-subsys ``` +The `--max-subsys` parameter defines the maximum number of NVMe-oF subsystems (and hence, in the default +one-volume-per-subsystem layout, logical volumes) this node will serve. It drives the node's memory reservation +and is capped by a hard limit of 75 subsystems per node (see [Limits](../../reference/limits.md)). + ```plain title="Example output of storage node configure" -[demo@demo-3 ~]# sudo {{ cliname }} storage-node configure --nodes-per-socket=2 --max-lvol=50 +[demo@demo-3 ~]# sudo {{ cliname }} storage-node configure --nodes-per-socket=2 --max-subsys=50 2025-05-14 10:40:17,460: INFO: 0000:00:04.0 is already bound to nvme. 0000:00:1e.0 0000:00:1e.0 @@ -131,20 +135,26 @@ When all storage nodes are prepared, they can be added to the storage cluster. ```bash title="Attaching a storage node to the storage plane" sudo {{ cliname }} storage-node add-node \ - --journal-partition \ - --data-nics + --data-nics [,] ``` -If a separate NIC (e.g., BOND device) is used for storage traffic (no matter if in the cluster and between hosts and -cluster nodes), the `--data-nics` parameter must be specified. In R25.10, zero or one data NICs are supported. Zero data -NICs will utilize the management interface for all traffic. +If separate NICs (e.g., a BOND device, or dedicated interfaces per storage VLAN) are used for storage traffic +(no matter if in the cluster or between hosts and cluster nodes), the `--data-nics` parameter must be specified. +Without it, the management interface carries all traffic. Multiple interfaces are given as a comma-separated +list (e.g., `--data-nics eth1,eth2`), in which case all NVMe-oF subsystems listen on every data interface and +connections are established once per interface — see +[Storage Network Multipathing](storage-network-multipathing.md). !!! info - The number of partitions (_NUM_OF_PARTITIONS_) depends on the storage node setup. If a storage node has a - separate journaling device (e.g., an SLC NVMe device), the value should be zero (_0_) to prevent the storage - devices from being partitioned. This improves the performance and prevents device sharing between the journal and - the actual data storage location. However, in most cases, a separate journaling device is not available or required - and the value of `--journal-partition` has to be 1 (default if nothing is specified). + By default, simplyblock auto-creates small journal partitions on the NVMe data devices (a maximum of 3% of + the total available raw disk space). If a storage node has a separate journaling device (e.g., an SLC NVMe + device), pass `--enable-journal-device` to use the smallest NVMe device of the node exclusively for the + journal. This improves performance and prevents device sharing between the journal and the actual data + storage location. + +If the cluster was created with failure-domain support, every node must additionally be tagged with its +failure-domain id via `--failure-domain `; see +[Managing Failure Domains](../operations/failure-domains.md). The output will look something like the following example: diff --git a/docs/non-kubernetes/operations/backup-recovery.md b/docs/non-kubernetes/operations/backup-recovery.md index 166879b6..17a9fab9 100644 --- a/docs/non-kubernetes/operations/backup-recovery.md +++ b/docs/non-kubernetes/operations/backup-recovery.md @@ -48,16 +48,23 @@ Restoring a backup creates a new logical volume with the data reconstructed from ```bash title="Restore a backup" {{ cliname }} backup restore \ - --lvol --pool \ - [--node ] [--cluster-id ] + --lvol --pool --cluster-id \ + [--node ] ``` +The `--lvol`, `--pool`, and `--cluster-id` parameters are required. Any node of the cluster can restore any +backup; without `--node`, the node that took the backup is used. + The restore process downloads and applies each backup in the chain. The new volume is set to a restoring state during the transfer and transitions to online once complete. !!! warning The restore operation creates a new volume. It does not overwrite or modify any existing volume. +!!! note + The restored volume is created with the default high-availability type and NVMe/TCP, regardless of the + settings of the original volume. Deleting the original snapshot or volume does not affect its backups. + ### Deleting Backups To delete all backups for a specific volume: @@ -139,15 +146,23 @@ This produces a JSON file containing backup metadata (not the actual data, which #### Importing Backup Metadata -##### Switching Backup Source +On the target cluster, import the metadata: + +```bash title="Import backup metadata" +{{ cliname }} backup import --cluster-id +``` + +#### Switching the Backup Source -Before restoring imported backups, switch the cluster's S3 source to read from the original cluster's bucket: +Before restoring imported backups, switch the target cluster's S3 source to read from the original cluster's +bucket: ```bash title="Switch backup source" -{{ cliname }} backup source-switch [--cluster-id ] +{{ cliname }} backup source-switch [--cluster-id ] ``` -To list available backup sources: +The switch changes only the bucket that is read; the target cluster's own S3 credentials and endpoint are reused, +so they must have access to the source cluster's bucket. To list available backup sources: ```bash title="List backup sources" {{ cliname }} backup source-list [--cluster-id ] @@ -157,17 +172,16 @@ To list available backup sources: While the backup source is switched to an external cluster, new backups cannot be created on the local cluster. Switch back to the local source after completing restore operations. -After switching the source, use the standard `backup restore` command to restore from the imported backups. +After switching the source, use the standard `backup restore` command to restore from the imported backups. For a +cross-cluster restore, pass `--node ` explicitly, since the node recorded in the backup belongs to +the source cluster. -On the target cluster, import the metadata: +Once the restores are complete, switch the source back: -```bash title="Import backup metadata" -{{ cliname }} backup import [--cluster-id ] +```bash title="Switch back to the local backup source" +{{ cliname }} backup source-switch local [--cluster-id ] ``` -!!! warning - Do not forget to switch back the source to the internal cluster to resume normal backup operations. - ## Kubernetes CRD Operations In Kubernetes environments, backups can be managed declaratively using Custom Resource Definitions (CRDs). This diff --git a/docs/non-kubernetes/operations/find-secondary-node.md b/docs/non-kubernetes/operations/find-secondary-node.md index 20a12b17..fc7d3645 100644 --- a/docs/non-kubernetes/operations/find-secondary-node.md +++ b/docs/non-kubernetes/operations/find-secondary-node.md @@ -4,11 +4,11 @@ description: "Finding the Secondary Node: Simplyblock, in high-availability mode weight: 20070 --- -Simplyblock, in high-availability mode, creates two connections per logical volume: a primary and a secondary -connection. +Simplyblock, in high-availability mode, creates multiple connections per logical volume: a primary and a secondary +connection, plus a tertiary connection on clusters with two parity chunks (FTT 2). -The secondary connection will be used in case of issues or failures of the primary storage node which owns the logical -volume. +The secondary (and tertiary) connections will be used in case of issues or failures of the primary storage node +which owns the logical volume. ## When to Use This diff --git a/docs/non-kubernetes/operations/migrating-storage-node.md b/docs/non-kubernetes/operations/migrating-storage-node.md index a8d40dd4..8279dd45 100644 --- a/docs/non-kubernetes/operations/migrating-storage-node.md +++ b/docs/non-kubernetes/operations/migrating-storage-node.md @@ -40,7 +40,7 @@ To prepare the new storage host, the following commands must be executed. ```bash title="Preparing the configuration" {{ cliname }} storage-node configure \ - --max-lvol= \ + --max-subsys= \ --max-size= \ [--nodes-per-socket=] ``` diff --git a/docs/non-kubernetes/operations/reconnect-nvme-device.md b/docs/non-kubernetes/operations/reconnect-nvme-device.md index 0bc58c42..d4c0fcc4 100644 --- a/docs/non-kubernetes/operations/reconnect-nvme-device.md +++ b/docs/non-kubernetes/operations/reconnect-nvme-device.md @@ -21,8 +21,8 @@ will immediately reconnect missing controllers and connection paths. ```plain title="Example output for connection string retrieval" [demo@demo ~]# {{ cliname }} volume connect 82e587c5-4a94-42a1-86e5-a5b8a6a75fc4 -sudo nvme connect --reconnect-delay=2 --ctrl-loss-tmo=60 --nr-io-queues=6 --keep-alive-tmo=5 --transport=tcp --traddr=192.168.10.112 --trsvcid=9100 --nqn=nqn.2023-02.io.simplyblock:0f2c4cb0-a71c-4830-bcff-11112f0ee51a:lvol:82e587c5-4a94-42a1-86e5-a5b8a6a75fc4 -sudo nvme connect --reconnect-delay=2 --ctrl-loss-tmo=60 --nr-io-queues=6 --keep-alive-tmo=5 --transport=tcp --traddr=192.168.10.113 --trsvcid=9100 --nqn=nqn.2023-02.io.simplyblock:0f2c4cb0-a71c-4830-bcff-11112f0ee51a:lvol:82e587c5-4a94-42a1-86e5-a5b8a6a75fc4 +sudo nvme connect --reconnect-delay=2 --ctrl-loss-tmo=60 --nr-io-queues=6 --keep-alive-tmo=5 --transport=tcp --traddr=192.168.10.112 --trsvcid=4420 --nqn=nqn.2023-02.io.simplyblock:0f2c4cb0-a71c-4830-bcff-11112f0ee51a:lvol:82e587c5-4a94-42a1-86e5-a5b8a6a75fc4 +sudo nvme connect --reconnect-delay=2 --ctrl-loss-tmo=60 --nr-io-queues=6 --keep-alive-tmo=5 --transport=tcp --traddr=192.168.10.113 --trsvcid=4420 --nqn=nqn.2023-02.io.simplyblock:0f2c4cb0-a71c-4830-bcff-11112f0ee51a:lvol:82e587c5-4a94-42a1-86e5-a5b8a6a75fc4 ``` ## Increase Loss Timeout diff --git a/docs/non-kubernetes/operations/replacing-storage-node.md b/docs/non-kubernetes/operations/replacing-storage-node.md index 701d1771..1903b827 100644 --- a/docs/non-kubernetes/operations/replacing-storage-node.md +++ b/docs/non-kubernetes/operations/replacing-storage-node.md @@ -30,8 +30,10 @@ To start a new storage node, follow the storage node installation according to t ## Remove the old Storage Node -!!! danger - All volumes on this storage node, which haven't been migrated before the removal, will become inaccessible! +!!! important + A storage node can only be removed when it hosts no logical volumes or snapshots. Migrate all volumes off the + node first (see [Volume Migration](volume-migration.md)); the removal is refused otherwise. The node to be + removed must be online or suspended, and all other storage nodes must be online. To remove the old storage node, use the `{{ cliname }}` command line tool. @@ -39,7 +41,9 @@ To remove the old storage node, use the `{{ cliname }}` command line tool. {{ cliname }} storage-node remove ``` -Wait until the operation has successfully finished. Afterward, the storage node is removed from the cluster. +The removal runs as a background task: it shuts the node down, rewires the failover paths hosted by the node onto +other nodes, and migrates its devices' data before marking the node removed. Wait until the operation has +successfully finished. Afterward, the storage node is removed from the cluster. This can be checked again with the `{{ cliname }}` command line tool. diff --git a/docs/non-kubernetes/usage/encrypting.md b/docs/non-kubernetes/usage/encrypting.md index 169693b0..7dfe672b 100644 --- a/docs/non-kubernetes/usage/encrypting.md +++ b/docs/non-kubernetes/usage/encrypting.md @@ -26,31 +26,24 @@ Simplyblock supports the encryption of logical volumes. Internally, simplyblock [crypto bdev](https://spdk.io/doc/bdev.html){:target="_blank" rel="noopener"} provided by SPDK to implement its encryption functionality. -The encryption uses an AES_XTS variable-length block cipher. This cipher requires two keys of 16 to 32 bytes each. The -keys need to have the same length, meaning that if one key is 32 bytes long, the other one has to be 32 bytes, too. +The encryption uses an AES_XTS variable-length block cipher. -!!! recommendation - Simplyblock strongly recommends two keys of 32 bytes. +The encryption keys are created and stored by the cluster's key management system (KMS) when the volume is +created. By default, simplyblock manages the keys internally; alternatively, an external KMS (HashiCorp Vault or +OpenBao) can be configured at cluster creation time via `--hashicorp-vault-url`. See +[External Key Management](../../architecture/concepts/external-key-management.md) for the architecture. -## Generate Random Keys - -Simplyblock does not provide an integrated way to generate encryption keys, but recommends using the OpenSSL tool chain. - -To generate the two keys, the following command is run twice. The result must be stored for later. - -```bash title="Create an Encryption Key" -openssl rand -hex 32 -``` +!!! note + Earlier releases required manually generated keys passed via `--crypto-key1` and `--crypto-key2`. These + parameters are deprecated since 26.2 and cannot be used anymore; key handling is fully KMS-based. ## Creating an Encrypted Logical Volume To provision a new Logical Volume with encryption enabled: -```bash +```bash title="Create an encrypted logical volume" {{ cliname }} volume add \ --encrypt \ - --crypto-key1 \ - --crypto-key2 \ \ \ @@ -60,11 +53,9 @@ To see all available parameters when creating a logical volume, see [Provisionin ### Parameters -| Parameter | Description | Default | -|---------------------------|--------------------------------------------------|---------| -| --encrypt | Enables inline encryption on the logical volume. | false | -| --crypto-key1 CRYPTO_KEY1 | The hex value of the first encryption key. | | -| --crypto-key2 CRYPTO_KEY2 | The hex value of the second encryption key. | | +| Parameter | Description | Default | +|-------------|--------------------------------------------------|---------| +| `--encrypt` | Enables inline encryption on the logical volume. | false | ## Verification diff --git a/docs/non-kubernetes/usage/provisioning.md b/docs/non-kubernetes/usage/provisioning.md index ab47c7a0..dd274134 100644 --- a/docs/non-kubernetes/usage/provisioning.md +++ b/docs/non-kubernetes/usage/provisioning.md @@ -29,19 +29,23 @@ To create a new logical volume: ### Available Parameters -| Parameter | Description | Default | -|-------------------------------|-----------------------------------------------------------------------------|---------| -| --snapshot, -s | Enables snapshot capability on the logical volume. | false | -| --max-size | Maximum size of the logical volume. | 0 | -| --ha-type {single,ha,default} | High-availability mode of the logical volume. | ha | -| --encrypt | Enables inline encryption on the logical volume. | false | -| --crypto-key1 CRYPTO_KEY1 | The hex value of the first encryption key. | | -| --crypto-key2 CRYPTO_KEY2 | The hex value of the second encryption key. | | -| --max-rw-iops MAX_RW_IOPS | Maximum I/O operations per second. | 0 | -| --max-rw-mbytes MAX_RW_MBYTES | Maximum read/write throughput. | 0 | -| --max-r-mbytes MAX_R_MBYTES | Maximum read throughout. | 0 | -| --max-w-mbytes MAX_W_MBYTES | Maximum write throughput. | 0 | -| --allowed-hosts | Path to JSON file with host NQNs allowed to access this volume's subsystem. | | +| Parameter | Description | Default | +|---------------------------------|-----------------------------------------------------------------|---------| +| `--snapshot`, `-s` | Enables snapshot capability on the logical volume. | false | +| `--max-size` | Maximum size of the logical volume. | 1000T | +| `--ha-type {single,ha,default}` | High-availability mode of the logical volume. | default | +| `--encrypt` | Enables inline encryption on the logical volume. | false | +| `--max-rw-iops ` | Maximum I/O operations per second. | 0 | +| `--max-rw-mbytes ` | Maximum read/write throughput. | 0 | +| `--max-r-mbytes ` | Maximum read throughput. | 0 | +| `--max-w-mbytes ` | Maximum write throughput. | 0 | +| `--replicate` | Enables snapshot-based asynchronous replication for the volume. | false | + +The encryption keys of a volume created with `--encrypt` are managed by the cluster's key management system. See +[Encrypting a Logical Volume](encrypting.md). + +Host access restrictions (allowed host NQNs) are configured on the storage pool with +`{{ cliname }} storage-pool add-host` and `{{ cliname }} storage-pool remove-host`, not per volume. ## Verification diff --git a/docs/non-kubernetes/usage/quality-of-service.md b/docs/non-kubernetes/usage/quality-of-service.md index d1f5dc6c..b3afbce7 100644 --- a/docs/non-kubernetes/usage/quality-of-service.md +++ b/docs/non-kubernetes/usage/quality-of-service.md @@ -20,18 +20,18 @@ QoS can be applied when creating a new logical volume: \ \ \ - --max-rw-iops MAX_RW_IOPS 3500 \ - --max-rw-mbytes MAX_RW_MBYTES 125 + --max-rw-iops 3500 \ + --max-rw-mbytes 125 ``` ### Parameters -| Parameter | Description | Default | -|-------------------------------|------------------------------------|---------| -| --max-rw-iops MAX_RW_IOPS | Maximum I/O operations per second. | 0 | -| --max-rw-mbytes MAX_RW_MBYTES | Maximum read/write throughput. | 0 | -| --max-r-mbytes MAX_R_MBYTES | Maximum read throughout. | 0 | -| --max-w-mbytes MAX_W_MBYTES | Maximum write throughput. | 0 | +| Parameter | Description | Default | +|----------------------------|------------------------------------|---------| +| `--max-rw-iops ` | Maximum I/O operations per second. | 0 | +| `--max-rw-mbytes ` | Maximum read/write throughput. | 0 | +| `--max-r-mbytes ` | Maximum read throughput. | 0 | +| `--max-w-mbytes ` | Maximum write throughput. | 0 | To see all available parameters when creating a logical volume, see [Provisioning](provisioning.md). @@ -42,8 +42,8 @@ QoS settings can also be updated on an existing logical volume: ```bash {{ cliname }} volume qos-set \ \ - --max-rw-iops MAX_RW_IOPS 5000 \ - --max-rw-mbytes MAX_RW_MBYTES 250 + --max-rw-iops 5000 \ + --max-rw-mbytes 250 ``` ## Verification diff --git a/docs/reference/troubleshooting/storage-plane.md b/docs/reference/troubleshooting/storage-plane.md index 957cd852..11846d59 100644 --- a/docs/reference/troubleshooting/storage-plane.md +++ b/docs/reference/troubleshooting/storage-plane.md @@ -9,18 +9,19 @@ weight: 30200 **Symptom:** After a fresh deployment, the cluster cannot be activated. The activation process hangs or fails, and the storage nodes show `n/0` disks available in the disks column (`{{ cliname }} storage-node list`). -1. Shutdown all storage nodes: `{{ cliname }} storage-node shutdown --force` -2. Force remove all storage nodes: `{{ cliname }} storage-node remove --force-remove` -3. Delete all storage nodes: `{{ cliname }} storage-node delete ` -4. Re-add all storage nodes. The disks should become active. -5. Try to activate the cluster. +1. Remove all storage nodes: `{{ cliname }} storage-node remove `. The node must be online or + suspended; the removal shuts the node down itself. (`--force-remove` does not force a removal — it only + cancels active tasks of the node.) +2. Delete all storage nodes: `{{ cliname }} storage-node delete ` +3. Re-add all storage nodes. The disks should become active. +4. Try to activate the cluster. ## Storage Node Health Check Shows Health=False **Symptom:** The storage node health check returns _health=false_ (`{{ cliname }} storage-node list`). 1. First run `{{ cliname }} storage-node check `. -2. If the command keeps showing an unhealthy storage node, _suspend_, _shutdown_, and restart the storage node. +2. If the command keeps showing an unhealthy storage node, _shutdown_ and _restart_ the storage node. !!! danger Never shutdown or restart a storage node while the cluster is in **degraded** state. This can lead to potential diff --git a/docs/release-notes/26-2.md b/docs/release-notes/26-2.md index 9b10668a..9fbe8eaf 100644 --- a/docs/release-notes/26-2.md +++ b/docs/release-notes/26-2.md @@ -51,4 +51,4 @@ It is possible to upgrade from `26.1` and `25.10.5`. - A significant performance optimization during node outages (journal writes). - The ability to live migrate volumes between storage clusters. - Inline data integrity validation to prevent silent data corruption. -- Support for hierarchical failure domains. +- Support for failure domains, spreading data, journal copies, and failover paths across racks, cabinets, or availability zones. diff --git a/snippets/network-port-table.md b/snippets/network-port-table.md index 50767658..79dffe70 100644 --- a/snippets/network-port-table.md +++ b/snippets/network-port-table.md @@ -1,7 +1,7 @@ | Service | Direction | Hosts | Network | Port(s) | Protocol(s) | |----------------------|-----------------|------------------|---------|---------------------------------|-------------| | ICMP | ingress | control | Control | - | ICMP | -| spdk-http-proxy | ingress, egress | storage, control | Control | 5000 | TCP | +| storage-node-api | ingress, egress | storage, control | Control | 5000 | TCP | | NVMf (client-target) | egress | client | Storage | 4420-4499 | TCP | | NVMf (internal) | ingress, egress | storage | Storage | 4420-4499 | TCP | | FoundationDB | ingress | control | Control | 4500 | TCP | From 52733f1c36b8227fe1795cc93aaa9f7f7da4b52e Mon Sep 17 00:00:00 2001 From: "Christoph Engelbert (noctarius)" Date: Fri, 7 Aug 2026 14:38:54 +0200 Subject: [PATCH 3/7] docs: clear quality gate errors on the pages added in this branch The new failure-domain, multipathing, volume-migration, asynchronous replication, and limits pages predate the quality gate rules on main. Applied the auto-fixers: American English (canceled, toward, afterward), bold list-item subjects with the colon inside the asterisks, and table re-alignment. Co-Authored-By: Claude Opus 5 (1M context) --- docs/architecture/concepts/failure-domains.md | 16 +++--- .../storage-network-multipathing.md | 10 ++-- .../operations/failure-domains.md | 14 +++--- .../operations/volume-migration.md | 26 +++++----- docs/reference/limits.md | 50 +++++++++---------- 5 files changed, 58 insertions(+), 58 deletions(-) diff --git a/docs/architecture/concepts/failure-domains.md b/docs/architecture/concepts/failure-domains.md index 6551fad9..5bb5d260 100644 --- a/docs/architecture/concepts/failure-domains.md +++ b/docs/architecture/concepts/failure-domains.md @@ -21,14 +21,14 @@ is added to the cluster. With failure domains enabled, placement decisions consider the domain tag in four independent dimensions: -1. **Data and parity chunks**: The distributed erasure coding spreads the chunks of each stripe across distinct +1. **Data and parity chunks:** The distributed erasure coding spreads the chunks of each stripe across distinct failure domains, so that a full domain outage leaves enough chunks to reconstruct all data within the configured erasure coding scheme. -2. **Journal copies**: The copies of the high-availability write journal are balanced across domains with a +2. **Journal copies:** The copies of the high-availability write journal are balanced across domains with a per-domain cap, so that losing a whole domain always leaves enough journal copies to maintain the journal quorum. -3. **Failover paths**: The secondary (and, with two parity chunks, tertiary) failover nodes of each logical volume +3. **Failover paths:** The secondary (and, with two parity chunks, tertiary) failover nodes of each logical volume are placed in different failure domains than the primary node wherever possible. -4. **Cluster status**: The health assessment of the cluster understands domains. Any combination of node and +4. **Cluster status:** The health assessment of the cluster understands domains. Any combination of node and device outages confined to a single failure domain keeps the cluster serving I/O in a degraded state instead of suspending it. @@ -67,15 +67,15 @@ a same-domain secondary path, and its tertiary path is still guaranteed to be cr !!! note Balance is counted in physical hosts, not storage nodes. On multi-socket hosts running two storage nodes, both nodes count as one host and must carry the same failure-domain id. Dedicated secondary nodes are not counted - towards the balance. + toward the balance. ## Failure Domains and Erasure Coding Schemes The number of failure domains should match the data protection goal: -| Goal | Recommendation | -|------|----------------| -| Survive one full domain outage | At least `parity chunks + 1` distinct failure domains | +| Goal | Recommendation | +|---------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------| +| Survive one full domain outage | At least `parity chunks + 1` distinct failure domains | | Survive one full domain outage plus one further node or drive failure elsewhere | Erasure coding scheme with two parity chunks (e.g., `1+2`, `2+2`) and at least as many domains as data chunks | The high-availability journal requires at least **four** journal copies on failure-domain clusters (instead of diff --git a/docs/non-kubernetes/installation/storage-network-multipathing.md b/docs/non-kubernetes/installation/storage-network-multipathing.md index 85b5ebef..bb8cf49d 100644 --- a/docs/non-kubernetes/installation/storage-network-multipathing.md +++ b/docs/non-kubernetes/installation/storage-network-multipathing.md @@ -30,11 +30,11 @@ switch paths. A typical layout separates management and storage traffic completely: -| Network interface | Purpose | Subnet (example) | -|-------------------|---------------------|------------------| -| eth0 | Management / control plane | 192.168.10.0/24 | -| eth1 | Storage path A | 10.10.10.0/24 | -| eth2 | Storage path B | 10.10.20.0/24 | +| Network interface | Purpose | Subnet (example) | +|-------------------|----------------------------|------------------| +| eth0 | Management / control plane | 192.168.10.0/24 | +| eth1 | Storage path A | 10.10.10.0/24 | +| eth2 | Storage path B | 10.10.20.0/24 | The management network should still be highly available (a simple bond is sufficient), but it does not carry storage traffic. diff --git a/docs/non-kubernetes/operations/failure-domains.md b/docs/non-kubernetes/operations/failure-domains.md index 0352f433..75ea5afc 100644 --- a/docs/non-kubernetes/operations/failure-domains.md +++ b/docs/non-kubernetes/operations/failure-domains.md @@ -16,7 +16,7 @@ failure domains are assigned declaratively through the Simplyblock Operator ## Enabling Failure Domains -Failure-domain support is enabled when the storage cluster is created and is immutable afterwards: +Failure-domain support is enabled when the storage cluster is created and is immutable afterward: ```bash title="Create a cluster with failure-domain support" {{ cliname }} cluster create --enable-failure-domain @@ -58,12 +58,12 @@ The assigned domains are shown in the node list once at least one node carries a Activating a freshly assembled failure-domain cluster enforces the following rules: -| Rule | Enforcement | -|------|-------------| -| Every node carries a failure-domain id | Hard — activation fails | -| A host does not span two domains | Hard — activation fails | -| At least two distinct domains exist | Hard — activation fails | -| All domains hold an equal number of hosts | Hard — activation fails | +| Rule | Enforcement | +|-----------------------------------------------|--------------------------------------------------------------------------------------| +| Every node carries a failure-domain id | Hard — activation fails | +| A host does not span two domains | Hard — activation fails | +| At least two distinct domains exist | Hard — activation fails | +| All domains hold an equal number of hosts | Hard — activation fails | | At least `parity chunks + 1` distinct domains | Recommendation — a warning is logged, activation proceeds with best-effort placement | During activation, simplyblock computes the interleaved host rotation across the domains and assigns all secondary diff --git a/docs/non-kubernetes/operations/volume-migration.md b/docs/non-kubernetes/operations/volume-migration.md index 09f32dd0..6640e33b 100644 --- a/docs/non-kubernetes/operations/volume-migration.md +++ b/docs/non-kubernetes/operations/volume-migration.md @@ -30,7 +30,7 @@ A migration is a two-step operation with a client action in between: !!! warning A pre-created migration must be continued within **five minutes**. If `migrate-continue` is not run in time, - the migration is automatically cancelled and the target resources are released. + the migration is automatically canceled and the target resources are released. ## Starting a Migration @@ -69,28 +69,28 @@ step with `--host-nqn `. The list shows source and target node, the current phase, status, snapshot progress (migrated/planned), the retry counter, and the last error, if any. -| Phase | Meaning | -|-------|---------| -| `pre_created` | Target subsystem exists, waiting for `migrate-continue`. | -| `snap_copy` | The snapshot chain is being copied to the target. | -| `lvol_migrate` | The final delta is being transferred. This is the only phase with a (short) I/O freeze. | -| `cleanup_source` | Data has moved; source-side objects are being removed. | -| `cleanup_target` | Rollback after a failure or cancellation: target-side objects are being removed. | -| `completed` | The migration has finished. | +| Phase | Meaning | +|------------------|-----------------------------------------------------------------------------------------| +| `pre_created` | Target subsystem exists, waiting for `migrate-continue`. | +| `snap_copy` | The snapshot chain is being copied to the target. | +| `lvol_migrate` | The final delta is being transferred. This is the only phase with a (short) I/O freeze. | +| `cleanup_source` | Data has moved; source-side objects are being removed. | +| `cleanup_target` | Rollback after a failure or cancellation: target-side objects are being removed. | +| `completed` | The migration has finished. | A volume that is part of an active migration shows the migration ID in the `migrating` field of `{{ cliname }} volume get `. -## Cancelling a Migration +## Canceling a Migration ```bash title="Cancel a migration" {{ cliname }} volume migrate-cancel ``` -A migration cancelled in the `pre_created` phase is torn down immediately. In later phases, the cancellation is +A migration canceled in the `pre_created` phase is torn down immediately. In later phases, the cancellation is picked up asynchronously by the migration runner, which rolls the target back (`cleanup_target`); it may take a few seconds to reflect in `migrate-list`. Data on the source remains intact and authoritative until the final -cutover, so a migration can be cancelled at any phase before `cleanup_source`. +cutover, so a migration can be canceled at any phase before `cleanup_source`. ## Migrating Shared Subsystems (Batch Migration) @@ -116,7 +116,7 @@ A migration is admitted only if: - The volume is online; the target node is online and different from the source node; the source node is online or suspended. - The volume has no other active migration. Re-running `volume migrate` with the same volume and target returns - the existing migration ID; a different target requires cancelling the existing migration first. + the existing migration ID; a different target requires canceling the existing migration first. Additional operational constraints while a migration is active: diff --git a/docs/reference/limits.md b/docs/reference/limits.md index 20da8cf1..c021298d 100644 --- a/docs/reference/limits.md +++ b/docs/reference/limits.md @@ -9,11 +9,11 @@ control plane; others depend on the node's vCPU count and memory configuration. ## Hard Per-Node Object Limits -| Limit | Value | What it counts | -|-------|------:|----------------| -| Objects per node | 6000 | Logical volumes, clones, and snapshots owned by the node (its logical volume store) | -| NVMe-oF subsystems per node | 75 | Subsystems for which the node is the primary; namespaced volumes sharing one subsystem count as one | -| Namespaces per subsystem | 50 | Volumes (namespaces) sharing one NVMe-oF subsystem | +| Limit | Value | What it counts | +|-----------------------------|------:|-----------------------------------------------------------------------------------------------------| +| Objects per node | 6000 | Logical volumes, clones, and snapshots owned by the node (its logical volume store) | +| NVMe-oF subsystems per node | 75 | Subsystems for which the node is the primary; namespaced volumes sharing one subsystem count as one | +| Namespaces per subsystem | 50 | Volumes (namespaces) sharing one NVMe-oF subsystem | These limits are enforced on every create path (volume create, snapshot create, clone). When a limit is reached, the operation fails with an explanatory error, for example: @@ -61,32 +61,32 @@ the node's subsystem limit). On top of the hard object limits, several resource limits scale with the vCPU count of the storage node: -| Limit | Rule | -|-------|------| -| CPU cores per storage node | At most 64 cores can be assigned to one storage node (SPDK instance). | -| Distribution services per node | Scales with the assigned cores, capped at 12. | -| NVMe-oF buffer pools | Scale with core count and `--max-subsys`; they determine part of the huge-page demand. | -| Huge-page memory | The minimum huge-page memory grows with the core count and the configured maximum number of subsystems. Nodes refuse to start with insufficient huge pages. | -| Storage nodes per host | 1 or 2 (`--nodes-per-socket`), aligned to NUMA sockets. | +| Limit | Rule | +|--------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| +| CPU cores per storage node | At most 64 cores can be assigned to one storage node (SPDK instance). | +| Distribution services per node | Scales with the assigned cores, capped at 12. | +| NVMe-oF buffer pools | Scale with core count and `--max-subsys`; they determine part of the huge-page demand. | +| Huge-page memory | The minimum huge-page memory grows with the core count and the configured maximum number of subsystems. Nodes refuse to start with insufficient huge pages. | +| Storage nodes per host | 1 or 2 (`--nodes-per-socket`), aligned to NUMA sockets. | In practice, the **memory** derived from vCPU count and `--max-subsys` is the sizing driver: see [Hardware Requirements](../deployment-preparation/hardware-requirements.md) for the RAM formula per subsystem. ## Cluster-Level Limits and Gates -| Limit | Default | Description | -|-------|--------:|-------------| -| Fault tolerance (FTT) | 1 | 1 or 2, derived from the parity chunks of the erasure coding scheme. | -| Minimum online devices at activation | — | Data chunks + parity chunks + 1. | -| Minimum online nodes for volume creation | — | At least data chunks + parity chunks online nodes. | -| Journal copies (`--ha-jm-count`) | 3 (FTT 1) / 4 (FTT 2) | Failure-domain clusters require 4 even at FTT 1. | -| Minimum volume size | 100 MiB | Smaller volumes are rejected. | -| Provisioning warning (`--prov-cap-warn`) | 250 % | Warning when total provisioned capacity exceeds this ratio of the cluster capacity. | -| Provisioning limit (`--prov-cap-crit`) | 500 % | Volume creation fails beyond this over-provisioning ratio. | -| Utilization warning / critical (`--cap-warn` / `--cap-crit`) | 89 % / 99 % | Alerts on used physical capacity. | -| Storage pool caps (`--pool-max`, `--lvol-max`) | unlimited | Optional per-pool caps for total provisioned size and per-volume size. | -| NVMe/TCP qpairs per volume (`--qpair-count`) | 32 | Cluster-internal queue pair count per volume connection. | -| Client qpairs (`--client-qpair-count`) | 3 | Queue pairs per client connection. | +| Limit | Default | Description | +|--------------------------------------------------------------|----------------------:|-------------------------------------------------------------------------------------| +| Fault tolerance (FTT) | 1 | 1 or 2, derived from the parity chunks of the erasure coding scheme. | +| Minimum online devices at activation | — | Data chunks + parity chunks + 1. | +| Minimum online nodes for volume creation | — | At least data chunks + parity chunks online nodes. | +| Journal copies (`--ha-jm-count`) | 3 (FTT 1) / 4 (FTT 2) | Failure-domain clusters require 4 even at FTT 1. | +| Minimum volume size | 100 MiB | Smaller volumes are rejected. | +| Provisioning warning (`--prov-cap-warn`) | 250 % | Warning when total provisioned capacity exceeds this ratio of the cluster capacity. | +| Provisioning limit (`--prov-cap-crit`) | 500 % | Volume creation fails beyond this over-provisioning ratio. | +| Utilization warning / critical (`--cap-warn` / `--cap-crit`) | 89 % / 99 % | Alerts on used physical capacity. | +| Storage pool caps (`--pool-max`, `--lvol-max`) | unlimited | Optional per-pool caps for total provisioned size and per-volume size. | +| NVMe/TCP qpairs per volume (`--qpair-count`) | 32 | Cluster-internal queue pair count per volume connection. | +| Client qpairs (`--client-qpair-count`) | 3 | Queue pairs per client connection. | There is no built-in limit on the number of storage nodes per cluster, clusters per control plane, or storage pools per cluster. From 8a8d5de47e88fffc2e1cff52bf7981a2e5c88037 Mon Sep 17 00:00:00 2001 From: "Christoph Engelbert (noctarius)" Date: Fri, 7 Aug 2026 16:30:23 +0200 Subject: [PATCH 4/7] docs: apply house text style to the pages added in this branch Clears the 73 style warnings on the lines this branch introduces, all of them rules the gates report but do not fail on: - em dashes setting off a clause replaced by parentheses, a comma, or a colon (34) - semicolons between clauses split into sentences (34) - a numbered list whose items no longer read as continuations (3) - two Oxford-comma series disambiguated (2) Also applies rules no gate checks: the actor 'the operator' removed from a migration step in favor of the passive, four sentences past 30 words split at their natural break, and three run-on conditions in volume-migration turned into separate list items. No facts changed. Co-Authored-By: Claude Opus 5 (1M context) --- docs/architecture/concepts/failure-domains.md | 16 +++++----- .../high-availability-fault-tolerance.md | 4 +-- .../non-kubernetes/installation/install-sp.md | 4 +-- .../storage-network-multipathing.md | 28 +++++++++--------- .../operations/asynchronous-replication.md | 22 +++++++------- .../operations/backup-recovery.md | 4 +-- .../operations/failure-domains.md | 26 ++++++++--------- .../operations/replacing-storage-node.md | 7 +++-- .../operations/volume-migration.md | 29 ++++++++++--------- docs/non-kubernetes/usage/encrypting.md | 4 +-- docs/reference/limits.md | 14 ++++----- .../troubleshooting/storage-plane.md | 4 +-- 12 files changed, 82 insertions(+), 80 deletions(-) diff --git a/docs/architecture/concepts/failure-domains.md b/docs/architecture/concepts/failure-domains.md index 5bb5d260..67127475 100644 --- a/docs/architecture/concepts/failure-domains.md +++ b/docs/architecture/concepts/failure-domains.md @@ -4,7 +4,7 @@ description: "How simplyblock failure domains group storage nodes by rack, cabin weight: 30750 --- -A failure domain groups storage nodes that share a common infrastructure dependency — a rack, a cabinet, a power +A failure domain groups storage nodes that share a common infrastructure dependency: a rack, a cabinet, a power distribution unit, or an availability zone. When failure domains are enabled, simplyblock spreads data chunks, journal copies, and failover paths across the domains so that the loss of one entire domain does not interrupt the availability of the cluster. @@ -44,8 +44,8 @@ For the failover paths, the guaranteed invariant is: With two failure domains and three paths (primary, secondary, tertiary), it is mathematically impossible to place all three paths in distinct domains. Simplyblock therefore guarantees at least one cross-domain failover path per -volume store — enough to survive a full domain outage — and places the remaining paths cross-domain wherever the -topology allows it. +volume store, which is enough to survive a full domain outage. The remaining paths are placed cross-domain +wherever the topology allows it. At cluster activation, simplyblock arranges the hosts in a round-robin order across the failure domains and derives all secondary and tertiary assignments from this interleaved rotation. On a cluster with equally sized domains, this @@ -61,7 +61,7 @@ Failure-domain placement only works if the domains stay comparable in size. Simp removing a node is refused if it would unbalance the domains further. - Every domain must keep at least **two hosts** once the cluster holds data. -A cluster with a one-host imbalance stays fully within the availability contract: exactly one volume store then has +A cluster with a one-host imbalance stays fully within the availability contract. Exactly one volume store then has a same-domain secondary path, and its tertiary path is still guaranteed to be cross-domain. !!! note @@ -92,14 +92,14 @@ prevents accidental topology changes that would silently invalidate the placemen Failure domains also change how the cluster recovers from large outages: -- An outage confined to one domain — up to and including every node of the domain — keeps the cluster **degraded +- An outage confined to one domain (up to and including every node of the domain) keeps the cluster **degraded but serving**. The cluster is not suspended. - With two parity chunks, the cluster additionally tolerates the loss of one entire domain **plus** one further node or device outage in exactly one other domain. - When a whole domain returns from an outage (for example, after a rack power loss), its nodes are restarted **in parallel** instead of strictly one-by-one, substantially shortening the recovery of large domains. -For operating instructions — creating a failure-domain cluster, adding and removing nodes, and expansion rules — -see [Managing Failure Domains](../../non-kubernetes/operations/failure-domains.md). For Kubernetes-based -deployments, failure domains are assigned through the Simplyblock Operator; see the +For operating instructions (cluster creation, node addition, node removal, and the expansion rules), see +[Managing Failure Domains](../../non-kubernetes/operations/failure-domains.md). For Kubernetes-based +deployments, failure domains are assigned through the Simplyblock Operator. See the [Operator Reference](../../reference/operator/index.md). diff --git a/docs/architecture/high-availability-fault-tolerance.md b/docs/architecture/high-availability-fault-tolerance.md index c6f98a07..793ed3f5 100644 --- a/docs/architecture/high-availability-fault-tolerance.md +++ b/docs/architecture/high-availability-fault-tolerance.md @@ -64,8 +64,8 @@ availability of the logical volume. In addition to the node-level path redundancy, simplyblock supports multipathing across independent **storage networks**: storage nodes attached with multiple data interfaces in separate VLANs expose every path on every network, multiplying the number of client connections (for example, four connections with one failover path and -two networks, or six with two failover paths). This provides an alternative to link aggregation (LACP, MLAG) — -see [Storage Network Multipathing](../non-kubernetes/installation/storage-network-multipathing.md). +two networks, or six with two failover paths). This provides an alternative to link aggregation (LACP, MLAG). +See [Storage Network Multipathing](../non-kubernetes/installation/storage-network-multipathing.md). ### 3. Redundant Control Plane and Storage Plane diff --git a/docs/non-kubernetes/installation/install-sp.md b/docs/non-kubernetes/installation/install-sp.md index b2470958..df65f492 100644 --- a/docs/non-kubernetes/installation/install-sp.md +++ b/docs/non-kubernetes/installation/install-sp.md @@ -142,7 +142,7 @@ If separate NICs (e.g., a BOND device, or dedicated interfaces per storage VLAN) (no matter if in the cluster or between hosts and cluster nodes), the `--data-nics` parameter must be specified. Without it, the management interface carries all traffic. Multiple interfaces are given as a comma-separated list (e.g., `--data-nics eth1,eth2`), in which case all NVMe-oF subsystems listen on every data interface and -connections are established once per interface — see +connections are established once per interface. See [Storage Network Multipathing](storage-network-multipathing.md). !!! info @@ -153,7 +153,7 @@ connections are established once per interface — see storage location. If the cluster was created with failure-domain support, every node must additionally be tagged with its -failure-domain id via `--failure-domain `; see +failure-domain id via `--failure-domain `. See [Managing Failure Domains](../operations/failure-domains.md). The output will look something like the following example: diff --git a/docs/non-kubernetes/installation/storage-network-multipathing.md b/docs/non-kubernetes/installation/storage-network-multipathing.md index bb8cf49d..234d0066 100644 --- a/docs/non-kubernetes/installation/storage-network-multipathing.md +++ b/docs/non-kubernetes/installation/storage-network-multipathing.md @@ -7,7 +7,7 @@ weight: 36000 Simplyblock supports two ways to make the storage network redundant: - A **redundant network** below a single interface, built with link aggregation (LACP), stacked switches, MLAG, or - active/passive bonding. Simplyblock sees one data interface; the redundancy is handled entirely in the network + active/passive bonding. Simplyblock sees one data interface. The redundancy is handled entirely in the network layer. - **NVMe-oF multipathing** over two (or more) independent storage networks. Each storage node is attached with multiple data interfaces in separate VLANs or subnets, routed over separate NIC ports and switches. Simplyblock @@ -53,9 +53,9 @@ The data interfaces of a storage node are declared when the node is attached to The interface list is comma-separated without spaces (`eth1,eth2`). If `--data-nics` is omitted, the management interface carries the storage traffic and no multipathing is available. -There is no separate switch to enable multipathing: as soon as a node has more than one usable data interface, -all of its NVMe-oF subsystems — logical volumes as well as cluster-internal device and journal subsystems — listen -on every data interface, and all connections to the node are established once per interface. +There is no separate switch to enable multipathing. As soon as a node has more than one usable data interface, all +of its NVMe-oF subsystems listen on every data interface, and all connections to the node are established once per +interface. These subsystems include logical volumes as well as cluster-internal device and journal subsystems. Multipathing applies per node, but a consistent configuration across all nodes is strongly recommended: use the same number of data interfaces, in the same set of VLANs, on every storage node. @@ -64,18 +64,18 @@ same number of data interfaces, in the same set of VLANs, on every storage node. With multipathing, `{{ cliname }} volume connect` returns one `nvme connect` command per combination of node and data interface. A volume with one failover path (erasure coding with one parity chunk) on nodes with two data -interfaces yields **four** connection strings; with two failover paths (two parity chunks), **six**: +interfaces yields **four** connection strings. With two failover paths (two parity chunks), it yields **six**: ```bash title="Retrieve all connection strings for a volume" {{ cliname }} volume connect ``` Run **all** returned `nvme connect` commands on the host. The commands connect the same NVMe subsystem (the same -NQN) over the different paths; the Linux kernel's native NVMe multipathing merges them into a single block device +NQN) over the different paths. The Linux kernel's native NVMe multipathing merges them into a single block device and steers I/O based on the ANA (Asymmetric Namespace Access) state that simplyblock manages per path. No `dm-multipath` configuration is required or supported. -If a path fails — a NIC, a switch, or an entire network — the kernel transparently continues on the remaining +If a path fails (a NIC, a switch, or an entire network), the kernel transparently continues on the remaining paths. When a primary node fails over to a secondary node, simplyblock switches the ANA states, and the host follows without a reconnect. @@ -83,11 +83,11 @@ follows without a reconnect. After attaching the nodes, verify that all paths exist: -1. `{{ cliname }} storage-node list --json` — every node reports all of its data interfaces (`data_nics`). -2. `{{ cliname }} storage-node port-list ` — lists the data interfaces of a node. -3. `{{ cliname }} volume connect ` — returns one connection string per node and interface (for +1. `{{ cliname }} storage-node list --json`: every node reports all of its data interfaces (`data_nics`). +2. `{{ cliname }} storage-node port-list `: lists the data interfaces of a node. +3. `{{ cliname }} volume connect `: returns one connection string per node and interface (for example, four entries for a volume with one failover path on dual-interface nodes). -4. `{{ cliname }} storage-node check ` — verifies all NVMe-oF connections to and from the node, +4. `{{ cliname }} storage-node check `: verifies all NVMe-oF connections to and from the node, including all paths of the cluster-internal connections. Per-interface I/O statistics are available with `{{ cliname }} storage-node port-io-stats `. @@ -96,7 +96,7 @@ Per-interface I/O statistics are available with `{{ cliname }} storage-node port In Kubernetes-based deployments, the data interfaces are declared in the `StorageNodeSet` resource: the `dataIfname` field takes a list of interface names, equivalent to `--data-nics`. Volume connections made by the -CSI driver automatically use all paths; no storage-class parameter is required. See the +CSI driver automatically use all paths. No storage-class parameter is required. See the [Operator Reference](../../reference/operator/index.md) for details. ## Interaction with Failure Domains and Migration @@ -105,5 +105,5 @@ CSI driver automatically use all paths; no storage-class parameter is required. that combine naturally: failure domains protect against the loss of a rack or site, multipathing against the loss of a network path. - During a [volume migration](../operations/volume-migration.md), the target subsystem is exposed on all data - interfaces of the target node. The client must connect all returned target paths before continuing the - migration, so that the cutover is seamless on every path. + interfaces of the target node. All returned target paths must be connected on the client before the migration + is continued, so that the cutover is seamless on every path. diff --git a/docs/non-kubernetes/operations/asynchronous-replication.md b/docs/non-kubernetes/operations/asynchronous-replication.md index be041f2c..6eeb0790 100644 --- a/docs/non-kubernetes/operations/asynchronous-replication.md +++ b/docs/non-kubernetes/operations/asynchronous-replication.md @@ -1,6 +1,6 @@ --- title: "Asynchronous Replication" -description: "Asynchronous replication between simplyblock clusters: disaster recovery, cross-cluster volume migration, failover and failback." +description: "Asynchronous replication between simplyblock clusters: disaster recovery, cross-cluster volume migration, failover, and failback." weight: 20045 --- @@ -20,7 +20,7 @@ Kubernetes environments, where replication is managed through the `SnapshotRepli ## Prerequisites - **Both clusters are managed by the same control plane.** The first cluster is created with - `{{ cliname }} cluster create`; the second is attached to the same control plane with `{{ cliname }} cluster add`. + `{{ cliname }} cluster create`. The second is attached to the same control plane with `{{ cliname }} cluster add`. - The storage nodes of the source cluster can reach the storage nodes of the target cluster over the storage network: replication transfers data directly between the nodes over NVMe-oF. - Both clusters are activated, and both have an active storage pool. @@ -55,9 +55,9 @@ run the command once in each direction. - `--mode migration`: planned cutover. The target subsystem is pre-created up front (inaccessible), and the volume is cut over on an explicit commit. - `--interval-min `: take an internal replication snapshot every `N` minutes (the first one immediately). - `0` disables interval snapshots; then only user-created snapshots replicate. + `0` disables interval snapshots. Only user-created snapshots then replicate. -Every snapshot of a replicated volume — interval-based or user-created — is queued for transfer to the target +Every snapshot of a replicated volume (interval-based or user-created) is queued for transfer to the target cluster. Snapshots that were taken before replication was enabled are transferred as well. The achievable recovery point (RPO) is roughly the snapshot interval plus the transfer time. @@ -78,9 +78,9 @@ To take an immediate replication snapshot outside the interval: ``` The output shows the last snapshot, the last completed replication and its duration, the number of replicated -snapshots, the **time lag** (the age of the newest point-in-time that exists on the target — the actual RPO), and -the outstanding backlog (count and bytes of not-yet-replicated snapshots). A volume is caught up when the -outstanding count is zero. +snapshots, the **time lag**, and the outstanding backlog (count and bytes of not-yet-replicated snapshots). The +time lag is the age of the newest point-in-time that exists on the target, which is the actual RPO. A volume is +caught up when the outstanding count is zero. ```bash title="All replication tasks of a cluster" {{ cliname }} volume replication-status @@ -103,7 +103,7 @@ Wait until `{{ cliname }} volume replication-info ` reports an outsta ``` The commit takes a final snapshot to minimize the delta, builds the target volume on the last replicated -snapshot — with the **same NQN and namespace ID** as the source — exposes it as inaccessible, and queues the final +snapshot (with the **same NQN and namespace ID** as the source), exposes it as inaccessible, and queues the final cutover task. The cutover freezes source I/O, transfers the residual delta, and flips the ANA states so that the client fails over to the target paths without a disconnect. @@ -113,7 +113,7 @@ client fails over to the target paths without a disconnect. `nvme connect` commands on the client before the cutover task completes. Because source and target expose the same NQN and namespace ID, the new paths join the existing multipath device. -Since continuous replication keeps the backlog small, the final freeze only covers the residual delta — typically +Since continuous replication keeps the backlog small, the final freeze only covers the residual delta, typically a fraction of a second to a few seconds. ## Failover (Disaster Recovery) @@ -132,7 +132,7 @@ with the same device identity. !!! warning Data written on the source after the last successfully replicated snapshot is not available on the target. - The data gap is at most the replication interval plus the replication lag; check + The data gap is at most the replication interval plus the replication lag. Check `volume replication-info` to see the effective lag. Unlike the planned cutover, a failover interrupts I/O: workloads must reconnect (and typically restart) against the target paths. @@ -159,7 +159,7 @@ interruption-free cutback. ``` Stopping cancels the pending replication tasks of the volume and disables further snapshots from replicating. The -already replicated snapshots on the target are kept; they can be removed individually without touching the source +already replicated snapshots on the target are kept. They can be removed individually without touching the source snapshot: ```bash title="Delete only the replicated copy of a snapshot" diff --git a/docs/non-kubernetes/operations/backup-recovery.md b/docs/non-kubernetes/operations/backup-recovery.md index 17a9fab9..94892874 100644 --- a/docs/non-kubernetes/operations/backup-recovery.md +++ b/docs/non-kubernetes/operations/backup-recovery.md @@ -53,7 +53,7 @@ Restoring a backup creates a new logical volume with the data reconstructed from ``` The `--lvol`, `--pool`, and `--cluster-id` parameters are required. Any node of the cluster can restore any -backup; without `--node`, the node that took the backup is used. +backup. Without `--node`, the node that took the backup is used. The restore process downloads and applies each backup in the chain. The new volume is set to a restoring state during the transfer and transitions to online once complete. @@ -161,7 +161,7 @@ bucket: {{ cliname }} backup source-switch [--cluster-id ] ``` -The switch changes only the bucket that is read; the target cluster's own S3 credentials and endpoint are reused, +The switch changes only the bucket that is read. The target cluster's own S3 credentials and endpoint are reused, so they must have access to the source cluster's bucket. To list available backup sources: ```bash title="List backup sources" diff --git a/docs/non-kubernetes/operations/failure-domains.md b/docs/non-kubernetes/operations/failure-domains.md index 75ea5afc..57eccf2f 100644 --- a/docs/non-kubernetes/operations/failure-domains.md +++ b/docs/non-kubernetes/operations/failure-domains.md @@ -11,7 +11,7 @@ the placement guarantees are described in This page describes how to deploy and operate a failure-domain cluster with the CLI. In Kubernetes environments, failure domains are assigned declaratively through the Simplyblock Operator -(`enableFailureDomains` on the `StorageCluster` and `failureDomain` per node); see the +(`enableFailureDomains` on the `StorageCluster` and `failureDomain` per node). See the [Operator Reference](../../reference/operator/index.md). ## Enabling Failure Domains @@ -31,8 +31,8 @@ plane. ## Tagging Storage Nodes -On a failure-domain cluster, every storage node must be added with a failure-domain id — a non-negative integer -identifying the rack, cabinet, or availability zone. All nodes in the same physical fault group share the same id. +On a failure-domain cluster, every storage node must be added with a failure-domain id (a non-negative integer +identifying the rack, cabinet, or availability zone). All nodes in the same physical fault group share the same id. ```bash title="Add storage nodes with failure-domain tags" # Rack A (domain 0) @@ -58,13 +58,13 @@ The assigned domains are shown in the node list once at least one node carries a Activating a freshly assembled failure-domain cluster enforces the following rules: -| Rule | Enforcement | -|-----------------------------------------------|--------------------------------------------------------------------------------------| -| Every node carries a failure-domain id | Hard — activation fails | -| A host does not span two domains | Hard — activation fails | -| At least two distinct domains exist | Hard — activation fails | -| All domains hold an equal number of hosts | Hard — activation fails | -| At least `parity chunks + 1` distinct domains | Recommendation — a warning is logged, activation proceeds with best-effort placement | +| Rule | Enforcement | +|-----------------------------------------------|-------------------------------------------------------------------------------------| +| Every node carries a failure-domain id | Hard: activation fails | +| A host does not span two domains | Hard: activation fails | +| At least two distinct domains exist | Hard: activation fails | +| All domains hold an equal number of hosts | Hard: activation fails | +| At least `parity chunks + 1` distinct domains | Recommendation: a warning is logged, activation proceeds with best-effort placement | During activation, simplyblock computes the interleaved host rotation across the domains and assigns all secondary and tertiary failover paths from it. Re-activation of an existing cluster (for example, during disaster recovery) @@ -88,7 +88,7 @@ journal quorum. Once the cluster holds data, topology changes are admitted only if the failure domains stay balanced: - The host count per domain may never diverge by more than one (±1 rule). On a balanced cluster, one host can be - added to any domain; the next host must then go to a different domain. + added to any domain. The next host must then go to a different domain. - No domain may drop below two hosts. - Adding another storage node slot on an already-member host (multi-socket systems) is balance-neutral and always admitted, as long as the host keeps its original domain id. @@ -111,7 +111,7 @@ before any change is made. !!! important On clusters with a single parity chunk (FTT 1), an odd total host count cannot satisfy the cross-domain - invariant, because there is no tertiary path to fall back on. Grow such clusters in pairs — one host per + invariant, because there is no tertiary path to fall back on. Grow such clusters in pairs, one host per domain at a time. ## Removing a Storage Node @@ -119,7 +119,7 @@ before any change is made. Node removal applies the same balance rules (±1, minimum two hosts per domain). In addition, the failover paths hosted by the node being removed are relocated to other nodes. If the path being relocated is the only cross-domain path of its volume store, the replacement node **must** be in a different failure domain than the -primary; if no such node exists, the removal is refused. +primary. If no such node exists, the removal is refused. ## Moving a Host Between Domains diff --git a/docs/non-kubernetes/operations/replacing-storage-node.md b/docs/non-kubernetes/operations/replacing-storage-node.md index 1903b827..b5dc47a8 100644 --- a/docs/non-kubernetes/operations/replacing-storage-node.md +++ b/docs/non-kubernetes/operations/replacing-storage-node.md @@ -31,9 +31,10 @@ To start a new storage node, follow the storage node installation according to t ## Remove the old Storage Node !!! important - A storage node can only be removed when it hosts no logical volumes or snapshots. Migrate all volumes off the - node first (see [Volume Migration](volume-migration.md)); the removal is refused otherwise. The node to be - removed must be online or suspended, and all other storage nodes must be online. + A storage node can only be removed when it hosts no logical volumes or snapshots. Otherwise the removal is + refused, so all volumes have to be migrated off the node first (see + [Volume Migration](volume-migration.md)). The node to be removed must be online or suspended, and all other + storage nodes must be online. To remove the old storage node, use the `{{ cliname }}` command line tool. diff --git a/docs/non-kubernetes/operations/volume-migration.md b/docs/non-kubernetes/operations/volume-migration.md index 6640e33b..0840343d 100644 --- a/docs/non-kubernetes/operations/volume-migration.md +++ b/docs/non-kubernetes/operations/volume-migration.md @@ -4,14 +4,14 @@ description: "Migrate a logical volume between storage nodes with the simplybloc weight: 20040 --- -Simplyblock can move a logical volume — including its snapshots — from one storage node to another while the +Simplyblock can move a logical volume (including its snapshots) from one storage node to another while the volume stays online. I/O is only frozen for the brief moment needed to transfer the final delta at the end of the migration. This page describes the CLI-driven migration between nodes of the **same cluster**. For moving volumes between **clusters**, see [Asynchronous Replication](asynchronous-replication.md), which provides a replication-based cross-cluster migration. In Kubernetes environments, migrations are managed declaratively -through the `VolumeMigration` resource; see +through the `VolumeMigration` resource. See [Volume Migration on Kubernetes](../../kubernetes/operations/volume-migration.md). ## How a Migration Works @@ -21,8 +21,8 @@ A migration is a two-step operation with a client action in between: 1. `volume migrate` **pre-creates** the target: the NVMe-oF subsystem for the volume is created on the target node with the same NQN as on the source, with all listeners in the ANA state `inaccessible`. The command returns a migration ID and the NVMe connect strings for the new target paths. -2. The operator runs the returned `nvme connect` commands **on the client**. The new paths join the client's - native NVMe multipath for the volume; because they are `inaccessible`, they carry no I/O yet. +2. The returned `nvme connect` commands are run **on the client**. The new paths join the client's native NVMe + multipath for the volume. Because they are `inaccessible`, they carry no I/O yet. 3. `volume migrate-continue` starts the data transfer. The snapshot chain is copied oldest-first, the live delta is progressively shrunk with intermediate snapshots, and the final delta is transferred under a short I/O freeze. At cutover, the ANA states flip: the target paths become active and the source paths become @@ -54,8 +54,8 @@ sudo nvme connect --transport=tcp --traddr= --trsvcid= --nqn= ``` -`migrate-continue` accepts `--max-retries ` (default 10) and `--deadline ` (default 14400; `0` -disables the deadline). +`migrate-continue` accepts `--max-retries ` (default 10) and `--deadline ` (default 14400). Setting +the deadline to `0` disables it. If the volume has host authentication configured (DH-HMAC-CHAP), pass the client's host NQN to the pre-create step with `--host-nqn `. @@ -74,7 +74,7 @@ counter, and the last error, if any. | `pre_created` | Target subsystem exists, waiting for `migrate-continue`. | | `snap_copy` | The snapshot chain is being copied to the target. | | `lvol_migrate` | The final delta is being transferred. This is the only phase with a (short) I/O freeze. | -| `cleanup_source` | Data has moved; source-side objects are being removed. | +| `cleanup_source` | Data has moved. Source-side objects are being removed. | | `cleanup_target` | Rollback after a failure or cancellation: target-side objects are being removed. | | `completed` | The migration has finished. | @@ -88,15 +88,15 @@ A volume that is part of an active migration shows the migration ID in the `migr ``` A migration canceled in the `pre_created` phase is torn down immediately. In later phases, the cancellation is -picked up asynchronously by the migration runner, which rolls the target back (`cleanup_target`); it may take a +picked up asynchronously by the migration runner, which rolls the target back (`cleanup_target`). It may take a few seconds to reflect in `migrate-list`. Data on the source remains intact and authoritative until the final cutover, so a migration can be canceled at any phase before `cleanup_source`. ## Migrating Shared Subsystems (Batch Migration) Volumes that share one NVMe-oF subsystem (namespaced volumes) can only be migrated together. Pass `--batch` with -any member volume; simplyblock migrates all volumes of the subsystem as one coordinated group and returns a -migration group ID, which is then used with `--batch` on the other commands: +any member volume. Simplyblock then migrates all volumes of the subsystem as one coordinated group and returns a +migration group ID, which is used with `--batch` on the other commands: ```bash title="Migrate all volumes of a shared subsystem" {{ cliname }} volume migrate --batch @@ -113,16 +113,17 @@ A migration is admitted only if: - The cluster is active and not currently rebalancing (no device migration or post-restart rebalancing tasks are running). -- The volume is online; the target node is online and different from the source node; the source node is online - or suspended. +- The volume is online. +- The target node is online and different from the source node. +- The source node is online or suspended. - The volume has no other active migration. Re-running `volume migrate` with the same volume and target returns - the existing migration ID; a different target requires canceling the existing migration first. + the existing migration ID. A different target requires canceling the existing migration first. Additional operational constraints while a migration is active: - **Snapshots of volumes on the source node cannot be created** until the migration completes. - New volumes cannot be attached to a subsystem that has an active migration. -- The erasure coding scheme of the volume is preserved; it is not re-negotiated on the target. +- The erasure coding scheme of the volume is preserved. It is not re-negotiated on the target. - Simplyblock does not pre-check the free capacity of the target node. Ensure the target has enough capacity for the volume and its snapshots before starting the migration. diff --git a/docs/non-kubernetes/usage/encrypting.md b/docs/non-kubernetes/usage/encrypting.md index 7dfe672b..138e12aa 100644 --- a/docs/non-kubernetes/usage/encrypting.md +++ b/docs/non-kubernetes/usage/encrypting.md @@ -29,13 +29,13 @@ functionality. The encryption uses an AES_XTS variable-length block cipher. The encryption keys are created and stored by the cluster's key management system (KMS) when the volume is -created. By default, simplyblock manages the keys internally; alternatively, an external KMS (HashiCorp Vault or +created. By default, simplyblock manages the keys internally. Alternatively, an external KMS (HashiCorp Vault or OpenBao) can be configured at cluster creation time via `--hashicorp-vault-url`. See [External Key Management](../../architecture/concepts/external-key-management.md) for the architecture. !!! note Earlier releases required manually generated keys passed via `--crypto-key1` and `--crypto-key2`. These - parameters are deprecated since 26.2 and cannot be used anymore; key handling is fully KMS-based. + parameters are deprecated since 26.2 and cannot be used anymore. Key handling is fully KMS-based. ## Creating an Encrypted Logical Volume diff --git a/docs/reference/limits.md b/docs/reference/limits.md index c021298d..b61b423d 100644 --- a/docs/reference/limits.md +++ b/docs/reference/limits.md @@ -5,14 +5,14 @@ weight: 20140 --- Simplyblock enforces a set of limits per storage node and per cluster. Some are hard limits built into the -control plane; others depend on the node's vCPU count and memory configuration. +control plane. Others depend on the node's vCPU count and memory configuration. ## Hard Per-Node Object Limits | Limit | Value | What it counts | |-----------------------------|------:|-----------------------------------------------------------------------------------------------------| | Objects per node | 6000 | Logical volumes, clones, and snapshots owned by the node (its logical volume store) | -| NVMe-oF subsystems per node | 75 | Subsystems for which the node is the primary; namespaced volumes sharing one subsystem count as one | +| NVMe-oF subsystems per node | 75 | Subsystems for which the node is the primary. Namespaced volumes sharing one subsystem count as one | | Namespaces per subsystem | 50 | Volumes (namespaces) sharing one NVMe-oF subsystem | These limits are enforced on every create path (volume create, snapshot create, clone). When a limit is reached, @@ -27,8 +27,8 @@ max_namespace_per_subsys=64 exceeds the hard limit of 50 namespaces per subsyste Notes on what counts against the limits: - Only the **primary** node of a volume is charged. Failover copies on secondary and tertiary nodes do not count - against those nodes' limits — their resource reservation already provisions for them. -- Deleted objects do not count; objects in creation or deletion still do. + against those nodes' limits, because their resource reservation already provisions for them. +- Deleted objects do not count. Objects in creation or deletion still do. - When volume placement finds no node below its subsystem limit, volume creation fails with `No nodes found with enough resources to create the LVol`. @@ -46,8 +46,8 @@ It can be changed later via `{{ cliname }} storage-node restart --max-subsys ## Namespaces per Subsystem -By default, simplyblock places each volume in its own NVMe-oF subsystem. Namespaced volumes share a subsystem; -the default maximum is **32 namespaces per subsystem**, configurable per volume at creation time up to the hard +By default, simplyblock places each volume in its own NVMe-oF subsystem. Namespaced volumes share a subsystem. +The default maximum is **32 namespaces per subsystem**, configurable per volume at creation time up to the hard ceiling of 50: ```bash title="Create a namespaced volume with a custom namespace limit" @@ -65,7 +65,7 @@ On top of the hard object limits, several resource limits scale with the vCPU co |--------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| | CPU cores per storage node | At most 64 cores can be assigned to one storage node (SPDK instance). | | Distribution services per node | Scales with the assigned cores, capped at 12. | -| NVMe-oF buffer pools | Scale with core count and `--max-subsys`; they determine part of the huge-page demand. | +| NVMe-oF buffer pools | Scale with core count and `--max-subsys`. They determine part of the huge-page demand. | | Huge-page memory | The minimum huge-page memory grows with the core count and the configured maximum number of subsystems. Nodes refuse to start with insufficient huge pages. | | Storage nodes per host | 1 or 2 (`--nodes-per-socket`), aligned to NUMA sockets. | diff --git a/docs/reference/troubleshooting/storage-plane.md b/docs/reference/troubleshooting/storage-plane.md index 11846d59..bce6d80a 100644 --- a/docs/reference/troubleshooting/storage-plane.md +++ b/docs/reference/troubleshooting/storage-plane.md @@ -10,8 +10,8 @@ weight: 30200 storage nodes show `n/0` disks available in the disks column (`{{ cliname }} storage-node list`). 1. Remove all storage nodes: `{{ cliname }} storage-node remove `. The node must be online or - suspended; the removal shuts the node down itself. (`--force-remove` does not force a removal — it only - cancels active tasks of the node.) + suspended, and the removal shuts the node down itself. `--force-remove` does not force a removal, but only + cancels active tasks of the node. 2. Delete all storage nodes: `{{ cliname }} storage-node delete ` 3. Re-add all storage nodes. The disks should become active. 4. Try to activate the cluster. From f0eadcffb1b65516cebda4c1c948f9d2d6d760a2 Mon Sep 17 00:00:00 2001 From: "Christoph Engelbert (noctarius)" Date: Sat, 8 Aug 2026 13:10:33 +0200 Subject: [PATCH 5/7] docs: gate the multipathing spelling and the vCPU casing The documentation had 41 'multipathing' against 3 'multi-pathing', and 32 'vCPU' against 7 lowercase spellings, none of which any gate reported. 'multi-pathing' goes to check-prose.py next to hyper-converged and disaggregated: it is a hyphenation, and that check matches case insensitively while preserving an initial capital, so the 16 headings spelling it 'Multipathing' stay correct. The terminology gate compares the canonical form byte for byte and would have failed every one of them. 'vCPU' goes to check-terminology.py next to CPU, where the mixed casing is the point. The seven lowercase occurrences are all '--vcpu-count' and '--number-of-vcpus' inside code spans, which the check ignores, so the rule reports nothing today and catches the next one written in prose. Fixes the two occurrences in the release notes. A third, in docs/reference/cli/volume.md, is generated and has to be fixed in sbcli. Co-Authored-By: Claude Opus 5 (1M context) --- docs/release-notes/25-10-5.md | 2 +- docs/release-notes/26-2.md | 2 +- scripts/check-prose.py | 4 ++++ scripts/check-terminology.py | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/release-notes/25-10-5.md b/docs/release-notes/25-10-5.md index 98645d12..0ec2fd3f 100644 --- a/docs/release-notes/25-10-5.md +++ b/docs/release-notes/25-10-5.md @@ -43,6 +43,6 @@ It is possible to upgrade from `25.10.4` and `25.10.4.2`. - The ability to asynchronously replicate volumes via snapshots in regular intervals and support fail-over in Kubernetes. - The integration via a new Simplyblock Kubernetes Operator which uses CRDs to specify, create, and track a cluster, storage nodes, volumes, snapshots, and replications. - A significant performance optimization during node outages (journal writes). -- The integration of cluster-internal multi-pathing for both RDMA- and TCP-based NVMe connections. +- The integration of cluster-internal multipathing for both RDMA- and TCP-based NVMe connections. - The ability to send snapshot backups to remote S3-compatible storage. - The ability to support multiple NVMe-oF failover connections with one primary and two secondary storage nodes for higher availability. diff --git a/docs/release-notes/26-2.md b/docs/release-notes/26-2.md index 9fbe8eaf..b129919a 100644 --- a/docs/release-notes/26-2.md +++ b/docs/release-notes/26-2.md @@ -13,7 +13,7 @@ Simplyblock is happy to release the general availability release of Simplyblock - Storage Plane: Added support for remote snapshot replication. Snapshots can be asynchronously replicated btw. nodes and sites in a network-efficient manner. - Storage Plane: Added support for asynchronous replication which adds the ability to automatically failing-over and failing-back of selected volumes across sites with a certain time-gap/backlog (e.g., 5 minutes). This is useful for slow links and provides significantly better RTO (zero) and RPO (minimum: 1 minute) than traditional backups. - Storage Plane: Added support for FTT2 (failure to tolerate) which enables up to two failover paths. When enabled, simplyblock creates three communication paths between the initiators and the NVMe targets (primary, secondary, and tertiary) with identical subsystems able to process I/O. This allows the loss or maintenance of any two nodes in the cluster, regardless of the combination, at a point in time. -- Storage Plane: Added support for full multi-pathing within the storage cluster and between initiators and NVMe targets. This means that clients can have four (FTT=1) or six (FTT=2) connections via separate VLANs using separate networking paths. Each endpoint-combination uses two separate connections through the available network paths. It has advantages over simple bonding and multi-chassis link aggregation groups (MLAG). +- Storage Plane: Added support for full multipathing within the storage cluster and between initiators and NVMe targets. This means that clients can have four (FTT=1) or six (FTT=2) connections via separate VLANs using separate networking paths. Each endpoint-combination uses two separate connections through the available network paths. It has advantages over simple bonding and multi-chassis link aggregation groups (MLAG). - Kubernetes: Added support for OpenShift-managed TLS certificates for communication within the control plane and cluster endpoints. - Kubernetes: Added support for mutual TLS (mTLS) on the control plane, using cert-manager-issued certificates. - Kubernetes: Added support for offloading volume encryption keys to an external KMS (Hashicorp Vault or OpenBao). diff --git a/scripts/check-prose.py b/scripts/check-prose.py index 5195ad06..0de81d10 100755 --- a/scripts/check-prose.py +++ b/scripts/check-prose.py @@ -90,6 +90,10 @@ "disagregated": "disaggregated", "dissagregated": "disaggregated", "dissaggregated": "disaggregated", "dis-aggregation": "disaggregation", "dissaggregation": "disaggregation", + # Multipathing is one word, in the NVMe specification as well as in the + # "dm-multipath" and "nvme multipath" spellings it is written next to. + "multi-pathing": "multipathing", "multi pathing": "multipathing", + "multi-path": "multipath", "multi path": "multipath", } MISSPELLING_PATTERN = re.compile( r"\b(?:" diff --git a/scripts/check-terminology.py b/scripts/check-terminology.py index 802014ea..b77a41f2 100755 --- a/scripts/check-terminology.py +++ b/scripts/check-terminology.py @@ -274,6 +274,7 @@ def term(canonical, aliases=(), plural="", wrong=()): "PVC", # Hardware. "CPU", + "vCPU", "GPU", "RAM", "NVIDIA", From b8e7db2116eea1f53b6285ed18e2461f700d5679 Mon Sep 17 00:00:00 2001 From: "Christoph Engelbert (noctarius)" Date: Sat, 8 Aug 2026 13:17:06 +0200 Subject: [PATCH 6/7] docs: gate the FTT spelling The fault-tolerance level is written 'FTT', 'FTT 1', 'FTT 2', 'FTT=1' or 'FTT=2'. 'FTT+1' is a separate thing and stays: it means the level plus one node, not a level of its own. The casing goes to check-terminology.py, which is what that gate is for. The glued and hyphenated spellings go to check-prose.py instead, because a separator has to be inserted rather than a casing changed, and that check is the one that rewrites a word into a different word. Fixes 'FTT2' in the 26.2 release notes. A second, in docs/reference/cli/storage-node.md, is generated and has to be fixed in sbcli; the 'FTT+1' on that same line is correct and must stay. Co-Authored-By: Claude Opus 5 (1M context) --- docs/release-notes/26-2.md | 2 +- scripts/check-prose.py | 6 ++++++ scripts/check-terminology.py | 5 +++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/26-2.md b/docs/release-notes/26-2.md index b129919a..d3f66d96 100644 --- a/docs/release-notes/26-2.md +++ b/docs/release-notes/26-2.md @@ -12,7 +12,7 @@ Simplyblock is happy to release the general availability release of Simplyblock - Storage Plane: Added support volume backup and restore. The new mechanism takes backups of snapshots and sends the snapshot-chains to an S3-compatible storage. Backup retention, merge policies, and auto-backup schedules are configured via the operator. Snapshots can be restored into volumes (PVCs) on the same cluster or into a different cluster. Backup is storage-efficient (delta-only, compressed). - Storage Plane: Added support for remote snapshot replication. Snapshots can be asynchronously replicated btw. nodes and sites in a network-efficient manner. - Storage Plane: Added support for asynchronous replication which adds the ability to automatically failing-over and failing-back of selected volumes across sites with a certain time-gap/backlog (e.g., 5 minutes). This is useful for slow links and provides significantly better RTO (zero) and RPO (minimum: 1 minute) than traditional backups. -- Storage Plane: Added support for FTT2 (failure to tolerate) which enables up to two failover paths. When enabled, simplyblock creates three communication paths between the initiators and the NVMe targets (primary, secondary, and tertiary) with identical subsystems able to process I/O. This allows the loss or maintenance of any two nodes in the cluster, regardless of the combination, at a point in time. +- Storage Plane: Added support for FTT 2 (failure to tolerate) which enables up to two failover paths. When enabled, simplyblock creates three communication paths between the initiators and the NVMe targets (primary, secondary, and tertiary) with identical subsystems able to process I/O. This allows the loss or maintenance of any two nodes in the cluster, regardless of the combination, at a point in time. - Storage Plane: Added support for full multipathing within the storage cluster and between initiators and NVMe targets. This means that clients can have four (FTT=1) or six (FTT=2) connections via separate VLANs using separate networking paths. Each endpoint-combination uses two separate connections through the available network paths. It has advantages over simple bonding and multi-chassis link aggregation groups (MLAG). - Kubernetes: Added support for OpenShift-managed TLS certificates for communication within the control plane and cluster endpoints. - Kubernetes: Added support for mutual TLS (mTLS) on the control plane, using cert-manager-issued certificates. diff --git a/scripts/check-prose.py b/scripts/check-prose.py index 0de81d10..6f64e096 100755 --- a/scripts/check-prose.py +++ b/scripts/check-prose.py @@ -94,6 +94,12 @@ # "dm-multipath" and "nvme multipath" spellings it is written next to. "multi-pathing": "multipathing", "multi pathing": "multipathing", "multi-path": "multipath", "multi path": "multipath", + # The fault-tolerance level is written "FTT 1" or "FTT=1", never glued or + # hyphenated. "FTT+1" is left alone: it means the level plus one node, not a + # level of its own. + "ftt1": "FTT 1", "ftt2": "FTT 2", + "ftt-1": "FTT 1", "ftt-2": "FTT 2", + "ftt_1": "FTT 1", "ftt_2": "FTT 2", } MISSPELLING_PATTERN = re.compile( r"\b(?:" diff --git a/scripts/check-terminology.py b/scripts/check-terminology.py index b77a41f2..2a4114b7 100755 --- a/scripts/check-terminology.py +++ b/scripts/check-terminology.py @@ -230,6 +230,11 @@ def term(canonical, aliases=(), plural="", wrong=()): "S3", "QoS", "IOPS", + # The fault-tolerance level. A number behind it is separated by a space or an + # equals sign ("FTT 2", "FTT=2"), and "FTT+1" means the level plus one. The + # glued and hyphenated spellings are corrected by check-prose.py, since a + # separator has to be inserted rather than a casing changed. + "FTT", term("I/O", wrong=("IO",)), "TCP", "UDP", From 4d1ce2dffa7f14ec1e89c8962468d2f86e3a35fa Mon Sep 17 00:00:00 2001 From: "Christoph Engelbert (noctarius)" Date: Sat, 8 Aug 2026 13:23:18 +0200 Subject: [PATCH 7/7] docs: add LACP, MLAG, RTO, RPO, NQN, and ANA to the terminology gate All six are already spelled correctly throughout the documentation, so the gate reports nothing today and holds the spelling from here on. The lowercase spellings of NQN and ANA are not misspellings but literals: the '--host-nqn' flag, the 'nqn.2023-02.io.simplyblock' subsystem name, the 'nqn' JSON key, and the 'hardware_handler "1 ana"' line of a multipath configuration. The check reads none of them, since a code span is a literal and only the comments of a code block are looked at. The 12 'Nqn' titles in reference/api/openapi.json are not Markdown at all. ANA is given a plural wording for the same reason NVMe has one: it names an access state, so what is plural is the states ('ANA states'), and 'ANAs' is reported with that wording instead of just its casing. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/check-terminology.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/check-terminology.py b/scripts/check-terminology.py index 2a4114b7..10bc3c4e 100755 --- a/scripts/check-terminology.py +++ b/scripts/check-terminology.py @@ -202,6 +202,12 @@ def term(canonical, aliases=(), plural="", wrong=()): term("NVMe-oF", ("NVMe over Fabrics", "NVMf"), plural="NVMe-oF connections"), term("NVMe/TCP", plural="NVMe/TCP connections"), term("NVMe/RDMA", plural="NVMe/RDMA connections"), + # The NVMe qualified name and the multipathing state. Their lowercase + # spellings are the flag, the key and the config value that carry them + # ("--host-nqn", "nqn.2023-02.io.simplyblock", "hardware_handler \"1 ana\""), + # and those are literals the check leaves alone. + "NQN", + term("ANA", plural="ANA states"), "SPDK", "DPDK", "iSCSI", @@ -230,6 +236,8 @@ def term(canonical, aliases=(), plural="", wrong=()): "S3", "QoS", "IOPS", + "RTO", + "RPO", # The fault-tolerance level. A number behind it is separated by a space or an # equals sign ("FTT 2", "FTT=2"), and "FTT+1" means the level plus one. The # glued and hyphenated spellings are corrected by check-prose.py, since a @@ -249,6 +257,8 @@ def term(canonical, aliases=(), plural="", wrong=()): "MTU", "NIC", "CIDR", + "LACP", + "MLAG", # Protocols, formats and interfaces. "TLS", "mTLS",