feat(api): add spec.cluster.privateBackend so flushed parts actually replicate

README.md

@@ -115,6 +115,7 @@ for a fuller example including the S3 backend.
 | `cluster.replicationFactor` | Replicas per write (RF). |
 | `cluster.shardsPerTenant` | Per-tenant series sharding across placement units. |
 | `cluster.peerPort` | Peer replication port (default 7946). |
+| `cluster.privateBackend` | Whether each node's durable backend is its own. Gates `cluster/partsync`, which replicates *flushed parts* and backfills a node that lost its disk — without it only the in-memory head is replicated and every part exists in one copy regardless of RF. Unset derives from `storage.backend`: `true` for `file` (per-pod PVC), `false` for `s3` (shared bucket). Needs an oteldb carrying [oteldb/oteldb#1264](https://github.com/oteldb/oteldb/pull/1264); older builds ignore the key silently. |
 | `cluster.etcdPrefix` | etcd key prefix (storage "root", default `/oteldb`). |
 | `cluster.staticZone` | Fixed failure-domain label for the cluster's nodes (ring zone-spreading). |
 | `signals` | Which signals to serve (all default on). Disabling one drops its backend, its API bind and its ports; disabling all is rejected. |
@@ -194,19 +195,21 @@ config file.
 ## Ports
 
 Client APIs (exposed by the client Service): `4317` OTLP gRPC, `4318` OTLP HTTP, `19291` Prometheus
-remote-write, `9090` PromQL, `3200` TraceQL (Tempo), `3100` LogQL (Loki), `4040` Pyroscope, `8090`
+remote-write, `9090` PromQL, `3200` TraceQL (Tempo), `3100` LogQL (Loki), `4040` Pyroscope, `9464`
 self-metrics, `13133` health. Peer replication: `7946` (headless Service).
 
 The ingest Service (`<name>-ingest`) exposes `4317` OTLP gRPC, `4318` OTLP HTTP, `19291` Prometheus
-remote-write and `8090` self-metrics. `odbingest` serves OTLP/HTTP, remote write and its health
+remote-write and `9464` self-metrics. `odbingest` serves OTLP/HTTP, remote write and its health
 endpoints on **one** listener (`19291`), so the Service publishes `4318` — the port stock OTLP/HTTP
 exporters target — and remaps it onto that listener.
 
 The query Service (`<name>-query`) exposes `9090` PromQL, `3200` TraceQL, `3100` LogQL, `4040`
-Pyroscope and `8090` self-metrics, dropping any whose signal is disabled. `odbselect` serves each on
+Pyroscope and `9464` self-metrics, dropping any whose signal is disabled. `odbselect` serves each on
 its own listener, so these are published one-to-one; its health listener (`13133`) is probed, not
 published.
 
+`8090` is deliberately not exposed: it is oteldb's admin API bind, not a self-metrics endpoint.
+
 ## Getting Started
 
 Prerequisites: Go 1.26+, a Kubernetes cluster, `kubectl`, and (for building the image) Docker.

api/v1alpha1/oteldbcluster_types.go

@@ -456,6 +456,17 @@ type ClusterSpec struct {
 	// +optional
 	ShardsPerTenant *int32 `json:"shardsPerTenant,omitempty"`
 
+	// PrivateBackend declares that every node's durable backend is its own, unshared by peers.
+	// It gates cluster/partsync: only a private backend replicates *flushed parts* between nodes
+	// and backfills a node that lost its disk. Without it the ring replicates just the in-memory
+	// head, so each flushed part exists in exactly one copy regardless of ReplicationFactor.
+	//
+	// Unset derives from Storage.Backend: "file" is per-node PVC and therefore private (true),
+	// "s3" is a shared bucket (false). Set explicitly only to override that — e.g. a per-node
+	// S3 bucket, or a "file" backend on a ReadWriteMany volume shared by all pods.
+	// +optional
+	PrivateBackend *bool `json:"privateBackend,omitempty"`
+
 	// PeerPort is the port peers use to reach each node's replication server.
 	// +kubebuilder:default=7946
 	// +optional

config/crd/bases/db.oteldb.io_oteldbclusters.yaml

@@ -987,6 +987,17 @@ spec:
                       replication server.
                     format: int32
                     type: integer
+                  privateBackend:
+                    description: |-
+                      PrivateBackend declares that every node's durable backend is its own, unshared by peers.
+                      It gates cluster/partsync: only a private backend replicates *flushed parts* between nodes
+                      and backfills a node that lost its disk. Without it the ring replicates just the in-memory
+                      head, so each flushed part exists in exactly one copy regardless of ReplicationFactor.
+
+                      Unset derives from Storage.Backend: "file" is per-node PVC and therefore private (true),
+                      "s3" is a shared bucket (false). Set explicitly only to override that — e.g. a per-node
+                      S3 bucket, or a "file" backend on a ReadWriteMany volume shared by all pods.
+                    type: boolean
                   replicationFactor:
                     description: |-
                       ReplicationFactor is the number of replicas per write (RF). Should be <= Replicas. The

config/samples/db_v1alpha1_oteldbcluster.yaml

@@ -28,6 +28,10 @@ spec:
     replicationFactor: 3
     shardsPerTenant: 4   # spread each tenant's series across 4 placement units
     peerPort: 7946
+    # privateBackend gates part replication (cluster/partsync). Unset it derives from
+    # storage.backend: true for file (per-pod PVC), false for the shared s3 bucket. Override only
+    # for an unusual topology, e.g. a file backend on a ReadWriteMany volume shared by all pods.
+    # privateBackend: true
 
   # All signals served from the clustered embedded engine.
   signals:

internal/controller/builders_test.go

@@ -78,6 +78,10 @@ func TestRenderConfigFileBackendDefaults(t *testing.T) {
 	if !ok || len(etcd) != 1 || etcd[0] != "http://etcd:2379" {
 		t.Errorf("etcd = %v", cluster["etcd"])
 	}
+	// The file backend is one PVC per pod, so partsync must be on or flushed parts never replicate.
+	if cluster["private_backend"] != true {
+		t.Errorf("private_backend = %v, want true for the file backend", cluster["private_backend"])
+	}
 	// Per-pod identity must NOT be baked into the shared config.
 	for _, k := range []string{"id", "addr", "zone"} {
 		if _, present := cluster[k]; present {
@@ -130,6 +134,49 @@ func TestRenderConfigS3(t *testing.T) {
 	}
 }
 
+func TestRenderConfigPrivateBackend(t *testing.T) {
+	for _, tt := range []struct {
+		name     string
+		backend  dbv1alpha1.StorageBackend
+		override *bool
+		want     bool
+	}{
+		{"file backend is per-pod PVC", dbv1alpha1.StorageBackendFile, nil, true},
+		{"s3 backend is a shared bucket", dbv1alpha1.StorageBackendS3, nil, false},
+		{"unset backend defaults to file", "", nil, true},
+		{"file backend on a shared volume", dbv1alpha1.StorageBackendFile, ptr.To(false), false},
+		{"s3 backend with per-node buckets", dbv1alpha1.StorageBackendS3, ptr.To(true), true},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			cr := testCluster()
+			cr.Spec.Storage.Backend = tt.backend
+			cr.Spec.Cluster.PrivateBackend = tt.override
+			if tt.backend == dbv1alpha1.StorageBackendS3 {
+				cr.Spec.Storage.S3 = &dbv1alpha1.S3Spec{Bucket: "oteldb"}
+			}
+
+			out, err := renderConfig(cr, cr.Spec.Etcd.Endpoints)
+			require.NoError(t, err)
+			var cfg map[string]any
+			require.NoError(t, yaml.Unmarshal([]byte(out), &cfg))
+
+			cluster := cfg["storage"].(map[string]any)["cluster"].(map[string]any)
+			require.Equal(t, tt.want, cluster["private_backend"])
+		})
+	}
+}
+
+// Regression: storage.cluster is reserved, so extraConfig cannot reach private_backend and the CRD
+// field is the only supported route (oteldb/operator#16).
+func TestRenderConfigPrivateBackendNotReachableViaExtraConfig(t *testing.T) {
+	cr := testCluster()
+	cr.Spec.ExtraConfig = &runtime.RawExtension{
+		Raw: []byte(`{"storage":{"cluster":{"private_backend":true}}}`),
+	}
+	_, err := renderConfig(cr, cr.Spec.Etcd.Endpoints)
+	require.ErrorContains(t, err, "storage.cluster")
+}
+
 func TestRenderConfigS3RequiresBucket(t *testing.T) {
 	cr := testCluster()
 	cr.Spec.Storage.Backend = dbv1alpha1.StorageBackendS3

internal/controller/config.go

@@ -67,6 +67,7 @@ func renderConfig(cr *dbv1alpha1.OtelDBCluster, etcdEndpoints []string) (string,
 	// The cluster block. Only the deployment-wide settings live here; id/addr/zone come from env.
 	cluster := ringConfig(cr, etcdEndpoints)
 	cluster[keyPort] = int(peerPortOf(cr))
+	cluster[keyPrivateBackend] = privateBackendOf(cr)
 	storage[keyCluster] = cluster
 
 	// S3 shared backend (optional). The data dir doubles as the WAL dir so unflushed head data
@@ -163,6 +164,18 @@ func backendOf(cr *dbv1alpha1.OtelDBCluster) dbv1alpha1.StorageBackend {
 	return dbv1alpha1.StorageBackendFile
 }
 
+// privateBackendOf reports whether each node's durable backend is its own, unshared by peers. It
+// gates cluster/partsync, which replicates flushed parts and backfills a node that lost its disk;
+// without it only the in-memory head is replicated, so every flushed part exists in exactly one
+// copy regardless of RF. The file backend is one PVC per pod and so is always private; s3 is a
+// bucket every node shares. spec.cluster.privateBackend overrides the derivation.
+func privateBackendOf(cr *dbv1alpha1.OtelDBCluster) bool {
+	if p := cr.Spec.Cluster.PrivateBackend; p != nil {
+		return *p
+	}
+	return backendOf(cr) == dbv1alpha1.StorageBackendFile
+}
+
 func dirOf(cr *dbv1alpha1.OtelDBCluster) string {
 	if d := cr.Spec.Storage.Dir; d != "" {
 		return d

internal/controller/naming.go

@@ -65,11 +65,12 @@ const (
 	keyLogsBackend     = "logs_backend"
 	keyProfilesBackend = "profiles_backend"
 
-	keyBackend = "backend"
-	keyCluster = "cluster"
-	keyDir     = "dir"
-	keyEtcd    = "etcd"
-	keyPort    = "port"
+	keyBackend        = "backend"
+	keyCluster        = "cluster"
+	keyDir            = "dir"
+	keyEtcd           = "etcd"
+	keyPort           = "port"
+	keyPrivateBackend = "private_backend"
 )
 
 // Query API config block keys, shared by cmd/oteldb and cmd/odbselect.
@@ -85,15 +86,17 @@ const (
 
 // Container ports exposed by every oteldb node. Names must be <= 15 chars (k8s port-name limit).
 const (
-	portOTLPGRPC   = 4317
-	portOTLPHTTP   = 4318
-	portPromRW     = 19291
-	portPromHTTP   = 9090
-	portTempoHTTP  = 3200
-	portLokiHTTP   = 3100
-	portPyroscope  = 4040
-	portHealth     = 13133
-	portSelfMetric = 8090
+	portOTLPGRPC  = 4317
+	portOTLPHTTP  = 4318
+	portPromRW    = 19291
+	portPromHTTP  = 9090
+	portTempoHTTP = 3200
+	portLokiHTTP  = 3100
+	portPyroscope = 4040
+	portHealth    = 13133
+	// portSelfMetric is the OpenTelemetry default Prometheus exporter port. It must not be 8090:
+	// that is oteldb's own admin API bind (admin.bind defaults to :8090).
+	portSelfMetric = 9464
 	// portPeer is the default; the effective value comes from spec.cluster.peerPort.
 	portPeer = 7946
 )