From 6b633d5dff39d128b43183149af012a7b710ca99 Mon Sep 17 00:00:00 2001 From: tdakkota Date: Mon, 17 Aug 2026 23:20:14 +0300 Subject: [PATCH 1/3] fix(controller): move self-metrics port off oteldb's admin API bind portSelfMetric was 8090, which is oteldb's admin API bind (admin.bind defaults to :8090 and the operator never overrides it), so the Service port named metrics routed to the admin API and UI. Fixes #12 --- README.md | 4 +++- internal/controller/naming.go | 20 +++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d24a1f4..cdee7a0 100644 --- a/README.md +++ b/README.md @@ -135,9 +135,11 @@ config file. ## Ports Client APIs (exposed by the client Service): `4317` OTLP gRPC, `4318` OTLP HTTP, `19291` Prometheus -remote-write, `9090` PromQL, `3200` TraceQL (Tempo), `3100` LogQL (Loki), `4040` Pyroscope, `8090` +remote-write, `9090` PromQL, `3200` TraceQL (Tempo), `3100` LogQL (Loki), `4040` Pyroscope, `9464` self-metrics, `13133` health. Peer replication: `7946` (headless Service). +`8090` is deliberately not exposed: it is oteldb's admin API bind, not a self-metrics endpoint. + ## Getting Started Prerequisites: Go 1.26+, a Kubernetes cluster, `kubectl`, and (for building the image) Docker. diff --git a/internal/controller/naming.go b/internal/controller/naming.go index 2005630..28ecf5c 100644 --- a/internal/controller/naming.go +++ b/internal/controller/naming.go @@ -51,15 +51,17 @@ const ( // Container ports exposed by every oteldb node. Names must be <= 15 chars (k8s port-name limit). const ( - portOTLPGRPC = 4317 - portOTLPHTTP = 4318 - portPromRW = 19291 - portPromHTTP = 9090 - portTempoHTTP = 3200 - portLokiHTTP = 3100 - portPyroscope = 4040 - portHealth = 13133 - portSelfMetric = 8090 + portOTLPGRPC = 4317 + portOTLPHTTP = 4318 + portPromRW = 19291 + portPromHTTP = 9090 + portTempoHTTP = 3200 + portLokiHTTP = 3100 + portPyroscope = 4040 + portHealth = 13133 + // portSelfMetric is the OpenTelemetry default Prometheus exporter port. It must not be 8090: + // that is oteldb's own admin API bind (admin.bind defaults to :8090). + portSelfMetric = 9464 // portPeer is the default; the effective value comes from spec.cluster.peerPort. portPeer = 7946 ) From 8295b79e30a5af552c3862a09a3f8bba7b259719 Mon Sep 17 00:00:00 2001 From: tdakkota Date: Tue, 18 Aug 2026 10:50:28 +0300 Subject: [PATCH 2/3] feat(api): render storage.cluster.private_backend Without it cluster/partsync never runs, so only the unflushed head is replicated and every flushed part exists in exactly one copy regardless of spec.cluster.replicationFactor. A node that loses its PVC never recovers: bootstrapShard's syncParts no-ops for a supposedly shared backend, so no engine is ever created. spec.cluster.privateBackend is a *bool: unset derives from spec.storage.backend, since file is one PVC per pod (private) and s3 is a bucket every node shares. storage.cluster stays reserved from extraConfig, so the CRD field is the only route. --- README.md | 1 + api/v1alpha1/oteldbcluster_types.go | 11 +++++ config/samples/db_v1alpha1_oteldbcluster.yaml | 4 ++ internal/controller/builders_test.go | 47 +++++++++++++++++++ internal/controller/config.go | 17 ++++++- internal/controller/naming.go | 9 ++-- 6 files changed, 83 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index cdee7a0..a73ad0d 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ for a fuller example including the S3 backend. | `cluster.replicationFactor` | Replicas per write (RF). | | `cluster.shardsPerTenant` | Per-tenant series sharding across placement units. | | `cluster.peerPort` | Peer replication port (default 7946). | +| `cluster.privateBackend` | Whether each node's durable backend is its own. Gates `cluster/partsync`, which replicates *flushed parts* and backfills a node that lost its disk — without it only the in-memory head is replicated and every part exists in one copy regardless of RF. Unset derives from `storage.backend`: `true` for `file` (per-pod PVC), `false` for `s3` (shared bucket). Needs an oteldb carrying [oteldb/oteldb#1264](https://github.com/oteldb/oteldb/pull/1264); older builds ignore the key silently. | | `cluster.etcdPrefix` | etcd key prefix (storage "root", default `/oteldb`). | | `cluster.staticZone` | Fixed failure-domain label for the cluster's nodes (ring zone-spreading). | | `signals` | Which signals to serve (all default on). Disabling one drops its backend, its API bind and its ports; disabling all is rejected. | diff --git a/api/v1alpha1/oteldbcluster_types.go b/api/v1alpha1/oteldbcluster_types.go index ec72423..7ad73d8 100644 --- a/api/v1alpha1/oteldbcluster_types.go +++ b/api/v1alpha1/oteldbcluster_types.go @@ -261,6 +261,17 @@ type ClusterSpec struct { // +optional ShardsPerTenant *int32 `json:"shardsPerTenant,omitempty"` + // PrivateBackend declares that every node's durable backend is its own, unshared by peers. + // It gates cluster/partsync: only a private backend replicates *flushed parts* between nodes + // and backfills a node that lost its disk. Without it the ring replicates just the in-memory + // head, so each flushed part exists in exactly one copy regardless of ReplicationFactor. + // + // Unset derives from Storage.Backend: "file" is per-node PVC and therefore private (true), + // "s3" is a shared bucket (false). Set explicitly only to override that — e.g. a per-node + // S3 bucket, or a "file" backend on a ReadWriteMany volume shared by all pods. + // +optional + PrivateBackend *bool `json:"privateBackend,omitempty"` + // PeerPort is the port peers use to reach each node's replication server. // +kubebuilder:default=7946 // +optional diff --git a/config/samples/db_v1alpha1_oteldbcluster.yaml b/config/samples/db_v1alpha1_oteldbcluster.yaml index ce4635f..d1c25c2 100644 --- a/config/samples/db_v1alpha1_oteldbcluster.yaml +++ b/config/samples/db_v1alpha1_oteldbcluster.yaml @@ -28,6 +28,10 @@ spec: replicationFactor: 3 shardsPerTenant: 4 # spread each tenant's series across 4 placement units peerPort: 7946 + # privateBackend gates part replication (cluster/partsync). Unset it derives from + # storage.backend: true for file (per-pod PVC), false for the shared s3 bucket. Override only + # for an unusual topology, e.g. a file backend on a ReadWriteMany volume shared by all pods. + # privateBackend: true # All signals served from the clustered embedded engine. signals: diff --git a/internal/controller/builders_test.go b/internal/controller/builders_test.go index 9e4a754..b3440e4 100644 --- a/internal/controller/builders_test.go +++ b/internal/controller/builders_test.go @@ -78,6 +78,10 @@ func TestRenderConfigFileBackendDefaults(t *testing.T) { if !ok || len(etcd) != 1 || etcd[0] != "http://etcd:2379" { t.Errorf("etcd = %v", cluster["etcd"]) } + // The file backend is one PVC per pod, so partsync must be on or flushed parts never replicate. + if cluster["private_backend"] != true { + t.Errorf("private_backend = %v, want true for the file backend", cluster["private_backend"]) + } // Per-pod identity must NOT be baked into the shared config. for _, k := range []string{"id", "addr", "zone"} { if _, present := cluster[k]; present { @@ -130,6 +134,49 @@ func TestRenderConfigS3(t *testing.T) { } } +func TestRenderConfigPrivateBackend(t *testing.T) { + for _, tt := range []struct { + name string + backend dbv1alpha1.StorageBackend + override *bool + want bool + }{ + {"file backend is per-pod PVC", dbv1alpha1.StorageBackendFile, nil, true}, + {"s3 backend is a shared bucket", dbv1alpha1.StorageBackendS3, nil, false}, + {"unset backend defaults to file", "", nil, true}, + {"file backend on a shared volume", dbv1alpha1.StorageBackendFile, ptr.To(false), false}, + {"s3 backend with per-node buckets", dbv1alpha1.StorageBackendS3, ptr.To(true), true}, + } { + t.Run(tt.name, func(t *testing.T) { + cr := testCluster() + cr.Spec.Storage.Backend = tt.backend + cr.Spec.Cluster.PrivateBackend = tt.override + if tt.backend == dbv1alpha1.StorageBackendS3 { + cr.Spec.Storage.S3 = &dbv1alpha1.S3Spec{Bucket: "oteldb"} + } + + out, err := renderConfig(cr, cr.Spec.Etcd.Endpoints) + require.NoError(t, err) + var cfg map[string]any + require.NoError(t, yaml.Unmarshal([]byte(out), &cfg)) + + cluster := cfg["storage"].(map[string]any)["cluster"].(map[string]any) + require.Equal(t, tt.want, cluster["private_backend"]) + }) + } +} + +// Regression: storage.cluster is reserved, so extraConfig cannot reach private_backend and the CRD +// field is the only supported route (oteldb/operator#16). +func TestRenderConfigPrivateBackendNotReachableViaExtraConfig(t *testing.T) { + cr := testCluster() + cr.Spec.ExtraConfig = &runtime.RawExtension{ + Raw: []byte(`{"storage":{"cluster":{"private_backend":true}}}`), + } + _, err := renderConfig(cr, cr.Spec.Etcd.Endpoints) + require.ErrorContains(t, err, "storage.cluster") +} + func TestRenderConfigS3RequiresBucket(t *testing.T) { cr := testCluster() cr.Spec.Storage.Backend = dbv1alpha1.StorageBackendS3 diff --git a/internal/controller/config.go b/internal/controller/config.go index b0114db..cb41776 100644 --- a/internal/controller/config.go +++ b/internal/controller/config.go @@ -66,8 +66,9 @@ func renderConfig(cr *dbv1alpha1.OtelDBCluster, etcdEndpoints []string) (string, // The cluster block. Only the deployment-wide settings live here; id/addr/zone come from env. cluster := map[string]any{ - keyEtcd: etcdEndpoints, - keyPort: int(peerPortOf(cr)), + keyEtcd: etcdEndpoints, + keyPort: int(peerPortOf(cr)), + keyPrivateBackend: privateBackendOf(cr), } if rf := cr.Spec.Cluster.ReplicationFactor; rf != nil { cluster["rf"] = int(*rf) @@ -157,6 +158,18 @@ func backendOf(cr *dbv1alpha1.OtelDBCluster) dbv1alpha1.StorageBackend { return dbv1alpha1.StorageBackendFile } +// privateBackendOf reports whether each node's durable backend is its own, unshared by peers. It +// gates cluster/partsync, which replicates flushed parts and backfills a node that lost its disk; +// without it only the in-memory head is replicated, so every flushed part exists in exactly one +// copy regardless of RF. The file backend is one PVC per pod and so is always private; s3 is a +// bucket every node shares. spec.cluster.privateBackend overrides the derivation. +func privateBackendOf(cr *dbv1alpha1.OtelDBCluster) bool { + if p := cr.Spec.Cluster.PrivateBackend; p != nil { + return *p + } + return backendOf(cr) == dbv1alpha1.StorageBackendFile +} + func dirOf(cr *dbv1alpha1.OtelDBCluster) string { if d := cr.Spec.Storage.Dir; d != "" { return d diff --git a/internal/controller/naming.go b/internal/controller/naming.go index 28ecf5c..c5dde25 100644 --- a/internal/controller/naming.go +++ b/internal/controller/naming.go @@ -43,10 +43,11 @@ const ( keyLogsBackend = "logs_backend" keyProfilesBackend = "profiles_backend" - keyBackend = "backend" - keyDir = "dir" - keyEtcd = "etcd" - keyPort = "port" + keyBackend = "backend" + keyDir = "dir" + keyEtcd = "etcd" + keyPort = "port" + keyPrivateBackend = "private_backend" ) // Container ports exposed by every oteldb node. Names must be <= 15 chars (k8s port-name limit). From b71647ad150bd91e50802efd675ac94ad2c16c2e Mon Sep 17 00:00:00 2001 From: tdakkota Date: Tue, 18 Aug 2026 10:50:34 +0300 Subject: [PATCH 3/3] chore(crd): regenerate manifests for spec.cluster.privateBackend --- api/v1alpha1/zz_generated.deepcopy.go | 5 +++++ config/crd/bases/db.oteldb.io_oteldbclusters.yaml | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a50e399..1593690 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -39,6 +39,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(int32) **out = **in } + if in.PrivateBackend != nil { + in, out := &in.PrivateBackend, &out.PrivateBackend + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSpec. diff --git a/config/crd/bases/db.oteldb.io_oteldbclusters.yaml b/config/crd/bases/db.oteldb.io_oteldbclusters.yaml index f32e5a6..5f46f6a 100644 --- a/config/crd/bases/db.oteldb.io_oteldbclusters.yaml +++ b/config/crd/bases/db.oteldb.io_oteldbclusters.yaml @@ -987,6 +987,17 @@ spec: replication server. format: int32 type: integer + privateBackend: + description: |- + PrivateBackend declares that every node's durable backend is its own, unshared by peers. + It gates cluster/partsync: only a private backend replicates *flushed parts* between nodes + and backfills a node that lost its disk. Without it the ring replicates just the in-memory + head, so each flushed part exists in exactly one copy regardless of ReplicationFactor. + + Unset derives from Storage.Backend: "file" is per-node PVC and therefore private (true), + "s3" is a shared bucket (false). Set explicitly only to override that — e.g. a per-node + S3 bucket, or a "file" backend on a ReadWriteMany volume shared by all pods. + type: boolean replicationFactor: description: |- ReplicationFactor is the number of replicas per write (RF). Should be <= Replicas. The