diff --git a/README.md b/README.md index e1ba01d..1a7898a 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,7 @@ for a fuller example including the S3 backend. | `cluster.replicationFactor` | Replicas per write (RF). | | `cluster.shardsPerTenant` | Per-tenant series sharding across placement units. | | `cluster.peerPort` | Peer replication port (default 7946). | +| `cluster.privateBackend` | Whether each node's durable backend is its own. Gates `cluster/partsync`, which replicates *flushed parts* and backfills a node that lost its disk — without it only the in-memory head is replicated and every part exists in one copy regardless of RF. Unset derives from `storage.backend`: `true` for `file` (per-pod PVC), `false` for `s3` (shared bucket). Needs an oteldb carrying [oteldb/oteldb#1264](https://github.com/oteldb/oteldb/pull/1264); older builds ignore the key silently. | | `cluster.etcdPrefix` | etcd key prefix (storage "root", default `/oteldb`). | | `cluster.staticZone` | Fixed failure-domain label for the cluster's nodes (ring zone-spreading). | | `signals` | Which signals to serve (all default on). Disabling one drops its backend, its API bind and its ports; disabling all is rejected. | @@ -194,19 +195,21 @@ config file. ## Ports Client APIs (exposed by the client Service): `4317` OTLP gRPC, `4318` OTLP HTTP, `19291` Prometheus -remote-write, `9090` PromQL, `3200` TraceQL (Tempo), `3100` LogQL (Loki), `4040` Pyroscope, `8090` +remote-write, `9090` PromQL, `3200` TraceQL (Tempo), `3100` LogQL (Loki), `4040` Pyroscope, `9464` self-metrics, `13133` health. Peer replication: `7946` (headless Service). The ingest Service (`-ingest`) exposes `4317` OTLP gRPC, `4318` OTLP HTTP, `19291` Prometheus -remote-write and `8090` self-metrics. `odbingest` serves OTLP/HTTP, remote write and its health +remote-write and `9464` self-metrics. `odbingest` serves OTLP/HTTP, remote write and its health endpoints on **one** listener (`19291`), so the Service publishes `4318` — the port stock OTLP/HTTP exporters target — and remaps it onto that listener. The query Service (`-query`) exposes `9090` PromQL, `3200` TraceQL, `3100` LogQL, `4040` -Pyroscope and `8090` self-metrics, dropping any whose signal is disabled. `odbselect` serves each on +Pyroscope and `9464` self-metrics, dropping any whose signal is disabled. `odbselect` serves each on its own listener, so these are published one-to-one; its health listener (`13133`) is probed, not published. +`8090` is deliberately not exposed: it is oteldb's admin API bind, not a self-metrics endpoint. + ## Getting Started Prerequisites: Go 1.26+, a Kubernetes cluster, `kubectl`, and (for building the image) Docker. diff --git a/api/v1alpha1/oteldbcluster_types.go b/api/v1alpha1/oteldbcluster_types.go index 474b7b4..f58a50d 100644 --- a/api/v1alpha1/oteldbcluster_types.go +++ b/api/v1alpha1/oteldbcluster_types.go @@ -456,6 +456,17 @@ type ClusterSpec struct { // +optional ShardsPerTenant *int32 `json:"shardsPerTenant,omitempty"` + // PrivateBackend declares that every node's durable backend is its own, unshared by peers. + // It gates cluster/partsync: only a private backend replicates *flushed parts* between nodes + // and backfills a node that lost its disk. Without it the ring replicates just the in-memory + // head, so each flushed part exists in exactly one copy regardless of ReplicationFactor. + // + // Unset derives from Storage.Backend: "file" is per-node PVC and therefore private (true), + // "s3" is a shared bucket (false). Set explicitly only to override that — e.g. a per-node + // S3 bucket, or a "file" backend on a ReadWriteMany volume shared by all pods. + // +optional + PrivateBackend *bool `json:"privateBackend,omitempty"` + // PeerPort is the port peers use to reach each node's replication server. // +kubebuilder:default=7946 // +optional diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ec26f00..ff08843 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -39,6 +39,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(int32) **out = **in } + if in.PrivateBackend != nil { + in, out := &in.PrivateBackend, &out.PrivateBackend + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSpec. diff --git a/config/crd/bases/db.oteldb.io_oteldbclusters.yaml b/config/crd/bases/db.oteldb.io_oteldbclusters.yaml index 19688c5..62685aa 100644 --- a/config/crd/bases/db.oteldb.io_oteldbclusters.yaml +++ b/config/crd/bases/db.oteldb.io_oteldbclusters.yaml @@ -987,6 +987,17 @@ spec: replication server. format: int32 type: integer + privateBackend: + description: |- + PrivateBackend declares that every node's durable backend is its own, unshared by peers. + It gates cluster/partsync: only a private backend replicates *flushed parts* between nodes + and backfills a node that lost its disk. Without it the ring replicates just the in-memory + head, so each flushed part exists in exactly one copy regardless of ReplicationFactor. + + Unset derives from Storage.Backend: "file" is per-node PVC and therefore private (true), + "s3" is a shared bucket (false). Set explicitly only to override that — e.g. a per-node + S3 bucket, or a "file" backend on a ReadWriteMany volume shared by all pods. + type: boolean replicationFactor: description: |- ReplicationFactor is the number of replicas per write (RF). Should be <= Replicas. The diff --git a/config/samples/db_v1alpha1_oteldbcluster.yaml b/config/samples/db_v1alpha1_oteldbcluster.yaml index 0907a18..c9814a3 100644 --- a/config/samples/db_v1alpha1_oteldbcluster.yaml +++ b/config/samples/db_v1alpha1_oteldbcluster.yaml @@ -28,6 +28,10 @@ spec: replicationFactor: 3 shardsPerTenant: 4 # spread each tenant's series across 4 placement units peerPort: 7946 + # privateBackend gates part replication (cluster/partsync). Unset it derives from + # storage.backend: true for file (per-pod PVC), false for the shared s3 bucket. Override only + # for an unusual topology, e.g. a file backend on a ReadWriteMany volume shared by all pods. + # privateBackend: true # All signals served from the clustered embedded engine. signals: diff --git a/internal/controller/builders_test.go b/internal/controller/builders_test.go index 9e4a754..b3440e4 100644 --- a/internal/controller/builders_test.go +++ b/internal/controller/builders_test.go @@ -78,6 +78,10 @@ func TestRenderConfigFileBackendDefaults(t *testing.T) { if !ok || len(etcd) != 1 || etcd[0] != "http://etcd:2379" { t.Errorf("etcd = %v", cluster["etcd"]) } + // The file backend is one PVC per pod, so partsync must be on or flushed parts never replicate. + if cluster["private_backend"] != true { + t.Errorf("private_backend = %v, want true for the file backend", cluster["private_backend"]) + } // Per-pod identity must NOT be baked into the shared config. for _, k := range []string{"id", "addr", "zone"} { if _, present := cluster[k]; present { @@ -130,6 +134,49 @@ func TestRenderConfigS3(t *testing.T) { } } +func TestRenderConfigPrivateBackend(t *testing.T) { + for _, tt := range []struct { + name string + backend dbv1alpha1.StorageBackend + override *bool + want bool + }{ + {"file backend is per-pod PVC", dbv1alpha1.StorageBackendFile, nil, true}, + {"s3 backend is a shared bucket", dbv1alpha1.StorageBackendS3, nil, false}, + {"unset backend defaults to file", "", nil, true}, + {"file backend on a shared volume", dbv1alpha1.StorageBackendFile, ptr.To(false), false}, + {"s3 backend with per-node buckets", dbv1alpha1.StorageBackendS3, ptr.To(true), true}, + } { + t.Run(tt.name, func(t *testing.T) { + cr := testCluster() + cr.Spec.Storage.Backend = tt.backend + cr.Spec.Cluster.PrivateBackend = tt.override + if tt.backend == dbv1alpha1.StorageBackendS3 { + cr.Spec.Storage.S3 = &dbv1alpha1.S3Spec{Bucket: "oteldb"} + } + + out, err := renderConfig(cr, cr.Spec.Etcd.Endpoints) + require.NoError(t, err) + var cfg map[string]any + require.NoError(t, yaml.Unmarshal([]byte(out), &cfg)) + + cluster := cfg["storage"].(map[string]any)["cluster"].(map[string]any) + require.Equal(t, tt.want, cluster["private_backend"]) + }) + } +} + +// Regression: storage.cluster is reserved, so extraConfig cannot reach private_backend and the CRD +// field is the only supported route (oteldb/operator#16). +func TestRenderConfigPrivateBackendNotReachableViaExtraConfig(t *testing.T) { + cr := testCluster() + cr.Spec.ExtraConfig = &runtime.RawExtension{ + Raw: []byte(`{"storage":{"cluster":{"private_backend":true}}}`), + } + _, err := renderConfig(cr, cr.Spec.Etcd.Endpoints) + require.ErrorContains(t, err, "storage.cluster") +} + func TestRenderConfigS3RequiresBucket(t *testing.T) { cr := testCluster() cr.Spec.Storage.Backend = dbv1alpha1.StorageBackendS3 diff --git a/internal/controller/config.go b/internal/controller/config.go index 047d7a8..55322e8 100644 --- a/internal/controller/config.go +++ b/internal/controller/config.go @@ -67,6 +67,7 @@ func renderConfig(cr *dbv1alpha1.OtelDBCluster, etcdEndpoints []string) (string, // The cluster block. Only the deployment-wide settings live here; id/addr/zone come from env. cluster := ringConfig(cr, etcdEndpoints) cluster[keyPort] = int(peerPortOf(cr)) + cluster[keyPrivateBackend] = privateBackendOf(cr) storage[keyCluster] = cluster // S3 shared backend (optional). The data dir doubles as the WAL dir so unflushed head data @@ -163,6 +164,18 @@ func backendOf(cr *dbv1alpha1.OtelDBCluster) dbv1alpha1.StorageBackend { return dbv1alpha1.StorageBackendFile } +// privateBackendOf reports whether each node's durable backend is its own, unshared by peers. It +// gates cluster/partsync, which replicates flushed parts and backfills a node that lost its disk; +// without it only the in-memory head is replicated, so every flushed part exists in exactly one +// copy regardless of RF. The file backend is one PVC per pod and so is always private; s3 is a +// bucket every node shares. spec.cluster.privateBackend overrides the derivation. +func privateBackendOf(cr *dbv1alpha1.OtelDBCluster) bool { + if p := cr.Spec.Cluster.PrivateBackend; p != nil { + return *p + } + return backendOf(cr) == dbv1alpha1.StorageBackendFile +} + func dirOf(cr *dbv1alpha1.OtelDBCluster) string { if d := cr.Spec.Storage.Dir; d != "" { return d diff --git a/internal/controller/naming.go b/internal/controller/naming.go index bf48604..1824006 100644 --- a/internal/controller/naming.go +++ b/internal/controller/naming.go @@ -65,11 +65,12 @@ const ( keyLogsBackend = "logs_backend" keyProfilesBackend = "profiles_backend" - keyBackend = "backend" - keyCluster = "cluster" - keyDir = "dir" - keyEtcd = "etcd" - keyPort = "port" + keyBackend = "backend" + keyCluster = "cluster" + keyDir = "dir" + keyEtcd = "etcd" + keyPort = "port" + keyPrivateBackend = "private_backend" ) // Query API config block keys, shared by cmd/oteldb and cmd/odbselect. @@ -85,15 +86,17 @@ const ( // Container ports exposed by every oteldb node. Names must be <= 15 chars (k8s port-name limit). const ( - portOTLPGRPC = 4317 - portOTLPHTTP = 4318 - portPromRW = 19291 - portPromHTTP = 9090 - portTempoHTTP = 3200 - portLokiHTTP = 3100 - portPyroscope = 4040 - portHealth = 13133 - portSelfMetric = 8090 + portOTLPGRPC = 4317 + portOTLPHTTP = 4318 + portPromRW = 19291 + portPromHTTP = 9090 + portTempoHTTP = 3200 + portLokiHTTP = 3100 + portPyroscope = 4040 + portHealth = 13133 + // portSelfMetric is the OpenTelemetry default Prometheus exporter port. It must not be 8090: + // that is oteldb's own admin API bind (admin.bind defaults to :8090). + portSelfMetric = 9464 // portPeer is the default; the effective value comes from spec.cluster.peerPort. portPeer = 7946 )