diff --git a/README.md b/README.md index d24a1f4..fb90b58 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,30 @@ The operator models this as **one StatefulSet** of oteldb pods (stable identity FQDN → peer address, per-pod PVC), a **headless Service** for peer DNS, and a **client Service** for the query/ingest APIs. +### Optional role node groups + +Symmetric nodes couple ingest capacity to storage capacity: the only way to absorb a write spike is +to add a node, and that node immediately takes ownership of a share of the shards. `spec.ingest` +breaks the coupling by deploying a pool of stateless **`odbingest`** write nodes as a separate +Deployment. It is opt-in — omit the stanza and the cluster stays symmetric, exactly as before. + +Unlike VictoriaMetrics' `vminsert`, an `odbingest` pod is **not** handed a list of storage nodes. It +watches etcd for membership and computes shard owners locally from the rendezvous-hash ring, so +`spec.replicas` and `spec.ingest.replicas` are genuinely independent: scaling storage triggers no +config re-render and no restart of the ingest pool. + +What the pool *does* need is the ring's shape — `replicationFactor`, `shardsPerTenant`, `etcdPrefix` +— and a mismatch there **fails silently**: the pool resolves a different owner set than the nodes +do, and writes land where no read will look for them. The operator renders both roles from the same +`spec.cluster`, so the mismatch cannot be expressed. + +> **`odbingest` is not in the released oteldb image yet.** It is absent from oteldb's goreleaser +> builds and `release.Dockerfile`, so no published `ghcr.io/oteldb/oteldb` tag contains the binary. +> Set `spec.ingest.image` to a build that ships it until that is fixed upstream. + +A stateless `odbselect` query pool (`spec.query`) is the matching read half; it is not implemented +yet, pending [oteldb/oteldb#1266](https://github.com/oteldb/oteldb/pull/1266). + ### etcd is bring-your-own The operator **does not manage etcd**. etcd's real operational needs — backups, disaster recovery, @@ -60,7 +84,8 @@ for a fuller example including the S3 backend. | Field | Purpose | |---|---| -| `replicas` | Number of oteldb nodes (StatefulSet size). Use `>= cluster.replicationFactor`. | +| `replicas` | Number of oteldb storage nodes (StatefulSet size). Use `>= cluster.replicationFactor`. | +| `ingest` | Optional stateless `odbingest` write pool (Deployment). Absent ⇒ symmetric nodes handle ingest. Takes `replicas` (default 2), `image`, `service`, `extraConfig` and the standard scheduling/security knobs. | | `image` / `imagePullPolicy` / `imagePullSecrets` | oteldb container image (default `ghcr.io/oteldb/oteldb:v0.46.0`). | | `etcd.endpoints` | **Required.** External etcd endpoint list. | | `storage.backend` | `file` (default, per-node PVC) or `s3` (shared object store). | @@ -116,11 +141,16 @@ spec field to use instead. `storage.policy` is now modelled in full, so the whole block is reserved. +`spec.ingest.extraConfig` is merged over the generated `odbingest.yml` under the same rules, with +the whole `cluster` block reserved for `spec.cluster` and `spec.etcd.endpoints`. + ### Status `status` reports `phase` (`Pending`/`Progressing`/`Ready`/`Degraded`), `replicas`, -`readyReplicas`, the resolved `etcdEndpoints`, and standard `Available`/`Progressing`/`Degraded` -conditions. `kubectl get oteldbcluster` prints replicas, ready count, and phase. +`readyReplicas`, `ingestReplicas`, `ingestReadyReplicas`, the resolved `etcdEndpoints`, and standard +`Available`/`Progressing`/`Degraded` conditions. `kubectl get oteldbcluster` prints replicas, ready +count, and phase. The phase tracks the storage nodes: they hold the data, and an ingest pool still +rolling out does not make the cluster unavailable. ## How per-pod identity works @@ -138,6 +168,11 @@ Client APIs (exposed by the client Service): `4317` OTLP gRPC, `4318` OTLP HTTP, remote-write, `9090` PromQL, `3200` TraceQL (Tempo), `3100` LogQL (Loki), `4040` Pyroscope, `8090` self-metrics, `13133` health. Peer replication: `7946` (headless Service). +The ingest Service (`-ingest`) exposes `4317` OTLP gRPC, `4318` OTLP HTTP, `19291` Prometheus +remote-write and `8090` self-metrics. `odbingest` serves OTLP/HTTP, remote write and its health +endpoints on **one** listener (`19291`), so the Service publishes `4318` — the port stock OTLP/HTTP +exporters target — and remaps it onto that listener. + ## Getting Started Prerequisites: Go 1.26+, a Kubernetes cluster, `kubectl`, and (for building the image) Docker. diff --git a/api/v1alpha1/oteldbcluster_types.go b/api/v1alpha1/oteldbcluster_types.go index ec72423..b9dbafa 100644 --- a/api/v1alpha1/oteldbcluster_types.go +++ b/api/v1alpha1/oteldbcluster_types.go @@ -30,11 +30,25 @@ type OtelDBClusterSpec struct { // Replicas is the number of oteldb storage nodes in the cluster. Each node is a symmetric // StatefulSet pod that ingests, queries, stores, and replicates. For real redundancy this // should be at least the replication factor (see Cluster.ReplicationFactor). + // + // This is the storage dimension only. Ingest capacity scales separately via spec.ingest, and + // scaling either has no effect on the other: membership lives in etcd and the ring is computed + // from it, so an ingest pod needs no knowledge of how many storage nodes there are. // +kubebuilder:default=3 // +kubebuilder:validation:Minimum=1 // +optional Replicas *int32 `json:"replicas,omitempty"` + // Ingest optionally deploys a pool of stateless odbingest write nodes, so ingest capacity + // scales independently of storage. Absent (the default) means no ingest pool: the storage + // nodes are symmetric and accept writes themselves. + // + // The pool holds no data and is not a ring member. It follows etcd membership read-only and + // routes each shard's write to that shard's primary, so scaling spec.replicas needs no change + // to it at all. + // +optional + Ingest *IngestSpec `json:"ingest,omitempty"` + // Image is the oteldb container image. Defaults to the operator's pinned image when empty. // +optional Image string `json:"image,omitempty"` @@ -157,6 +171,86 @@ type EtcdSpec struct { Endpoints []string `json:"endpoints"` } +// IngestSpec configures the stateless odbingest write pool: a Deployment of nodes that accept +// OTLP and Prometheus remote write and route each shard to its ring primary. +// +// The pool's ring parameters (etcd endpoints, replication factor, shards per tenant, key prefix) +// are rendered from spec.etcd and spec.cluster, the same source the storage nodes render from. A +// mismatch there does not fail — it resolves a different owner set than the nodes do, and writes +// land where no read will look for them — so it is deliberately not configurable per pool. +// +// The odbingest binary is not yet shipped in the released oteldb image (it is absent from oteldb's +// goreleaser builds and release Dockerfile). Until it is, set Image to a build that contains it. +type IngestSpec struct { + // Replicas is the number of odbingest pods. They hold no data, so this is purely a throughput + // and availability knob and can be changed freely. + // +kubebuilder:default=2 + // +kubebuilder:validation:Minimum=0 + // +optional + Replicas *int32 `json:"replicas,omitempty"` + + // Image is the container image running odbingest. Defaults to the cluster image + // (spec.image, else the operator's pinned image). + // +optional + Image string `json:"image,omitempty"` + + // Resources are the compute resources for each odbingest container. + // +optional + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // Service configures the client-facing Service that exposes the ingest endpoints. + // +optional + Service ServiceSpec `json:"service,omitempty"` + + // PodAnnotations are added to every odbingest pod. + // +optional + PodAnnotations map[string]string `json:"podAnnotations,omitempty"` + + // PodLabels are added to every odbingest pod. + // +optional + PodLabels map[string]string `json:"podLabels,omitempty"` + + // ServiceAccountName is the ServiceAccount for the odbingest pods. Empty falls back to + // spec.serviceAccountName. + // +optional + ServiceAccountName string `json:"serviceAccountName,omitempty"` + + // NodeSelector constrains odbingest pods to nodes with matching labels. + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // Affinity for odbingest pods. When empty, the operator applies a soft anti-affinity that + // spreads the pool across nodes. + // +optional + Affinity *corev1.Affinity `json:"affinity,omitempty"` + + // Tolerations for odbingest pods. + // +optional + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + + // TopologySpreadConstraints for odbingest pods. + // +optional + TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` + + // PodSecurityContext for odbingest pods. + // +optional + PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` + + // SecurityContext for the odbingest container. + // +optional + SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` + + // ExtraConfig is arbitrary additional odbingest config deeply merged over the generated + // config, as a top-level YAML/JSON object. Use it to set fields the CRD does not model, such as + // prometheus_remote_write.time_threshold or the OTLP body-size limits. + // + // The cluster block is reserved and rejected instead of merged: it must stay in step with the + // storage nodes. Configure it through spec.etcd and spec.cluster. + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + ExtraConfig *runtime.RawExtension `json:"extraConfig,omitempty"` +} + // StorageBackend selects the embedded storage engine's durable backend. // +kubebuilder:validation:Enum=file;s3 type StorageBackend string @@ -519,6 +613,15 @@ type OtelDBClusterStatus struct { // +optional ReadyReplicas int32 `json:"readyReplicas,omitempty"` + // IngestReplicas is the desired number of odbingest nodes. Zero when no ingest pool is + // configured. + // +optional + IngestReplicas int32 `json:"ingestReplicas,omitempty"` + + // IngestReadyReplicas is the number of ready odbingest nodes. + // +optional + IngestReadyReplicas int32 `json:"ingestReadyReplicas,omitempty"` + // EtcdEndpoints is the resolved etcd endpoint list the cluster is using. // +optional EtcdEndpoints []string `json:"etcdEndpoints,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a50e399..1e164d4 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -128,6 +128,83 @@ func (in *EtcdSpec) DeepCopy() *EtcdSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IngestSpec) DeepCopyInto(out *IngestSpec) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + in.Resources.DeepCopyInto(&out.Resources) + in.Service.DeepCopyInto(&out.Service) + if in.PodAnnotations != nil { + in, out := &in.PodAnnotations, &out.PodAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.PodLabels != nil { + in, out := &in.PodLabels, &out.PodLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(v1.Affinity) + (*in).DeepCopyInto(*out) + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.TopologySpreadConstraints != nil { + in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints + *out = make([]v1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.PodSecurityContext != nil { + in, out := &in.PodSecurityContext, &out.PodSecurityContext + *out = new(v1.PodSecurityContext) + (*in).DeepCopyInto(*out) + } + if in.SecurityContext != nil { + in, out := &in.SecurityContext, &out.SecurityContext + *out = new(v1.SecurityContext) + (*in).DeepCopyInto(*out) + } + if in.ExtraConfig != nil { + in, out := &in.ExtraConfig, &out.ExtraConfig + *out = new(runtime.RawExtension) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngestSpec. +func (in *IngestSpec) DeepCopy() *IngestSpec { + if in == nil { + return nil + } + out := new(IngestSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LimitsSpec) DeepCopyInto(out *LimitsSpec) { *out = *in @@ -235,6 +312,11 @@ func (in *OtelDBClusterSpec) DeepCopyInto(out *OtelDBClusterSpec) { *out = new(int32) **out = **in } + if in.Ingest != nil { + in, out := &in.Ingest, &out.Ingest + *out = new(IngestSpec) + (*in).DeepCopyInto(*out) + } if in.ImagePullSecrets != nil { in, out := &in.ImagePullSecrets, &out.ImagePullSecrets *out = make([]v1.LocalObjectReference, len(*in)) diff --git a/config/crd/bases/db.oteldb.io_oteldbclusters.yaml b/config/crd/bases/db.oteldb.io_oteldbclusters.yaml index f32e5a6..6580a70 100644 --- a/config/crd/bases/db.oteldb.io_oteldbclusters.yaml +++ b/config/crd/bases/db.oteldb.io_oteldbclusters.yaml @@ -1114,6 +1114,1697 @@ spec: type: object x-kubernetes-map-type: atomic type: array + ingest: + description: |- + Ingest optionally deploys a pool of stateless odbingest write nodes, so ingest capacity + scales independently of storage. Absent (the default) means no ingest pool: the storage + nodes are symmetric and accept writes themselves. + + The pool holds no data and is not a ring member. It follows etcd membership read-only and + routes each shard's write to that shard's primary, so scaling spec.replicas needs no change + to it at all. + properties: + affinity: + description: |- + Affinity for odbingest pods. When empty, the operator applies a soft anti-affinity that + spreads the pool across nodes. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for + the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with + the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + extraConfig: + description: |- + ExtraConfig is arbitrary additional odbingest config deeply merged over the generated + config, as a top-level YAML/JSON object. Use it to set fields the CRD does not model, such as + prometheus_remote_write.time_threshold or the OTLP body-size limits. + + The cluster block is reserved and rejected instead of merged: it must stay in step with the + storage nodes. Configure it through spec.etcd and spec.cluster. + type: object + x-kubernetes-preserve-unknown-fields: true + image: + description: |- + Image is the container image running odbingest. Defaults to the cluster image + (spec.image, else the operator's pinned image). + type: string + nodeSelector: + additionalProperties: + type: string + description: NodeSelector constrains odbingest pods to nodes with + matching labels. + type: object + podAnnotations: + additionalProperties: + type: string + description: PodAnnotations are added to every odbingest pod. + type: object + podLabels: + additionalProperties: + type: string + description: PodLabels are added to every odbingest pod. + type: object + podSecurityContext: + description: PodSecurityContext for odbingest pods. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + replicas: + default: 2 + description: |- + Replicas is the number of odbingest pods. They hold no data, so this is purely a throughput + and availability knob and can be changed freely. + format: int32 + minimum: 0 + type: integer + resources: + description: Resources are the compute resources for each odbingest + container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + securityContext: + description: SecurityContext for the odbingest container. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + service: + description: Service configures the client-facing Service that + exposes the ingest endpoints. + properties: + annotations: + additionalProperties: + type: string + description: Annotations added to the client Service. + type: object + type: + default: ClusterIP + description: Type of the client Service. + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + type: object + serviceAccountName: + description: |- + ServiceAccountName is the ServiceAccount for the odbingest pods. Empty falls back to + spec.serviceAccountName. + type: string + tolerations: + description: Tolerations for odbingest pods. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + description: TopologySpreadConstraints for odbingest pods. + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object logLevel: description: LogLevel sets OTEL_LOG_LEVEL for the oteldb process (e.g. DEBUG, INFO, WARN, ERROR). @@ -1538,6 +3229,10 @@ spec: Replicas is the number of oteldb storage nodes in the cluster. Each node is a symmetric StatefulSet pod that ingests, queries, stores, and replicates. For real redundancy this should be at least the replication factor (see Cluster.ReplicationFactor). + + This is the storage dimension only. Ingest capacity scales separately via spec.ingest, and + scaling either has no effect on the other: membership lives in etcd and the ring is computed + from it, so an ingest pod needs no knowledge of how many storage nodes there are. format: int32 minimum: 1 type: integer @@ -2214,6 +3909,17 @@ spec: items: type: string type: array + ingestReadyReplicas: + description: IngestReadyReplicas is the number of ready odbingest + nodes. + format: int32 + type: integer + ingestReplicas: + description: |- + IngestReplicas is the desired number of odbingest nodes. Zero when no ingest pool is + configured. + format: int32 + type: integer observedGeneration: description: ObservedGeneration is the most recent generation observed by the controller. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index a27dbac..104bbb3 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -27,6 +27,7 @@ rules: - apiGroups: - apps resources: + - deployments - statefulsets verbs: - create diff --git a/config/samples/db_v1alpha1_oteldbcluster.yaml b/config/samples/db_v1alpha1_oteldbcluster.yaml index ce4635f..904e8ad 100644 --- a/config/samples/db_v1alpha1_oteldbcluster.yaml +++ b/config/samples/db_v1alpha1_oteldbcluster.yaml @@ -110,3 +110,37 @@ spec: # keys default to AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY cluster: replicationFactor: 2 +--- +# Example splitting ingest off the storage nodes: three storage nodes plus a stateless odbingest +# pool that scales on its own. The pool holds no data and is not a ring member — it watches etcd +# for membership and routes each shard to its primary, so scaling `replicas` needs no change here. +# +# NOTE: odbingest is not shipped in the released oteldb image yet, hence the explicit image below. +apiVersion: db.oteldb.io/v1alpha1 +kind: OtelDBCluster +metadata: + name: oteldbcluster-ingest-sample +spec: + replicas: 3 + etcd: + endpoints: ["http://etcd:2379"] + cluster: + replicationFactor: 3 + shardsPerTenant: 4 + + ingest: + replicas: 6 + image: "ghcr.io/oteldb/oteldb:with-odbingest" + resources: + requests: + cpu: "1" + memory: 512Mi + limits: + memory: 1Gi + # Fields the CRD does not model. The whole `cluster` block is reserved: it is rendered from + # spec.cluster and spec.etcd so it cannot drift from the storage nodes. + extraConfig: + prometheus_remote_write: + time_threshold: 1h + service: + type: ClusterIP diff --git a/internal/controller/config.go b/internal/controller/config.go index b0114db..047d7a8 100644 --- a/internal/controller/config.go +++ b/internal/controller/config.go @@ -65,20 +65,9 @@ func renderConfig(cr *dbv1alpha1.OtelDBCluster, etcdEndpoints []string) (string, } // The cluster block. Only the deployment-wide settings live here; id/addr/zone come from env. - cluster := map[string]any{ - keyEtcd: etcdEndpoints, - keyPort: int(peerPortOf(cr)), - } - if rf := cr.Spec.Cluster.ReplicationFactor; rf != nil { - cluster["rf"] = int(*rf) - } - if s := cr.Spec.Cluster.ShardsPerTenant; s != nil { - cluster["shards_per_tenant"] = int(*s) - } - if p := cr.Spec.Cluster.EtcdPrefix; p != "" { - cluster["root"] = p - } - storage["cluster"] = cluster + cluster := ringConfig(cr, etcdEndpoints) + cluster[keyPort] = int(peerPortOf(cr)) + storage[keyCluster] = cluster // S3 shared backend (optional). The data dir doubles as the WAL dir so unflushed head data // survives restarts. Credentials are injected via env (AWS default chain), never the config. @@ -150,6 +139,23 @@ func renderConfig(cr *dbv1alpha1.OtelDBCluster, etcdEndpoints []string) (string, return string(out), nil } +// ringConfig renders the ring parameters every role must agree on. A role that resolves a different +// owner set than the storage nodes do does not fail — it reads and writes where the others do not +// look — so all of them render from this one function. +func ringConfig(cr *dbv1alpha1.OtelDBCluster, etcdEndpoints []string) map[string]any { + ring := map[string]any{keyEtcd: etcdEndpoints} + if rf := cr.Spec.Cluster.ReplicationFactor; rf != nil { + ring["rf"] = int(*rf) + } + if s := cr.Spec.Cluster.ShardsPerTenant; s != nil { + ring["shards_per_tenant"] = int(*s) + } + if p := cr.Spec.Cluster.EtcdPrefix; p != "" { + ring["root"] = p + } + return ring +} + func backendOf(cr *dbv1alpha1.OtelDBCluster) dbv1alpha1.StorageBackend { if b := cr.Spec.Storage.Backend; b != "" { return b diff --git a/internal/controller/extraconfig.go b/internal/controller/extraconfig.go index b94f3f8..86d3537 100644 --- a/internal/controller/extraconfig.go +++ b/internal/controller/extraconfig.go @@ -67,10 +67,15 @@ func invalidSpec(format string, args ...any) error { return validationError{err: fmt.Errorf(format, args...)} } -// validateExtraConfig rejects extraConfig that targets operator-owned paths. +// validateExtraConfig rejects spec.extraConfig that targets operator-owned paths. func validateExtraConfig(extra map[string]any) error { + return validateExtraConfigPaths("spec.extraConfig", extra, reservedConfigPaths) +} + +// validateExtraConfigPaths rejects an extraConfig block that targets any of reserved. +func validateExtraConfigPaths(field string, extra map[string]any, reserved map[string]string) error { var found []string - collectReservedPaths(extra, "", &found) + collectReservedPaths(extra, "", reserved, &found) if len(found) == 0 { return nil } @@ -78,24 +83,24 @@ func validateExtraConfig(extra map[string]any) error { msgs := make([]string, 0, len(found)) for _, p := range found { - msgs = append(msgs, fmt.Sprintf("%s (%s)", p, reservedConfigPaths[p])) + msgs = append(msgs, fmt.Sprintf("%s (%s)", p, reserved[p])) } - return invalidSpec("spec.extraConfig sets reserved config %s: %s", - plural(len(found), "path", "paths"), strings.Join(msgs, ", ")) + return invalidSpec("%s sets reserved config %s: %s", + field, plural(len(found), "path", "paths"), strings.Join(msgs, ", ")) } -func collectReservedPaths(m map[string]any, prefix string, found *[]string) { +func collectReservedPaths(m map[string]any, prefix string, reserved map[string]string, found *[]string) { for k, v := range m { path := k if prefix != "" { path = prefix + "." + k } - if _, reserved := reservedConfigPaths[path]; reserved { + if _, ok := reserved[path]; ok { *found = append(*found, path) continue // Everything below a reserved path is reserved as well. } if sub, ok := v.(map[string]any); ok { - collectReservedPaths(sub, path, found) + collectReservedPaths(sub, path, reserved, found) } } } diff --git a/internal/controller/ingest.go b/internal/controller/ingest.go new file mode 100644 index 0000000..8d6628b --- /dev/null +++ b/internal/controller/ingest.go @@ -0,0 +1,257 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "encoding/json" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/yaml" + + dbv1alpha1 "github.com/oteldb/operator/api/v1alpha1" +) + +// ingestReservedConfigPaths are the odbingest config paths the operator owns. The ring parameters +// must stay in step with the storage nodes, and a mismatch is silent: it resolves a different owner +// set, so writes land where no read will look for them. +var ingestReservedConfigPaths = map[string]string{ + keyCluster: "use spec.cluster and spec.etcd.endpoints", +} + +func ingestEnabled(cr *dbv1alpha1.OtelDBCluster) bool { return cr.Spec.Ingest != nil } + +func ingestReplicasOf(cr *dbv1alpha1.OtelDBCluster) int32 { + if !ingestEnabled(cr) { + return 0 + } + if r := cr.Spec.Ingest.Replicas; r != nil { + return *r + } + return 2 +} + +// renderIngestConfig builds odbingest.yml. odbingest holds no data and joins nothing, so the whole +// config is the ring it routes into plus the two listeners it serves. +func renderIngestConfig(cr *dbv1alpha1.OtelDBCluster, etcdEndpoints []string) (string, error) { + cfg := map[string]any{ + keyCluster: ringConfig(cr, etcdEndpoints), + "prometheus_remote_write": map[string]any{ + keyBind: fmt.Sprintf("0.0.0.0:%d", portPromRW), + }, + "otlp": map[string]any{ + "grpc_bind": fmt.Sprintf("0.0.0.0:%d", portOTLPGRPC), + }, + } + + if raw := cr.Spec.Ingest.ExtraConfig; raw != nil && len(raw.Raw) > 0 { + var extra map[string]any + if err := json.Unmarshal(raw.Raw, &extra); err != nil { + return "", invalidSpec("parse spec.ingest.extraConfig: %s", err) + } + if err := validateExtraConfigPaths("spec.ingest.extraConfig", extra, ingestReservedConfigPaths); err != nil { + return "", err + } + deepMerge(cfg, extra) + } + + out, err := yaml.Marshal(cfg) + if err != nil { + return "", fmt.Errorf("marshal ingest config: %w", err) + } + return string(out), nil +} + +// buildIngestConfigMap renders the shared odbingest config into a ConfigMap. +func buildIngestConfigMap(cr *dbv1alpha1.OtelDBCluster, etcdEndpoints []string) (*corev1.ConfigMap, error) { + data, err := renderIngestConfig(cr, etcdEndpoints) + if err != nil { + return nil, err + } + n := namesFor(cr) + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: n.ingestConfigMap(), + Namespace: cr.Namespace, + Labels: ingestCommonLabels(cr), + }, + Data: map[string]string{ingestConfigFileName: data}, + }, nil +} + +// buildIngestService exposes the ingest endpoints across the odbingest pool. This is where OTLP +// exporters and remote write senders point; the cluster's client Service keeps fronting the +// storage nodes. +func buildIngestService(cr *dbv1alpha1.OtelDBCluster) *corev1.Service { + n := namesFor(cr) + svcType := cr.Spec.Ingest.Service.Type + if svcType == "" { + svcType = corev1.ServiceTypeClusterIP + } + + // OTLP/HTTP is published on 4318, the port every OTLP/HTTP exporter targets by default, but + // odbingest serves it on the remote write listener, so it targets that container port. + ports := []corev1.ServicePort{ + { + Name: portNameOTLPGRPC, + Port: portOTLPGRPC, + TargetPort: intstr.FromString(portNameOTLPGRPC), + Protocol: corev1.ProtocolTCP, + }, + { + Name: portNameOTLPHTTP, + Port: portOTLPHTTP, + TargetPort: intstr.FromString(portNameIngestHTTP), + Protocol: corev1.ProtocolTCP, + }, + } + if metricsEnabled(cr) { + ports = append(ports, corev1.ServicePort{ + Name: portNamePromRW, + Port: portPromRW, + TargetPort: intstr.FromString(portNameIngestHTTP), + Protocol: corev1.ProtocolTCP, + }) + } + ports = append(ports, corev1.ServicePort{ + Name: portNameSelfMetric, + Port: portSelfMetric, + TargetPort: intstr.FromString(portNameSelfMetric), + Protocol: corev1.ProtocolTCP, + }) + + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: n.ingestService(), + Namespace: cr.Namespace, + Labels: ingestCommonLabels(cr), + Annotations: cr.Spec.Ingest.Service.Annotations, + }, + Spec: corev1.ServiceSpec{ + Type: svcType, + Selector: ingestSelectorLabels(cr), + Ports: ports, + }, + } +} + +// buildIngestDeployment builds the odbingest pool. It is a Deployment, not a StatefulSet: the pods +// hold no data, are not ring members, and need neither a stable identity nor a volume. +func buildIngestDeployment(cr *dbv1alpha1.OtelDBCluster, configHash string) *appsv1.Deployment { + n := namesFor(cr) + spec := cr.Spec.Ingest + + image := spec.Image + if image == "" { + image = imageOf(cr) + } + + serviceAccount := spec.ServiceAccountName + if serviceAccount == "" { + serviceAccount = cr.Spec.ServiceAccountName + } + + podLabels := mergeLabels(ingestCommonLabels(cr), spec.PodLabels) + annotations := mergeLabels(map[string]string{"oteldb.io/config-hash": configHash}, spec.PodAnnotations) + + container := corev1.Container{ + Name: ingestAppName, + Image: image, + ImagePullPolicy: cr.Spec.ImagePullPolicy, + Command: []string{ingestBinPath}, + Args: []string{"--config=" + configMountPath + "/" + ingestConfigFileName}, + Env: ingestPodEnv(cr), + Ports: []corev1.ContainerPort{ + {Name: portNameOTLPGRPC, ContainerPort: portOTLPGRPC, Protocol: corev1.ProtocolTCP}, + {Name: portNameIngestHTTP, ContainerPort: portPromRW, Protocol: corev1.ProtocolTCP}, + {Name: portNameSelfMetric, ContainerPort: portSelfMetric, Protocol: corev1.ProtocolTCP}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: configVolumeName, MountPath: configMountPath}, + }, + Resources: spec.Resources, + SecurityContext: spec.SecurityContext, + LivenessProbe: ingestProbe("/healthz"), + // /readyz answers 503 until the ring has a member, keeping a starting pod out of the load + // balancer while a write against an empty ring could only fail. + ReadinessProbe: ingestProbe("/readyz"), + } + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: n.ingestDeployment(), + Namespace: cr.Namespace, + Labels: ingestCommonLabels(cr), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(ingestReplicasOf(cr)), + Selector: &metav1.LabelSelector{MatchLabels: ingestSelectorLabels(cr)}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + Annotations: annotations, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: serviceAccount, + ImagePullSecrets: cr.Spec.ImagePullSecrets, + SecurityContext: spec.PodSecurityContext, + NodeSelector: spec.NodeSelector, + Affinity: affinityOr(spec.Affinity, ingestSelectorLabels(cr)), + Tolerations: spec.Tolerations, + TopologySpreadConstraints: spec.TopologySpreadConstraints, + Containers: []corev1.Container{container}, + Volumes: []corev1.Volume{{ + Name: configVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: n.ingestConfigMap()}, + }, + }, + }}, + }, + }, + }, + } +} + +// ingestPodEnv is the odbingest pod environment. Unlike a storage pod it carries no ring identity: +// odbingest joins nothing, so it has no id, address or zone to advertise. +func ingestPodEnv(cr *dbv1alpha1.OtelDBCluster) []corev1.EnvVar { + env := []corev1.EnvVar{ + {Name: "OTEL_EXPORTER_PROMETHEUS_HOST", Value: "0.0.0.0"}, + {Name: "OTEL_EXPORTER_PROMETHEUS_PORT", Value: fmt.Sprintf("%d", portSelfMetric)}, + } + if lvl := cr.Spec.LogLevel; lvl != "" { + env = append(env, corev1.EnvVar{Name: "OTEL_LOG_LEVEL", Value: lvl}) + } + return env +} + +func ingestProbe(path string) *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Path: path, Port: intstr.FromString(portNameIngestHTTP)}, + }, + PeriodSeconds: 10, + TimeoutSeconds: 3, + FailureThreshold: 3, + } +} diff --git a/internal/controller/ingest_test.go b/internal/controller/ingest_test.go new file mode 100644 index 0000000..8cf185f --- /dev/null +++ b/internal/controller/ingest_test.go @@ -0,0 +1,199 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + "sigs.k8s.io/yaml" + + dbv1alpha1 "github.com/oteldb/operator/api/v1alpha1" +) + +// ingestCluster is testCluster with an ingest pool and a non-default ring shape, so that a renderer +// dropping a ring parameter is visible. +func ingestCluster() *dbv1alpha1.OtelDBCluster { + cr := testCluster() + cr.Spec.Cluster.ReplicationFactor = ptr.To[int32](2) + cr.Spec.Cluster.ShardsPerTenant = ptr.To[int32](8) + cr.Spec.Cluster.EtcdPrefix = "/tenants/a" + cr.Spec.Ingest = &dbv1alpha1.IngestSpec{Replicas: ptr.To[int32](4)} + return cr +} + +func renderedIngest(t *testing.T, cr *dbv1alpha1.OtelDBCluster) map[string]any { + t.Helper() + + out, err := renderIngestConfig(cr, cr.Spec.Etcd.Endpoints) + require.NoError(t, err) + + var cfg map[string]any + require.NoError(t, yaml.Unmarshal([]byte(out), &cfg)) + return cfg +} + +func TestIngestDisabledByDefault(t *testing.T) { + cr := testCluster() + require.False(t, ingestEnabled(cr), "an unchanged cluster must stay symmetric") + require.Zero(t, ingestReplicasOf(cr)) +} + +func TestRenderIngestConfig(t *testing.T) { + cfg := renderedIngest(t, ingestCluster()) + + cluster, ok := cfg["cluster"].(map[string]any) + require.True(t, ok, "cluster block missing") + require.Equal(t, []any{"http://etcd:2379"}, cluster["etcd"]) + require.EqualValues(t, 2, cluster["rf"]) + require.EqualValues(t, 8, cluster["shards_per_tenant"]) + require.Equal(t, "/tenants/a", cluster["root"]) + // odbingest is not a ring member, so it has no peer port to advertise. + require.NotContains(t, cluster, "port") + + rw, ok := cfg["prometheus_remote_write"].(map[string]any) + require.True(t, ok, "prometheus_remote_write block missing") + require.Equal(t, "0.0.0.0:19291", rw["bind"]) + + otlp, ok := cfg["otlp"].(map[string]any) + require.True(t, ok, "otlp block missing") + require.Equal(t, "0.0.0.0:4317", otlp["grpc_bind"]) +} + +// A ring parameter that differs between the roles does not fail: the pool resolves a different +// owner set than the storage nodes do, and writes land where no read will look for them. Both +// renderers must therefore agree key for key. +func TestIngestRingMatchesStorageRing(t *testing.T) { + cr := ingestCluster() + + storageOut, err := renderConfig(cr, cr.Spec.Etcd.Endpoints) + require.NoError(t, err) + var storageCfg map[string]any + require.NoError(t, yaml.Unmarshal([]byte(storageOut), &storageCfg)) + + storageRing := storageCfg["storage"].(map[string]any)["cluster"].(map[string]any) + ingestRing := renderedIngest(t, cr)["cluster"].(map[string]any) + + for _, key := range []string{"etcd", "rf", "shards_per_tenant", "root"} { + require.Equal(t, storageRing[key], ingestRing[key], "ring parameter %q differs between roles", key) + } +} + +func TestRenderIngestConfigExtraConfig(t *testing.T) { + cr := ingestCluster() + cr.Spec.Ingest.ExtraConfig = &runtime.RawExtension{ + Raw: []byte(`{"prometheus_remote_write":{"time_threshold":"1h"}}`), + } + + rw := renderedIngest(t, cr)["prometheus_remote_write"].(map[string]any) + require.Equal(t, "1h", rw["time_threshold"]) + require.Equal(t, "0.0.0.0:19291", rw["bind"], "extraConfig must merge into the block, not replace it") +} + +func TestRenderIngestConfigReservedCluster(t *testing.T) { + cr := ingestCluster() + cr.Spec.Ingest.ExtraConfig = &runtime.RawExtension{Raw: []byte(`{"cluster":{"rf":9}}`)} + + _, err := renderIngestConfig(cr, cr.Spec.Etcd.Endpoints) + require.ErrorContains(t, err, "spec.ingest.extraConfig") + + var invalid validationError + require.True(t, errors.As(err, &invalid), "reserved paths must fail as a spec validation error") +} + +func TestBuildIngestDeployment(t *testing.T) { + cr := ingestCluster() + deploy := buildIngestDeployment(cr, "cafebabe") + + require.Equal(t, "obs-ingest", deploy.Name) + require.EqualValues(t, 4, *deploy.Spec.Replicas) + require.Equal(t, "cafebabe", deploy.Spec.Template.Annotations["oteldb.io/config-hash"]) + + container := deploy.Spec.Template.Spec.Containers[0] + require.Equal(t, []string{ingestBinPath}, container.Command) + require.Equal(t, []string{"--config=/etc/otel/odbingest.yml"}, container.Args) + require.Equal(t, defaultImage, container.Image, "the ingest pool defaults to the cluster image") + require.Empty(t, container.VolumeMounts[1:], "odbingest holds no data, so it mounts only its config") + + for _, e := range container.Env { + require.NotContains(t, e.Name, "OTELDB_CLUSTER_", + "odbingest joins nothing and must carry no ring identity") + } + + require.Equal(t, "/readyz", container.ReadinessProbe.HTTPGet.Path) + require.Equal(t, portNameIngestHTTP, container.ReadinessProbe.HTTPGet.Port.StrVal) +} + +func TestBuildIngestDeploymentImageOverride(t *testing.T) { + cr := ingestCluster() + cr.Spec.Ingest.Image = "ghcr.io/oteldb/oteldb:with-odbingest" + + deploy := buildIngestDeployment(cr, "x") + require.Equal(t, "ghcr.io/oteldb/oteldb:with-odbingest", deploy.Spec.Template.Spec.Containers[0].Image) +} + +// The storage StatefulSet's selector is immutable, so the two roles must be told apart by the app +// name alone. Neither selector may match the other role's pods. +func TestIngestAndStorageSelectorsAreDisjoint(t *testing.T) { + cr := ingestCluster() + + storagePods := labels.Set(mergeLabels(commonLabels(cr), cr.Spec.PodLabels)) + ingestPods := labels.Set(mergeLabels(ingestCommonLabels(cr), cr.Spec.Ingest.PodLabels)) + + storageSelector := labels.SelectorFromSet(selectorLabels(cr)) + ingestSelector := labels.SelectorFromSet(ingestSelectorLabels(cr)) + + require.True(t, storageSelector.Matches(storagePods)) + require.True(t, ingestSelector.Matches(ingestPods)) + require.False(t, storageSelector.Matches(ingestPods), "the client Service must not front ingest pods") + require.False(t, ingestSelector.Matches(storagePods), "the ingest Service must not front storage pods") +} + +// odbingest serves OTLP/HTTP on the remote write listener, so the Service publishes the port every +// OTLP/HTTP exporter defaults to and remaps it onto that listener. +func TestBuildIngestServicePorts(t *testing.T) { + svc := buildIngestService(ingestCluster()) + + require.Equal(t, "obs-ingest", svc.Name) + require.Equal(t, corev1.ServiceTypeClusterIP, svc.Spec.Type) + + byName := map[string]corev1.ServicePort{} + for _, p := range svc.Spec.Ports { + byName[p.Name] = p + } + + require.EqualValues(t, portOTLPHTTP, byName[portNameOTLPHTTP].Port) + require.Equal(t, portNameIngestHTTP, byName[portNameOTLPHTTP].TargetPort.StrVal) + require.EqualValues(t, portPromRW, byName[portNamePromRW].Port) + require.Equal(t, portNameIngestHTTP, byName[portNamePromRW].TargetPort.StrVal) + require.EqualValues(t, portOTLPGRPC, byName[portNameOTLPGRPC].Port) +} + +func TestBuildIngestServiceDropsRemoteWriteWithoutMetrics(t *testing.T) { + cr := ingestCluster() + cr.Spec.Signals.Metrics = ptr.To(false) + + svc := buildIngestService(cr) + for _, p := range svc.Spec.Ports { + require.NotEqual(t, portNamePromRW, p.Name) + } +} diff --git a/internal/controller/naming.go b/internal/controller/naming.go index 2005630..ba81726 100644 --- a/internal/controller/naming.go +++ b/internal/controller/naming.go @@ -28,7 +28,9 @@ const defaultImage = "ghcr.io/oteldb/oteldb:v0.46.0" // Repeated string values, extracted so a single source of truth drives config and labels. const ( appName = "oteldb" // app label value and container name + ingestAppName = "odbingest" // ingest app label value, container name and binary valStorage = "storage" // oteldb signal-backend value and component label + valIngest = "ingest" // ingest component label defaultDataDir = "/var/lib/oteldb" // default storage.dir / WAL dir keyBind = "bind" // oteldb per-API bind config key @@ -44,6 +46,7 @@ const ( keyProfilesBackend = "profiles_backend" keyBackend = "backend" + keyCluster = "cluster" keyDir = "dir" keyEtcd = "etcd" keyPort = "port" @@ -76,13 +79,21 @@ const ( portNameSelfMetric = "metrics" portNameHealth = "health-check" portNamePeer = "peer" + // portNameIngestHTTP is odbingest's single HTTP listener: OTLP/HTTP, Prometheus remote write + // and the health endpoints all share it (see cmd/odbingest/app.go, where otlp.Register and the + // remote write handler are mounted on one mux bound to prometheus_remote_write.bind). + portNameIngestHTTP = "ingest-http" ) const ( - configVolumeName = "config" - dataVolumeName = "data" - configMountPath = "/etc/otel" - configFileName = "config.yml" + configVolumeName = "config" + dataVolumeName = "data" + configMountPath = "/etc/otel" + configFileName = "config.yml" + ingestConfigFileName = "odbingest.yml" + // ingestBinPath is the odbingest entrypoint inside the oteldb image, which defaults to the + // oteldb binary. + ingestBinPath = "/usr/local/bin/odbingest" ) // resourceNames centralizes the derived names for a cluster's child objects. @@ -92,27 +103,51 @@ type resourceNames struct { func namesFor(cr *dbv1alpha1.OtelDBCluster) resourceNames { return resourceNames{base: cr.Name} } -func (n resourceNames) statefulSet() string { return n.base } -func (n resourceNames) configMap() string { return n.base + "-config" } -func (n resourceNames) peerService() string { return n.base + "-peers" } -func (n resourceNames) clientService() string { return n.base } - -// selectorLabels are the immutable pod-selector labels for a cluster's oteldb pods. -func selectorLabels(cr *dbv1alpha1.OtelDBCluster) map[string]string { +func (n resourceNames) statefulSet() string { return n.base } +func (n resourceNames) configMap() string { return n.base + "-config" } +func (n resourceNames) peerService() string { return n.base + "-peers" } +func (n resourceNames) clientService() string { return n.base } +func (n resourceNames) ingestDeployment() string { return n.base + "-ingest" } +func (n resourceNames) ingestConfigMap() string { return n.base + "-ingest-config" } +func (n resourceNames) ingestService() string { return n.base + "-ingest" } + +// roleSelectorLabels are the immutable pod-selector labels for one role's pods. The role's binary +// is the app name, which is what keeps the storage and ingest pod sets from selecting each other: +// the storage StatefulSet's selector is immutable after creation, so it must keep exactly the +// labels it has always had. +func roleSelectorLabels(cr *dbv1alpha1.OtelDBCluster, app string) map[string]string { return map[string]string{ - "app.kubernetes.io/name": appName, + "app.kubernetes.io/name": app, "app.kubernetes.io/instance": cr.Name, } } -// commonLabels are selectorLabels plus non-selector metadata labels. -func commonLabels(cr *dbv1alpha1.OtelDBCluster) map[string]string { - l := selectorLabels(cr) +// roleCommonLabels are roleSelectorLabels plus non-selector metadata labels. +func roleCommonLabels(cr *dbv1alpha1.OtelDBCluster, app, component string) map[string]string { + l := roleSelectorLabels(cr, app) l["app.kubernetes.io/managed-by"] = "oteldb-operator" - l["app.kubernetes.io/component"] = valStorage + l["app.kubernetes.io/component"] = component return l } +// selectorLabels are the pod-selector labels for a cluster's oteldb storage pods. +func selectorLabels(cr *dbv1alpha1.OtelDBCluster) map[string]string { + return roleSelectorLabels(cr, appName) +} + +// commonLabels are selectorLabels plus non-selector metadata labels. +func commonLabels(cr *dbv1alpha1.OtelDBCluster) map[string]string { + return roleCommonLabels(cr, appName, valStorage) +} + +func ingestSelectorLabels(cr *dbv1alpha1.OtelDBCluster) map[string]string { + return roleSelectorLabels(cr, ingestAppName) +} + +func ingestCommonLabels(cr *dbv1alpha1.OtelDBCluster) map[string]string { + return roleCommonLabels(cr, ingestAppName, valIngest) +} + // mergeLabels returns a new map combining base with extra (extra wins). func mergeLabels(base, extra map[string]string) map[string]string { out := make(map[string]string, len(base)+len(extra)) diff --git a/internal/controller/oteldbcluster_controller.go b/internal/controller/oteldbcluster_controller.go index fbb6448..f5f58c7 100644 --- a/internal/controller/oteldbcluster_controller.go +++ b/internal/controller/oteldbcluster_controller.go @@ -44,7 +44,7 @@ type OtelDBClusterReconciler struct { // +kubebuilder:rbac:groups=db.oteldb.io,resources=oteldbclusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=db.oteldb.io,resources=oteldbclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=db.oteldb.io,resources=oteldbclusters/finalizers,verbs=update -// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=statefulsets;deployments,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=services;configmaps,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch @@ -104,6 +104,47 @@ func (r *OtelDBClusterReconciler) reconcile(ctx context.Context, cr *dbv1alpha1. if err := r.apply(ctx, cr, buildStatefulSet(cr, hash)); err != nil { return fmt.Errorf("statefulset: %w", err) } + return r.reconcileIngest(ctx, cr, endpoints) +} + +// reconcileIngest applies the stateless odbingest pool, or removes it when spec.ingest is absent. +func (r *OtelDBClusterReconciler) reconcileIngest(ctx context.Context, cr *dbv1alpha1.OtelDBCluster, endpoints []string) error { + if !ingestEnabled(cr) { + return r.pruneIngest(ctx, cr) + } + + cm, err := buildIngestConfigMap(cr, endpoints) + if err != nil { + return err + } + if err := r.apply(ctx, cr, cm); err != nil { + return fmt.Errorf("ingest configmap: %w", err) + } + hash := configHash(cm.Data[ingestConfigFileName]) + + if err := r.apply(ctx, cr, buildIngestService(cr)); err != nil { + return fmt.Errorf("ingest service: %w", err) + } + if err := r.apply(ctx, cr, buildIngestDeployment(cr, hash)); err != nil { + return fmt.Errorf("ingest deployment: %w", err) + } + return nil +} + +// pruneIngest deletes the ingest pool's objects. Owner references only collect them when the whole +// cluster goes away, so dropping spec.ingest has to remove them explicitly. +func (r *OtelDBClusterReconciler) pruneIngest(ctx context.Context, cr *dbv1alpha1.OtelDBCluster) error { + n := namesFor(cr) + objs := []client.Object{ + &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: n.ingestDeployment(), Namespace: cr.Namespace}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: n.ingestService(), Namespace: cr.Namespace}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: n.ingestConfigMap(), Namespace: cr.Namespace}}, + } + for _, obj := range objs { + if err := r.Delete(ctx, obj); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("delete ingest %T: %w", obj, err) + } + } return nil } @@ -154,6 +195,11 @@ func reconcileInto(dst, src client.Object) { d.Spec.ServiceName = s.Spec.ServiceName d.Spec.PodManagementPolicy = s.Spec.PodManagementPolicy d.Spec.Template = s.Spec.Template + case *appsv1.Deployment: + s := src.(*appsv1.Deployment) + d.Spec.Replicas = s.Spec.Replicas + d.Spec.Selector = s.Spec.Selector + d.Spec.Template = s.Spec.Template } } @@ -167,11 +213,27 @@ func (r *OtelDBClusterReconciler) updateStatus(ctx context.Context, cr *dbv1alph return err } + ingestDesired := ingestReplicasOf(cr) + ingestReady := int32(0) + if ingestEnabled(cr) { + deploy := &appsv1.Deployment{} + key := client.ObjectKey{Namespace: cr.Namespace, Name: namesFor(cr).ingestDeployment()} + if err := r.Get(ctx, key, deploy); err == nil { + ingestReady = deploy.Status.ReadyReplicas + } else if !apierrors.IsNotFound(err) { + return err + } + } + cr.Status.Replicas = desired cr.Status.ReadyReplicas = ready + cr.Status.IngestReplicas = ingestDesired + cr.Status.IngestReadyReplicas = ingestReady cr.Status.EtcdEndpoints = cr.Spec.Etcd.Endpoints cr.Status.ObservedGeneration = cr.Generation + // The phase tracks the storage nodes: they are what holds the data, and an ingest pool that is + // still rolling out does not make the cluster unavailable. switch { case ready == 0: cr.Status.Phase = dbv1alpha1.PhasePending @@ -219,6 +281,7 @@ func (r *OtelDBClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&dbv1alpha1.OtelDBCluster{}). Owns(&appsv1.StatefulSet{}). + Owns(&appsv1.Deployment{}). Owns(&corev1.Service{}). Owns(&corev1.ConfigMap{}). Named("oteldbcluster"). diff --git a/internal/controller/resources.go b/internal/controller/resources.go index 4220be9..90a6367 100644 --- a/internal/controller/resources.go +++ b/internal/controller/resources.go @@ -143,17 +143,12 @@ func buildStatefulSet(cr *dbv1alpha1.OtelDBCluster, configHash string) *appsv1.S n := namesFor(cr) replicas := replicasOf(cr) - image := cr.Spec.Image - if image == "" { - image = defaultImage - } - podLabels := mergeLabels(commonLabels(cr), cr.Spec.PodLabels) annotations := mergeLabels(map[string]string{"oteldb.io/config-hash": configHash}, cr.Spec.PodAnnotations) container := corev1.Container{ Name: appName, - Image: image, + Image: imageOf(cr), ImagePullPolicy: cr.Spec.ImagePullPolicy, Args: []string{"--config=" + configMountPath + "/" + configFileName}, Env: podEnv(cr), @@ -310,11 +305,25 @@ func dataClaim(cr *dbv1alpha1.OtelDBCluster) corev1.PersistentVolumeClaim { } } +// imageOf is the cluster's oteldb image. +func imageOf(cr *dbv1alpha1.OtelDBCluster) string { + if cr.Spec.Image != "" { + return cr.Spec.Image + } + return defaultImage +} + // affinityFor returns the user's affinity, or a default soft anti-affinity spreading replicas // across nodes. func affinityFor(cr *dbv1alpha1.OtelDBCluster) *corev1.Affinity { - if cr.Spec.Affinity != nil { - return cr.Spec.Affinity + return affinityOr(cr.Spec.Affinity, selectorLabels(cr)) +} + +// affinityOr returns given, or a default soft anti-affinity spreading the pods matching selector +// across nodes. +func affinityOr(given *corev1.Affinity, selector map[string]string) *corev1.Affinity { + if given != nil { + return given } return &corev1.Affinity{ PodAntiAffinity: &corev1.PodAntiAffinity{ @@ -322,7 +331,7 @@ func affinityFor(cr *dbv1alpha1.OtelDBCluster) *corev1.Affinity { Weight: 100, PodAffinityTerm: corev1.PodAffinityTerm{ TopologyKey: "kubernetes.io/hostname", - LabelSelector: &metav1.LabelSelector{MatchLabels: selectorLabels(cr)}, + LabelSelector: &metav1.LabelSelector{MatchLabels: selector}, }, }}, },