From 60b7245d943d4707fe32ffe91926921bd2e2f16a Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Thu, 5 Nov 2026 01:34:23 +0100 Subject: [PATCH] feat(chart): run the Kates database on CloudNativePG Add an opt-in cnpg.enabled value that replaces the bundled single Postgres with a CloudNativePG Cluster: three instances by default, one per node, with synchronous replication that prefers availability. The API and the migration job reach the primary through the -rw Service and read the password from the operator's -app Secret. The JDBC URL gets socket, connect and keepalive timeouts so a dead primary fails fast and the API reconnects to the new one. The chart refuses to render with postgresql.enabled or externalDatabase.enabled set, or without the postgresql.cnpg.io/v1 API. The book documents the install and a tested migration. A single operator replica on the lost node delays failover to 366 s; two replicas spread across nodes brought database reads back in 83 s. --- .github/workflows/ci.yml | 20 +++++++ charts/kates-platform/Chart.yaml | 6 ++- charts/kates/Chart.yaml | 6 ++- charts/kates/README.md | 5 ++ charts/kates/templates/NOTES.txt | 2 +- charts/kates/templates/_helpers.tpl | 23 ++++++-- charts/kates/templates/cnpg-cluster.yaml | 49 +++++++++++++++++ charts/kates/templates/configmap.yaml | 2 +- charts/kates/templates/deployment.yaml | 6 +-- .../templates/kyverno-policy-exceptions.yaml | 2 +- charts/kates/templates/migration-job.yaml | 4 +- charts/kates/templates/networkpolicy.yaml | 9 ++++ charts/kates/templates/secret.yaml | 2 +- charts/kates/values.schema.json | 9 ++++ charts/kates/values.yaml | 30 +++++++++-- cli/cmd/mcp_tools_scenario.go | 2 +- docs/book/12-deployment.md | 52 +++++++++++++++++++ docs/book/appendix-d-versions.md | 4 +- 18 files changed, 209 insertions(+), 24 deletions(-) create mode 100644 charts/kates/templates/cnpg-cluster.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36a32b820..80fdd9eec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -202,6 +202,26 @@ jobs: # Without its own tolerations the API pod waits out Kubernetes' default # 300 s on a lost node. A user's tolerations must not drop them. + - name: Assert cnpg.enabled replaces the database + run: | + on=(--set cnpg.enabled=true --set postgresql.enabled=false --api-versions postgresql.cnpg.io/v1) + out=$(helm template kates charts/kates/ "${on[@]}") + cluster=$(printf '%s\n' "$out" | yq 'select(.kind == "Cluster")') + [ "$(printf '%s\n' "$cluster" | yq .spec.instances)" = 3 ] + [ "$(printf '%s\n' "$cluster" | yq .spec.affinity.podAntiAffinityType)" = required ] + [ "$(printf '%s\n' "$cluster" | yq .spec.postgresql.synchronous.dataDurability)" = preferred ] + [ "$(printf '%s\n' "$cluster" | yq .spec.bootstrap.initdb.database)" = kates ] + if printf '%s\n' "$out" | grep -q '^kind: StatefulSet'; then exit 1; fi + printf '%s\n' "$out" | grep -q 'jdbc:postgresql://kates-db-rw\.' + printf '%s\n' "$out" | grep -A3 'name: QUARKUS_DATASOURCE_PASSWORD' | grep -q 'name: kates-db-app' + printf '%s\n' "$out" | grep -q 'until nc -z kates-db-rw\.' + printf '%s\n' "$out" | grep -q 'cnpg.io/cluster: kates-db' + # One instance has no standby to wait for. + if helm template kates charts/kates/ "${on[@]}" --set cnpg.instances=1 | grep -q synchronous:; then exit 1; fi + # Refused: two databases, and no operator. + if helm template kates charts/kates/ --set cnpg.enabled=true --api-versions postgresql.cnpg.io/v1 >/dev/null 2>&1; then exit 1; fi + if helm template kates charts/kates/ --set cnpg.enabled=true --set postgresql.enabled=false >/dev/null 2>&1; then exit 1; fi + - name: Assert the API pod leaves a lost node quickly run: | out=$(helm template kates charts/kates/ \ diff --git a/charts/kates-platform/Chart.yaml b/charts/kates-platform/Chart.yaml index ea6bedc49..3d9ae1f44 100644 --- a/charts/kates-platform/Chart.yaml +++ b/charts/kates-platform/Chart.yaml @@ -3,9 +3,9 @@ name: kates-platform description: Umbrella chart for the full Kates platform — Kafka, Kates, and supporting infrastructure icon: https://raw.githubusercontent.com/bmscomp/kates/main/docs/assets/charts/kates-platform.svg type: application -# Patch: the umbrella packs kates 0.16.0, kafka-cluster 1.0.31, +# Patch: the umbrella packs kates 0.17.0, kafka-cluster 1.0.31, # connect-cluster 2.1.26, mirror-maker2 0.11.26. -version: 0.8.26 +version: 0.8.27 appVersion: "1.0.0" keywords: - kafka @@ -42,6 +42,8 @@ dependencies: condition: connect-cluster.enabled annotations: artifacthub.io/changes: | + - kind: changed + description: "Packs kates 0.17.0, with the opt-in CloudNativePG database (kates.cnpg.enabled). Templates are unchanged." - kind: changed description: "Packs kates 0.16.0, whose API pod is replaced 30 s after its node is lost (nodeFailureTolerationSeconds). Templates are unchanged." - kind: changed diff --git a/charts/kates/Chart.yaml b/charts/kates/Chart.yaml index 83f0cff12..970486a79 100644 --- a/charts/kates/Chart.yaml +++ b/charts/kates/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kates description: Kates — Kafka Advanced Testing & Engineering Suite type: application -# Minor: nodeFailureTolerationSeconds, a new value. -version: 0.16.0 +# Minor: cnpg, a new value block. +version: 0.17.0 # The non-blocking health endpoints this chart's probes now point at come from # a BUILD-TIME property (quarkus.messaging.health.enabled=false), so they only # exist in the image built from this branch. Keep appVersion and @@ -27,6 +27,8 @@ annotations: artifacthub.io/category: integration-delivery artifacthub.io/license: Apache-2.0 artifacthub.io/changes: | + - kind: added + description: "cnpg.enabled renders a CloudNativePG Cluster and connects the API to its primary, so the database survives a node crash. It needs the operator installed first and postgresql.enabled=false. Off by default." - kind: added description: "nodeFailureTolerationSeconds, default 30: the API pod is replaced 30 s after its node goes NotReady or unreachable, not the 300 s Kubernetes defaults to. A user's tolerations are kept beside these two." - kind: changed diff --git a/charts/kates/README.md b/charts/kates/README.md index d64411c31..366553e48 100644 --- a/charts/kates/README.md +++ b/charts/kates/README.md @@ -214,6 +214,11 @@ Upgrading from 0.10.7 or earlier: `values-prod.yaml` there ran an autoscaler of | `postgresql.enabled` | `true` | Deploy bundled PostgreSQL | | `postgresql.auth.existingSecret` | `""` | Pre-created Secret holding the DB password (the username is `postgresql.auth.username`) | | `postgresql.auth.passwordKey` | `password` | Key in that Secret holding the password | +| `cnpg.enabled` | `false` | Replicated PostgreSQL through CloudNativePG; needs the operator and `postgresql.enabled=false` | +| `cnpg.instances` | `3` | PostgreSQL instances, each on its own node | +| `cnpg.storage.size` | `10Gi` | Volume size of each instance | +| `cnpg.storage.storageClass` | `""` | Storage class; the cluster's default when empty | +| `cnpg.resources` | see `values.yaml` | Requests and limits of each instance | | `externalDatabase.enabled` | `false` | Use an external PostgreSQL | | `externalDatabase.existingSecret` | `""` | Pre-created Secret holding the external DB password (the username is `externalDatabase.username`) | | `externalDatabase.passwordKey` | `password` | Key in that Secret holding the password | diff --git a/charts/kates/templates/NOTES.txt b/charts/kates/templates/NOTES.txt index d75c4116b..37518900a 100644 --- a/charts/kates/templates/NOTES.txt +++ b/charts/kates/templates/NOTES.txt @@ -48,7 +48,7 @@ Status: RBAC: {{ if .Values.rbac.create }}enabled{{ else }}disabled{{ end }} Engine: {{ .Values.engine.defaultBackend }} - Database: {{ if .Values.postgresql.enabled }}bundled PostgreSQL{{ else if .Values.externalDatabase.enabled }}external{{ else }}none{{ end }} + Database: {{ if .Values.postgresql.enabled }}bundled PostgreSQL{{ else if .Values.cnpg.enabled }}CloudNativePG{{ else if .Values.externalDatabase.enabled }}external{{ else }}none{{ end }} Replicas: {{ if .Values.autoscaling.enabled }}auto ({{ .Values.autoscaling.minReplicas }}-{{ .Values.autoscaling.maxReplicas }}){{ else }}{{ .Values.replicaCount }}{{ end }} {{- if .Values.metrics.serviceMonitor.enabled }} diff --git a/charts/kates/templates/_helpers.tpl b/charts/kates/templates/_helpers.tpl index fce636f62..755692d8e 100644 --- a/charts/kates/templates/_helpers.tpl +++ b/charts/kates/templates/_helpers.tpl @@ -42,16 +42,31 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{ include "kates.fullname" . }}-postgresql {{- end }} +{{- define "kates.cnpg.fullname" -}} +{{ include "kates.fullname" . }}-db +{{- end }} + +{{/* The host the chart waits for: the bundled database's, or the primary's Service. */}} +{{- define "kates.dbHost" -}} +{{- $name := ternary (printf "%s-rw" (include "kates.cnpg.fullname" .)) (include "kates.postgresql.fullname" .) .Values.cnpg.enabled -}} +{{ $name }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }} +{{- end }} + {{- define "kates.postgresql.jdbcUrl" -}} {{- if .Values.postgresql.enabled -}} jdbc:postgresql://{{ include "kates.postgresql.fullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:5432/{{ .Values.postgresql.auth.database }} +{{- else if .Values.cnpg.enabled -}} +{{- /* Timeouts: a connection to a dead node otherwise hangs on TCP for minutes after a failover. */ -}} +jdbc:postgresql://{{ include "kates.dbHost" . }}:5432/{{ .Values.externalDatabase.database }}?socketTimeout=60&tcpKeepAlive=true&connectTimeout=5 {{- else -}} jdbc:postgresql://{{ .Values.externalDatabase.host }}:{{ .Values.externalDatabase.port | default 5432 }}/{{ .Values.externalDatabase.database }} {{- end -}} {{- end }} {{- define "kates.dbSecretName" -}} -{{- if .Values.postgresql.enabled -}} +{{- if .Values.cnpg.enabled -}} + {{- include "kates.cnpg.fullname" . -}}-app +{{- else if .Values.postgresql.enabled -}} {{- if .Values.postgresql.auth.existingSecret -}} {{- .Values.postgresql.auth.existingSecret -}} {{- else -}} @@ -74,7 +89,9 @@ postgresql.auth.username or externalDatabase.username. */}} {{- define "kates.dbPasswordKey" -}} {{- $db := ternary .Values.postgresql.auth .Values.externalDatabase .Values.postgresql.enabled -}} -{{- if $db.existingSecret -}} +{{- if .Values.cnpg.enabled -}} + password +{{- else if $db.existingSecret -}} {{- $db.passwordKey | default "password" -}} {{- else -}} QUARKUS_DATASOURCE_PASSWORD @@ -91,7 +108,7 @@ migration Job. The Secret is required: a wrong name or key stops the pod with CreateContainerConfigError instead of logging in with no password. */}} {{- define "kates.dbEnv" -}} -{{- if or .Values.postgresql.enabled .Values.externalDatabase.enabled -}} +{{- if or .Values.postgresql.enabled .Values.externalDatabase.enabled .Values.cnpg.enabled -}} - name: QUARKUS_DATASOURCE_USERNAME value: {{ include "kates.dbUsername" . | quote }} - name: QUARKUS_DATASOURCE_PASSWORD diff --git a/charts/kates/templates/cnpg-cluster.yaml b/charts/kates/templates/cnpg-cluster.yaml new file mode 100644 index 000000000..c20935f03 --- /dev/null +++ b/charts/kates/templates/cnpg-cluster.yaml @@ -0,0 +1,49 @@ +{{- if .Values.cnpg.enabled }} +{{- if or .Values.postgresql.enabled .Values.externalDatabase.enabled }} + {{- fail "cnpg.enabled replaces the database: set postgresql.enabled=false and externalDatabase.enabled=false." }} +{{- end }} +{{- if not (.Capabilities.APIVersions.Has "postgresql.cnpg.io/v1") }} + {{- fail "cnpg.enabled needs the CloudNativePG operator: install it first, see the deployment chapter of the book. helm template needs --api-versions postgresql.cnpg.io/v1." }} +{{- end }} +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: {{ include "kates.cnpg.fullname" . }} + labels: + {{- include "kates.labels" . | nindent 4 }} + app.kubernetes.io/component: database +spec: + instances: {{ .Values.cnpg.instances }} + primaryUpdateStrategy: unsupervised + inheritedMetadata: + labels: + app.kubernetes.io/name: {{ include "kates.cnpg.fullname" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: database + affinity: + enablePodAntiAffinity: true + podAntiAffinityType: required + topologyKey: kubernetes.io/hostname + {{- if gt (int .Values.cnpg.instances) 1 }} + postgresql: + # preferred: with no standby left the primary keeps writing, so a Kates API + # never stalls on its database. A double failure can lose the last writes. + synchronous: + method: any + number: 1 + dataDurability: preferred + {{- end }} + bootstrap: + initdb: + database: {{ .Values.externalDatabase.database }} + owner: {{ include "kates.dbUsername" . }} + storage: + size: {{ .Values.cnpg.storage.size }} + {{- with .Values.cnpg.storage.storageClass }} + storageClass: {{ . | quote }} + {{- end }} + {{- with .Values.cnpg.resources }} + resources: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/kates/templates/configmap.yaml b/charts/kates/templates/configmap.yaml index f2b5c8dca..416d04615 100644 --- a/charts/kates/templates/configmap.yaml +++ b/charts/kates/templates/configmap.yaml @@ -49,7 +49,7 @@ data: KATES_ENGINE_SUBMISSION_TIMEOUT_MS: {{ .Values.engine.submissionTimeoutMs | default "120000" | quote }} KATES_API_SECURITY_ENABLED: {{ .Values.apiKey.enabled | default false | quote }} - {{- if or .Values.postgresql.enabled .Values.externalDatabase.enabled }} + {{- if or .Values.postgresql.enabled .Values.externalDatabase.enabled .Values.cnpg.enabled }} QUARKUS_DATASOURCE_DB_KIND: "postgresql" QUARKUS_DATASOURCE_JDBC_URL: {{ include "kates.postgresql.jdbcUrl" . | quote }} QUARKUS_DATASOURCE_JDBC_MAX_SIZE: {{ .Values.postgresql.connectionPool.maxSize | default "20" | quote }} diff --git a/charts/kates/templates/deployment.yaml b/charts/kates/templates/deployment.yaml index eb05ee758..cf5265cc7 100644 --- a/charts/kates/templates/deployment.yaml +++ b/charts/kates/templates/deployment.yaml @@ -56,13 +56,13 @@ spec: topologySpreadConstraints: {{- toYaml . | nindent 8 }} {{- end }} - {{- if or .Values.postgresql.enabled .Values.extraInitContainers }} + {{- if or .Values.postgresql.enabled .Values.cnpg.enabled .Values.extraInitContainers }} initContainers: - {{- if .Values.postgresql.enabled }} + {{- if or .Values.postgresql.enabled .Values.cnpg.enabled }} - name: wait-for-postgres image: {{ include "kates.imageRef" (dict "image" .Values.initContainers.waitForPostgres.image "ctx" .) | quote }} imagePullPolicy: {{ .Values.initContainers.waitForPostgres.pullPolicy | default "IfNotPresent" }} - command: ['sh', '-c', 'until nc -z {{ include "kates.postgresql.fullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }} 5432; do echo "Waiting for PostgreSQL..."; sleep 2; done'] + command: ['sh', '-c', 'until nc -z {{ include "kates.dbHost" . }} 5432; do echo "Waiting for PostgreSQL..."; sleep 2; done'] {{- with .Values.initContainers.waitForPostgres.securityContext | default .Values.containerSecurityContext }} securityContext: {{- toYaml . | nindent 10 }} diff --git a/charts/kates/templates/kyverno-policy-exceptions.yaml b/charts/kates/templates/kyverno-policy-exceptions.yaml index a4e849bc8..05c824ca5 100644 --- a/charts/kates/templates/kyverno-policy-exceptions.yaml +++ b/charts/kates/templates/kyverno-policy-exceptions.yaml @@ -81,6 +81,6 @@ spec: matchExpressions: - key: app.kubernetes.io/name operator: In - values: [{{ include "kates.name" $ | quote }}, {{ include "kates.postgresql.fullname" $ | quote }}] + values: [{{ include "kates.name" $ | quote }}, {{ include "kates.postgresql.fullname" $ | quote }}, {{ include "kates.cnpg.fullname" $ | quote }}] {{- end }} {{- end }} diff --git a/charts/kates/templates/migration-job.yaml b/charts/kates/templates/migration-job.yaml index 2b04fe1f8..40492e222 100644 --- a/charts/kates/templates/migration-job.yaml +++ b/charts/kates/templates/migration-job.yaml @@ -25,12 +25,12 @@ spec: securityContext: {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.postgresql.enabled }} + {{- if or .Values.postgresql.enabled .Values.cnpg.enabled }} initContainers: - name: wait-for-postgres image: {{ include "kates.imageRef" (dict "image" .Values.initContainers.waitForPostgres.image "ctx" .) | quote }} imagePullPolicy: {{ .Values.initContainers.waitForPostgres.pullPolicy | default "IfNotPresent" }} - command: ['sh', '-c', 'until nc -z {{ include "kates.postgresql.fullname" . }}.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }} 5432; do echo "Waiting for PostgreSQL..."; sleep 2; done'] + command: ['sh', '-c', 'until nc -z {{ include "kates.dbHost" . }} 5432; do echo "Waiting for PostgreSQL..."; sleep 2; done'] {{- with .Values.initContainers.waitForPostgres.securityContext | default .Values.containerSecurityContext }} securityContext: {{- toYaml . | nindent 10 }} diff --git a/charts/kates/templates/networkpolicy.yaml b/charts/kates/templates/networkpolicy.yaml index b0046c445..3240503d0 100644 --- a/charts/kates/templates/networkpolicy.yaml +++ b/charts/kates/templates/networkpolicy.yaml @@ -47,6 +47,15 @@ spec: - port: 5432 protocol: TCP {{- end }} + {{- if .Values.cnpg.enabled }} + - to: + - podSelector: + matchLabels: + cnpg.io/cluster: {{ include "kates.cnpg.fullname" . }} + ports: + - port: 5432 + protocol: TCP + {{- end }} {{- if .Values.externalDatabase.enabled }} - to: - ipBlock: diff --git a/charts/kates/templates/secret.yaml b/charts/kates/templates/secret.yaml index 324b89ab3..e7ffcbceb 100644 --- a/charts/kates/templates/secret.yaml +++ b/charts/kates/templates/secret.yaml @@ -2,7 +2,7 @@ empty credentials. Enabled via productionMode (values-prod.yaml). */}} {{- if .Values.productionMode }} {{- if .Values.postgresql.enabled }} - {{- fail "productionMode: the embedded PostgreSQL StatefulSet is single-replica and intended for dev/test only. Use externalDatabase with existingSecret in production." }} + {{- fail "productionMode: the embedded PostgreSQL StatefulSet is single-replica and intended for dev/test only. Use cnpg.enabled, or externalDatabase with existingSecret, in production." }} {{- end }} {{- if and .Values.externalDatabase.enabled (not .Values.externalDatabase.existingSecret) (not .Values.externalDatabase.password) }} {{- fail "productionMode: externalDatabase has no credentials. Set externalDatabase.existingSecret (recommended) or externalDatabase.password." }} diff --git a/charts/kates/values.schema.json b/charts/kates/values.schema.json index 0481aeffc..2394835b3 100644 --- a/charts/kates/values.schema.json +++ b/charts/kates/values.schema.json @@ -565,6 +565,15 @@ } } }, + "cnpg": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "instances": { "type": "integer", "minimum": 1 }, + "storage": { "type": "object" }, + "resources": { "type": "object" } + } + }, "postgresql": { "type": "object", "properties": { diff --git a/charts/kates/values.yaml b/charts/kates/values.yaml index f44fc2102..c43b8b067 100644 --- a/charts/kates/values.yaml +++ b/charts/kates/values.yaml @@ -579,8 +579,8 @@ metrics: # -- Bundled PostgreSQL configuration. # A self-contained single-instance StatefulSet on the official `postgres` -# image (no Bitnami). For production HA, set enabled=false and point -# externalDatabase at a managed cluster (CloudNativePG recommended). +# image (no Bitnami). It does not recover alone from a node crash: for +# production HA, set enabled=false and use cnpg below. postgresql: # -- Deploy bundled PostgreSQL enabled: true @@ -640,10 +640,30 @@ postgresql: # -- Minimum pool size minSize: "5" +# -- Replicated PostgreSQL through CloudNativePG, which survives a node crash. +# Needs the operator installed first. Set postgresql.enabled=false with it; the +# database and owner names come from externalDatabase.database and .username, +# whose enabled stays false. No backups are configured: see the operator's docs. +cnpg: + # -- Render a CloudNativePG Cluster and connect the API to its primary + enabled: false + # -- PostgreSQL instances, each on its own node + instances: 3 + storage: + # -- Volume size of each instance + size: 10Gi + # -- Storage class; the cluster's default when empty + storageClass: "" + # -- Requests and limits of each instance + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + # -- External database configuration (when postgresql.enabled=false). -# Recommended for production HA: run a CloudNativePG `Cluster` (official -# postgres images, streaming replication, backups) and point host/port at its -# `-rw` service, e.g. host: kates-db-rw.database.svc — no Bitnami involved. externalDatabase: # -- Enable external database enabled: false diff --git a/cli/cmd/mcp_tools_scenario.go b/cli/cmd/mcp_tools_scenario.go index 4165bb7ce..ce78e71db 100644 --- a/cli/cmd/mcp_tools_scenario.go +++ b/cli/cmd/mcp_tools_scenario.go @@ -1262,7 +1262,7 @@ var mcpCaveatsScenario = []mcpCaveat{ mcpAnchoredRef(mcpJava+"engine/TestOrchestrator.java:1045-1091", "TestSpec applyTypeDefaults(", "return merged;"), mcpAnchoredRef("kates/src/main/resources/application.properties:73-114", "# Per-type overrides: STRESS", "kates.tests.roundtrip.throughput=10000"), - mcpAnchoredRef("charts/kates/values.yaml:665-733", "defaults:", "tests:", `throughput: "10000"`), + mcpAnchoredRef("charts/kates/values.yaml:686-754", "defaults:", "tests:", `throughput: "10000"`), mcpAnchoredRef("charts/kates/templates/configmap.yaml:88-184", "KATES_TESTS_LOAD_REPLICATION_FACTOR:", "KATES_TESTS_ROUNDTRIP_NUM_CONSUMERS:"), }, diff --git a/docs/book/12-deployment.md b/docs/book/12-deployment.md index 9ea8a388f..23bd440a4 100644 --- a/docs/book/12-deployment.md +++ b/docs/book/12-deployment.md @@ -71,6 +71,58 @@ The default deployment uses NodePort for all services (Grafana on 30080, Kafka U The Kates API keeps every run in PostgreSQL. The `kates` chart runs its own PostgreSQL on a 1 GiB [PersistentVolumeClaim](appendix-a-glossary.md#gl-persistentvolumeclaim) on the cluster's default StorageClass, so runs survive pod restarts; there is no emptyDir mode. For production, point the chart at an external database instead, with `postgresql.enabled: false` and `externalDatabase`, as `values-prod.yaml` does. [Architecture & Design](02-architecture.md) describes what else the Kates API keeps, and [Run Retention](#run-retention) how old runs are pruned. +### A Database That Survives a Node + +The bundled PostgreSQL does not recover alone from a node crash: its StatefulSet waits for a pod it cannot confirm dead. To survive one, run PostgreSQL through [CloudNativePG](https://cloudnative-pg.io) with `cnpg.enabled`. Install the operator first, once, with cluster-admin. Give it two replicas on different nodes, because the operator is what performs the failover: on a kind cluster with one replica on the lost node, the failover took 366 s. + +```yaml +# cnpg-operator.yaml +replicaCount: 2 +topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: cloudnative-pg +tolerations: + - key: node.kubernetes.io/not-ready + operator: Exists + effect: NoExecute + tolerationSeconds: 30 + - key: node.kubernetes.io/unreachable + operator: Exists + effect: NoExecute + tolerationSeconds: 30 +``` + +```bash +helm upgrade --install cnpg cloudnative-pg \ + --repo https://cloudnative-pg.github.io/charts \ + --version 0.29.1 -n cnpg-system --create-namespace -f cnpg-operator.yaml --wait +helm upgrade --install kates charts/kates -n kates \ + --set postgresql.enabled=false --set cnpg.enabled=true +``` + +This gives three instances, each required on its own node, so `cnpg.instances` must not exceed the nodes you have. With the operator set up as above, killing the node under the primary brought database reads back through the same Kates API pod in 83 s. The primary stays writable with no standby left, so a double failure can lose the last writes. No backups are configured; set them up through the operator. + +To move an existing install, dump the data first, because the upgrade starts an empty database. Run this in the release's namespace with its usual values: + +```bash +kubectl scale deployment/kates --replicas=0 +kubectl exec kates-postgresql-0 -- pg_dump -U kates --clean --if-exists \ + --no-owner --no-privileges kates > kates.sql +helm upgrade kates charts/kates --reuse-values \ + --set postgresql.enabled=false --set cnpg.enabled=true --set migration.enabled=false +kubectl wait --for=condition=Ready cluster.postgresql.cnpg.io/kates-db --timeout=300s +kubectl scale deployment/kates --replicas=0 +(echo 'SET ROLE kates;'; cat kates.sql) | kubectl exec -i kates-db-1 -c postgres -- \ + psql -d kates -q -v ON_ERROR_STOP=1 +kubectl scale deployment/kates --replicas=1 +``` + +The old PersistentVolumeClaim stays until you delete it. + For Kafka broker storage, always use persistent volumes — even in development. Kafka's log retention depends on data being durable, and losing broker data mid-test invalidates results. ## Resource Sizing diff --git a/docs/book/appendix-d-versions.md b/docs/book/appendix-d-versions.md index b0c65d028..c9b6b6a7f 100644 --- a/docs/book/appendix-d-versions.md +++ b/docs/book/appendix-d-versions.md @@ -22,8 +22,8 @@ Every version the platform pins, in one place. This table is generated from the | `kafka-common` chart | 0.1.3 (library) | `charts/kafka-common/Chart.yaml` | | `kafka-ui` chart | 0.3.4 (app v1.5.0) | `charts/kafka-ui/Chart.yaml` | | `kates-chaos` chart | 2.4.4 (app 3.28.0) | `charts/kates-chaos/Chart.yaml` | -| `kates-platform` chart | 0.8.26 (app 1.0.0) | `charts/kates-platform/Chart.yaml` | -| `kates` chart | 0.16.0 (app 1.37.0) | `charts/kates/Chart.yaml` | +| `kates-platform` chart | 0.8.27 (app 1.0.0) | `charts/kates-platform/Chart.yaml` | +| `kates` chart | 0.17.0 (app 1.37.0) | `charts/kates/Chart.yaml` | | `legacy-kafka` chart | 0.2.2 (app 3.9.1) | `charts/legacy-kafka/Chart.yaml` | | `mirror-maker2` chart | 0.11.26 (app 4.3.1) | `charts/mirror-maker2/Chart.yaml` | | `monitoring` chart | 1.8.1 (app 82.4.3) | `charts/monitoring/Chart.yaml` |