feat(alloy): add node, instance and cluster labels and kubelet volume stats

CHANGELOG.md

@@ -5,6 +5,10 @@
 - BREAKING: replaces the `grafana-agent` subchart with Grafana Alloy (`alloy` chart 1.13.1). Rename any `grafana-agent:` values overrides to `alloy:`.
 - Pod `job` label changes from `<ns>/grafana-agent/grafana-agent` to `<ns>/alloy/alloy`; update queries and alerts that match the old label.
 - Alloy listens on port 12345 instead of 80.
+- Series now carry the labels common Kubernetes dashboards select on. cAdvisor series get a `node` label, kube-state-metrics' own `node` label is no longer overwritten by the node its pod runs on, and `node_*` series keep their `instance` label (it is still dropped on everything else).
+- New `global.telemetry.metrics.externalLabels` adds labels (for example `cluster`) to every series the collector writes. Empty by default.
+- New `alloy.kubeletVolumeStats` scrapes `kubelet_volume_stats_*` (PVC capacity and usage) from each kubelet, enabled by default. Set `alloy.kubeletVolumeStats.enabled: false` to turn it off.
+- New `alloy.kubeletTLS` configures TLS for the cAdvisor and kubelet scrapes. The defaults (`insecure_skip_verify: true`) match the previous behaviour; set `insecure_skip_verify: false` and `ca_file` to verify the kubelet certificate.
 - Alloy `ClusterRole` is narrowed to `pods`, `nodes` and `nodes/metrics`. It no longer grants read access to `secrets` and `configmaps`. If you add components through `alloy.extraBlocks` that need more access, override `alloy.rbac.rules` and `alloy.rbac.clusterRules`.
 
 ## v0.3.0

README.md

@@ -329,6 +329,13 @@ values which are defined [here](https://github.com/grafana/helm-charts/tree/main
 | alloy.enabled | bool | `true` |  |
 | alloy.extraBlocks | string | `""` |  |
 | alloy.fullnameOverride | string | `"alloy"` |  |
+| alloy.kubeletTLS | object | `{"ca_file":"","insecure_skip_verify":true}` | TLS settings for scraping each node's kubelet (cAdvisor and volume stats) |
+| alloy.kubeletTLS.ca_file | string | `""` | CA bundle used to verify the kubelet's serving certificate, e.g. /var/run/secrets/kubernetes.io/serviceaccount/ca.crt; set together with insecure_skip_verify: false |
+| alloy.kubeletTLS.insecure_skip_verify | bool | `true` | skip verifying the kubelet's serving certificate; kubelets usually serve a self-signed one (EKS included, unless serverTLSBootstrap is enabled and the CSRs are approved) |
+| alloy.kubeletVolumeStats | object | `{"enabled":true,"scrape_interval":"60s","scrape_timeout":"30s"}` | scrape kubelet_volume_stats_* (PVC capacity and usage) from each kubelet's /metrics |
+| alloy.kubeletVolumeStats.enabled | bool | `true` | enable the kubelet volume stats scrape |
+| alloy.kubeletVolumeStats.scrape_interval | string | `"60s"` | how often to scrape each kubelet |
+| alloy.kubeletVolumeStats.scrape_timeout | string | `"30s"` | how long a request will be allowed to wait before being canceled |
 | alloy.podLogsRelabelRules | string | `""` |  |
 | alloy.podMetricsRelabelRules | string | `""` |  |
 | alloy.rbac.clusterRules[0].apiGroups[0] | string | `""` |  |
@@ -364,8 +371,9 @@ values which are defined [here](https://github.com/grafana/helm-charts/tree/main
 | global.externalZone | string | `"svc.cluster.local"` |  |
 | global.postgres | object | `{"alerts":{"groups":{"Basic":{"delay":"1m","enabled":true},"Connections":{"delay":"5m","enabled":true,"thresholds":{"critical":0.9,"notify":0.5,"warning":0.8}},"Notifications":{"delay":"15m","enabled":true,"thresholds":{"critical":0.9,"notify":0.5,"warning":0.8}}}},"database":"coder","exporter":{"enabled":true,"image":"quay.io/prometheuscommunity/postgres-exporter"},"hostname":"localhost","mountSecret":"secret-postgres","password":null,"port":5432,"sslmode":"disable","sslrootcert":null,"username":"coder","volumeMounts":[],"volumes":[]}` | postgres connection information NOTE: these settings are global so we can parameterise some values which get rendered by subcharts |
 | global.postgres.alerts | object | `{"groups":{"Basic":{"delay":"1m","enabled":true},"Connections":{"delay":"5m","enabled":true,"thresholds":{"critical":0.9,"notify":0.5,"warning":0.8}},"Notifications":{"delay":"15m","enabled":true,"thresholds":{"critical":0.9,"notify":0.5,"warning":0.8}}}}` | alerts for postgres |
-| global.telemetry | object | `{"metrics":{"native_histograms":false,"scrape_interval":"15s","scrape_timeout":"12s"},"profiling":{"scrape_interval":"60s","scrape_timeout":"70s"}}` | control telemetry collection |
-| global.telemetry.metrics | object | `{"native_histograms":false,"scrape_interval":"15s","scrape_timeout":"12s"}` | control metric collection |
+| global.telemetry | object | `{"metrics":{"externalLabels":{},"native_histograms":false,"scrape_interval":"15s","scrape_timeout":"12s"},"profiling":{"scrape_interval":"60s","scrape_timeout":"70s"}}` | control telemetry collection |
+| global.telemetry.metrics | object | `{"externalLabels":{},"native_histograms":false,"scrape_interval":"15s","scrape_timeout":"12s"}` | control metric collection |
+| global.telemetry.metrics.externalLabels | object | `{}` | labels added to every series the collector writes, e.g. `cluster: my-cluster` |
 | global.telemetry.metrics.native_histograms | bool | `false` | enable Prometheus native histograms or default to classic histograms |
 | global.telemetry.metrics.scrape_interval | string | `"15s"` | how often the collector will scrape discovered pods |
 | global.telemetry.metrics.scrape_timeout | string | `"12s"` | how long a request will be allowed to wait before being canceled |

coder-observability/templates/_collector-config.tpl

@@ -288,6 +288,14 @@ prometheus.relabel "pods" {
     target_label  = "namespace"
     replacement   = "$1"
   }
+  // The pod target `node` label is the node kube-state-metrics itself runs on, which moves the node each
+  // kube_* series is actually about into exported_node.
+  rule {
+    source_labels = ["__name__", "exported_node"]
+    regex         = "^kube_.+;(.+)$"
+    target_label  = "node"
+    replacement   = "$1"
+  }
   rule {
     regex         = "^(exported_.*|image_.*|container_id|id|uid)$"
     action        = "labeldrop"
@@ -300,14 +308,21 @@ discovery.relabel "cadvisor" {
     replacement   = "/metrics/cadvisor"
     target_label  = "__metrics_path__"
   }
+  rule {
+    source_labels = ["__meta_kubernetes_node_name"]
+    target_label  = "node"
+  }
 }
 
 prometheus.scrape "cadvisor" {
   targets    = discovery.relabel.cadvisor.output
   forward_to = [ prometheus.relabel.cadvisor.receiver ]
   scheme     = "https"
   tls_config {
-    insecure_skip_verify = true
+    insecure_skip_verify = {{ $agent.kubeletTLS.insecure_skip_verify }}
+    {{- with $agent.kubeletTLS.ca_file }}
+    ca_file = {{ . | quote }}
+    {{- end }}
   }
   bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
   scrape_interval   = "{{ .Values.global.telemetry.metrics.scrape_interval }}"
@@ -360,17 +375,84 @@ prometheus.relabel "cadvisor" {
   }
 }
 
+{{- if $agent.kubeletVolumeStats.enabled }}
+
+// kubelet volume stats (PVC capacity and usage) are only served on the kubelet's own /metrics
+discovery.relabel "kubelet" {
+  targets = discovery.kubernetes.nodes.targets
+  rule {
+    source_labels = ["__meta_kubernetes_node_name"]
+    target_label  = "node"
+  }
+}
+
+prometheus.scrape "kubelet" {
+  targets    = discovery.relabel.kubelet.output
+  forward_to = [ prometheus.relabel.kubelet.receiver ]
+  scheme     = "https"
+  tls_config {
+    insecure_skip_verify = {{ $agent.kubeletTLS.insecure_skip_verify }}
+    {{- with $agent.kubeletTLS.ca_file }}
+    ca_file = {{ . | quote }}
+    {{- end }}
+  }
+  bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+  scrape_interval   = "{{ $agent.kubeletVolumeStats.scrape_interval }}"
+  scrape_timeout    = "{{ $agent.kubeletVolumeStats.scrape_timeout }}"
+
+  clustering {
+    enabled = true
+  }
+}
+
+prometheus.relabel "kubelet" {
+  forward_to = [ prometheus.remote_write.default.receiver ]
+
+  // the kubelet exposes far more than we use; keep only the volume stats
+  rule {
+    source_labels = ["__name__"]
+    regex         = "kubelet_volume_stats_.*"
+    action        = "keep"
+  }
+}
+{{- end }}
+
 prometheus.remote_write "default" {
+{{- with .Values.global.telemetry.metrics.externalLabels }}
+  external_labels = {
+  {{- range $name, $value := . }}
+    {{ $name }} = {{ $value | quote }},
+  {{- end }}
+  }
+{{ end }}
   endpoint {
     send_native_histograms = {{ .Values.global.telemetry.metrics.native_histograms | default false }}
     url ="http://{{ include "prometheus.server.fullname" .Subcharts.prometheus }}.{{ .Release.Namespace }}.{{ .Values.global.zone }}/api/v1/write"
 
     // drop instance label which unnecessarily adds new series when pods are restarted, since pod IPs are dynamically assigned
     // NOTE: "__address__" is mapped to "instance", so will contain <hostname>:<port>
+    // Keep it on node_* series though: node-exporter dashboards tell nodes apart by it, and node-exporter pods
+    // only restart rarely. Stash it in a temporary label, drop instance everywhere, then restore.
+    write_relabel_config {
+      source_labels = ["__name__", "instance"]
+      regex         = "(node_.*);(.+)"
+      target_label  = "keep_instance"
+      replacement   = "$2"
+    }
     write_relabel_config {
       regex  = "instance"
       action = "labeldrop"
     }
+    write_relabel_config {
+      source_labels = ["keep_instance"]
+      regex         = "(.+)"
+      target_label  = "instance"
+      replacement   = "$1"
+    }
+    write_relabel_config {
+      regex  = "keep_instance"
+      action = "labeldrop"
+    }
   }
 }
 

coder-observability/values.yaml

@@ -115,6 +115,8 @@ global:
       scrape_timeout: 12s
       # global.telemetry.metrics.native_histograms -- enable Prometheus native histograms or default to classic histograms
       native_histograms: false
+      # global.telemetry.metrics.externalLabels -- labels added to every series the collector writes, e.g. `cluster: my-cluster`
+      externalLabels: {}
     profiling:
       # global.telemetry.profiling.scrape_interval -- how often the collector will scrape pprof endpoints
       scrape_interval: 60s
@@ -326,6 +328,20 @@ alloy:
     # }
   podMetricsRelabelRules: ""
   podLogsRelabelRules: ""
+  # alloy.kubeletVolumeStats -- scrape kubelet_volume_stats_* (PVC capacity and usage) from each kubelet's /metrics
+  kubeletVolumeStats:
+    # alloy.kubeletVolumeStats.enabled -- enable the kubelet volume stats scrape
+    enabled: true
+    # alloy.kubeletVolumeStats.scrape_interval -- how often to scrape each kubelet
+    scrape_interval: 60s
+    # alloy.kubeletVolumeStats.scrape_timeout -- how long a request will be allowed to wait before being canceled
+    scrape_timeout: 30s
+  # alloy.kubeletTLS -- TLS settings for scraping each node's kubelet (cAdvisor and volume stats)
+  kubeletTLS:
+    # alloy.kubeletTLS.insecure_skip_verify -- skip verifying the kubelet's serving certificate; kubelets usually serve a self-signed one (EKS included, unless serverTLSBootstrap is enabled and the CSRs are approved)
+    insecure_skip_verify: true
+    # alloy.kubeletTLS.ca_file -- CA bundle used to verify the kubelet's serving certificate, e.g. /var/run/secrets/kubernetes.io/serviceaccount/ca.crt; set together with insecure_skip_verify: false
+    ca_file: ""
 
 grafana:
   enabled: true