From ff5b56b2dd9decaa06849883cb82aa0444541c0d Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 14 Oct 2025 02:37:35 +0000 Subject: [PATCH 01/17] feat: add prometheus based grafana dashboard --- .../metrics_prometheus.json | 1604 +++++++++++++++++ 1 file changed, 1604 insertions(+) create mode 100644 src/grafana_dashboards/metrics_prometheus.json diff --git a/src/grafana_dashboards/metrics_prometheus.json b/src/grafana_dashboards/metrics_prometheus.json new file mode 100644 index 0000000000..29cebf3a1b --- /dev/null +++ b/src/grafana_dashboards/metrics_prometheus.json @@ -0,0 +1,1604 @@ +{ + "__inputs": [ + { + "name": "DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0", + "label": "juju_prod-cos-platform-engineering_cafadeb1-c97e-41d6-8c49-7dd8092cf06c_prometheus_0", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.5.3" + }, + { + "type": "panel", + "id": "histogram", + "name": "Histogram", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(1, sum(rate(runner_queue_duration_seconds_bucket[10m])) by (flavor, le))", + "format": "time_series", + "instant": false, + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Max job queue duration seconds by flavor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum(rate(runner_idle_duration_seconds_bucket[10m])) by (flavor, le))", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Runner idle duration seconds (100% quantile)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "description": "Time in seconds to initialize the VM and register the runner on GitHub.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum(rate(runner_spawn_duration_seconds_bucket[10m])) by (flavor, le))", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Runner spawn duration seconds (100% quantile)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "description": "Expected runners count - (idle runners count + busy runners count)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "code", + "expr": "expected_runners_count - (idle_runners_count + busy_runners_count)", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Runner expected count deviation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "description": "Duration of reconciliation (seconds)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum(rate(reconcile_duration_seconds_bucket[10m])) by (flavor, le))", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Reconcile duration (100% quantile)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "builder", + "expr": "idle_runners_count", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Idle runners count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "description": "Expected runners count - (idle runners count + busy runners count)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 3 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 14, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "color" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "code", + "expr": "abs(expected_runners_count - (idle_runners_count + busy_runners_count))", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Expected runners count difference", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "builder", + "expr": "busy_runners_count", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Busy runners count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, delete_vm_duration_seconds_bucket)", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Delete VM duration seconds (95% quantile)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "titleSize": 10, + "valueSize": 32 + }, + "textMode": "auto" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "builder", + "expr": "expected_runners_count", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Expected runners count", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "description": "Number of seconds taken to delete runners from GitHub for each reconcile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, delete_runner_duration_seconds_bucket)", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "GitHub runner delete duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "description": "The number of removed runners from GitHub during reconciliation.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "builder", + "expr": "deleted_runners_total", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Deleted runners total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "description": "OpenStack response time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 15, + "options": { + "bucketOffset": 0, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "code", + "expr": "openstack_http_response_time_bucket", + "legendFormat": "{{label}}", + "range": true, + "refId": "A" + } + ], + "title": "OpenStack response time", + "type": "histogram" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "description": "The number of VMs deleted during reconciliation.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "builder", + "expr": "", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Deleted VMs total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "description": "The number of GitHub runners that are older than maximum creation time but has not yet come online nor taken a job.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 56 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "builder", + "expr": "timed_out_runners_total", + "legendFormat": "{{juju_model}}", + "range": true, + "refId": "A" + } + ], + "title": "Timed out runners total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "description": "The number of GitHub runners without an underlying VM.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 56 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "builder", + "expr": "dangling_runners_total", + "legendFormat": "{{juju_model}}", + "range": true, + "refId": "A" + } + ], + "title": "Dangling runners created", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "description": "The number of GitHub runners not found on GitHub but exists on cloud.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 56 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" + }, + "editorMode": "builder", + "expr": "missing_runners_total", + "legendFormat": "{{juju_model}}", + "range": true, + "refId": "A" + } + ], + "title": "Missing runners total", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GitHub Prometheus Metrics", + "uid": "bf50dbd4-24ef-48ae-bf40-ff5390c19c86", + "version": 20, + "weekStart": "" +} \ No newline at end of file From 02c0bcbf71f88a122cee1a515cad748c197cccb2 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 14 Oct 2025 02:41:13 +0000 Subject: [PATCH 02/17] docs: changelog --- docs/changelog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 59e7b600e5..c0b89be4b3 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,9 @@ This changelog documents user-relevant changes to the GitHub runner charm. +## 2025-10-14 + +- Added Grafana dashboard using Prometheus data source. ## 2025-08-27 From 29cac633a98f1cd238df20470ccdcf7f8e87f13a Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Tue, 11 Nov 2025 13:39:35 +0100 Subject: [PATCH 03/17] update prometheus dashboard --- .../metrics_prometheus.json | 3065 ++++++++--------- 1 file changed, 1504 insertions(+), 1561 deletions(-) diff --git a/src/grafana_dashboards/metrics_prometheus.json b/src/grafana_dashboards/metrics_prometheus.json index 29cebf3a1b..e042dd2fc7 100644 --- a/src/grafana_dashboards/metrics_prometheus.json +++ b/src/grafana_dashboards/metrics_prometheus.json @@ -1,1604 +1,1547 @@ { - "__inputs": [ - { - "name": "DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0", - "label": "juju_prod-cos-platform-engineering_cafadeb1-c97e-41d6-8c49-7dd8092cf06c_prometheus_0", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 160, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 21, + "panels": [], + "title": "Durations", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "__elements": {}, - "__requires": [ + }, + "targets": [ { - "type": "panel", - "id": "bargauge", - "name": "Bar gauge", - "version": "" + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(\n $quantile,\n sum by (flavor, le) (\n rate(runner_queue_duration_seconds_bucket[10m])\n )\n)", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Job queue duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "description": "The amount of time a runner is not used.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" }, - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "9.5.3" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { - "type": "panel", - "id": "histogram", - "name": "Histogram", - "version": "" + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum by(flavor, le) (rate(runner_idle_duration_seconds_bucket[10m])))", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Runner idle duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "description": "Duration of reconciliation (seconds)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum(rate(reconcile_duration_seconds_bucket[10m])) by (flavor, le))", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Reconcile duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "description": "Time in seconds to initialize the VM and register the runner on GitHub.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum by (flavor, le) (rate(runner_spawn_duration_seconds_bucket[10m]))) ", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] + ], + "title": "Runner spawn duration", + "type": "timeseries" }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 11, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(1, sum(rate(runner_queue_duration_seconds_bucket[10m])) by (flavor, le))", - "format": "time_series", - "instant": false, - "legendFormat": "{{flavor}}", - "range": true, - "refId": "A" - } - ], - "title": "Max job queue duration seconds by flavor", - "type": "timeseries" + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "code", - "expr": "histogram_quantile(1, sum(rate(runner_idle_duration_seconds_bucket[10m])) by (flavor, le))", - "legendFormat": "{{flavor}}", - "range": true, - "refId": "A" - } - ], - "title": "Runner idle duration seconds (100% quantile)", - "type": "timeseries" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "description": "Time in seconds to initialize the VM and register the runner on GitHub.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 12, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "code", - "expr": "histogram_quantile(1, sum(rate(runner_spawn_duration_seconds_bucket[10m])) by (flavor, le))", - "legendFormat": "{{flavor}}", - "range": true, - "refId": "A" - } - ], - "title": "Runner spawn duration seconds (100% quantile)", - "type": "timeseries" + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "expr": "histogram_quantile(\n $quantile,\n sum by (flavor, le) (\n rate(delete_vm_duration_seconds_bucket[10m])\n )\n)", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Delete VM duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "description": "Number of seconds taken to delete runners from GitHub for each reconcile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "description": "Expected runners count - (idle runners count + busy runners count)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 17, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "code", - "expr": "expected_runners_count - (idle_runners_count + busy_runners_count)", - "legendFormat": "{{flavor}}", - "range": true, - "refId": "A" - } - ], - "title": "Runner expected count deviation", - "type": "timeseries" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "description": "Duration of reconciliation (seconds)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "code", - "expr": "histogram_quantile(1, sum(rate(reconcile_duration_seconds_bucket[10m])) by (flavor, le))", - "legendFormat": "{{flavor}}", - "range": true, - "refId": "A" - } - ], - "title": "Reconcile duration (100% quantile)", - "type": "timeseries" + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, delete_runner_duration_seconds_bucket)", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "GitHub runner delete duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "description": "OpenStack Server List response time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "builder", - "expr": "idle_runners_count", - "legendFormat": "{{flavor}}", - "range": true, - "refId": "A" - } - ], - "title": "Idle runners count", - "type": "timeseries" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "description": "Expected runners count - (idle runners count + busy runners count)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 3 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 14, - "options": { - "displayMode": "gradient", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "valueMode": "color" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "code", - "expr": "abs(expected_runners_count - (idle_runners_count + busy_runners_count))", - "legendFormat": "{{flavor}}", - "range": true, - "refId": "A" - } - ], - "title": "Expected runners count difference", - "type": "bargauge" + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "expr": "histogram_quantile(\n $quantile,\n sum by (le) (\n rate(openstack_http_response_time_bucket{endpoint=~\".*servers/detail\"}[10m])\n )\n)", + "legendFormat": "{{label}}", + "range": true, + "refId": "A" + } + ], + "title": "OpenStack Server List response time", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 23, + "panels": [], + "title": "Runner count", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "builder", - "expr": "busy_runners_count", - "legendFormat": "{{flavor}}", - "range": true, - "refId": "A" - } - ], - "title": "Busy runners count", - "type": "timeseries" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 3, + "options": { + "displayMode": "basic", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, + "showUnfilled": true, + "valueMode": "color" + }, + "pluginVersion": "9.5.3", + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, delete_vm_duration_seconds_bucket)", - "legendFormat": "{{flavor}}", - "range": true, - "refId": "A" - } - ], - "title": "Delete VM duration seconds (95% quantile)", - "type": "timeseries" + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "expr": "sum by (flavor) (expected_runners_count)", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Expected runners count", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "description": "Expected runners count - (idle runners count + busy runners count)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 3 + } + ] + } }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": { - "titleSize": 10, - "valueSize": 32 - }, - "textMode": "auto" - }, - "pluginVersion": "9.5.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "builder", - "expr": "expected_runners_count", - "legendFormat": "{{flavor}}", - "range": true, - "refId": "A" - } - ], - "title": "Expected runners count", - "type": "stat" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 14, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, + "showUnfilled": true, + "valueMode": "color" + }, + "pluginVersion": "9.5.3", + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "description": "Number of seconds taken to delete runners from GitHub for each reconcile", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, delete_runner_duration_seconds_bucket)", - "legendFormat": "{{flavor}}", - "range": true, - "refId": "A" - } - ], - "title": "GitHub runner delete duration", - "type": "timeseries" + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "expr": "abs(expected_runners_count - (idle_runners_count + busy_runners_count))", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Expected runners count difference", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "description": "The number of removed runners from GitHub during reconciliation.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "builder", - "expr": "deleted_runners_total", - "legendFormat": "{{flavor}}", - "range": true, - "refId": "A" - } - ], - "title": "Deleted runners total", - "type": "timeseries" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "description": "OpenStack response time", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 15, - "options": { - "bucketOffset": 0, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "code", - "expr": "openstack_http_response_time_bucket", - "legendFormat": "{{label}}", - "range": true, - "refId": "A" - } - ], - "title": "OpenStack response time", - "type": "histogram" + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "expr": "sum by(flavor) (idle_runners_count)", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Idle runners count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "description": "The number of VMs deleted during reconciliation.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "builder", - "expr": "", - "legendFormat": "{{flavor}}", - "range": true, - "refId": "A" - } - ], - "title": "Deleted VMs total", - "type": "timeseries" + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "expr": "sum by (flavor) (busy_runners_count)", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Busy runners count", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 22, + "panels": [], + "title": "Totals", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "description": "The number of GitHub runners not found on GitHub but exists on cloud.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 1, + "y": 51 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "description": "The number of GitHub runners that are older than maximum creation time but has not yet come online nor taken a job.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 56 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "builder", - "expr": "timed_out_runners_total", - "legendFormat": "{{juju_model}}", - "range": true, - "refId": "A" - } - ], - "title": "Timed out runners total", - "type": "timeseries" + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "expr": "sum by(juju_application) (increase(missing_runners_total[$__rate_interval]))", + "legendFormat": "{{juju_model}}", + "range": true, + "refId": "A" + } + ], + "title": "Missing runners total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "description": "The number of VMs deleted during reconciliation.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 10, + "y": 51 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "description": "The number of GitHub runners without an underlying VM.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 56 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "builder", - "expr": "dangling_runners_total", - "legendFormat": "{{juju_model}}", - "range": true, - "refId": "A" - } - ], - "title": "Dangling runners created", - "type": "timeseries" + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "expr": "sum by(juju_application) (increase(deleted_vms_total[$__rate_interval]))", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Deleted VMs total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "description": "The number of GitHub runners without an underlying VM.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 1, + "y": 59 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "description": "The number of GitHub runners not found on GitHub but exists on cloud.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 56 - }, - "id": 7, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_JUJU_PROD-COS-PLATFORM-ENGINEERING_CAFADEB1-C97E-41D6-8C49-7DD8092CF06C_PROMETHEUS_0}" - }, - "editorMode": "builder", - "expr": "missing_runners_total", - "legendFormat": "{{juju_model}}", - "range": true, - "refId": "A" - } - ], - "title": "Missing runners total", - "type": "timeseries" + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "expr": "sum by(juju_application) (increase(dangling_runners_total[$__rate_interval]))", + "legendFormat": "{{juju_model}}", + "range": true, + "refId": "A" } - ], - "refresh": "", - "schemaVersion": 38, - "style": "dark", - "tags": [], - "templating": { - "list": [] + ], + "title": "Dangling runners created total", + "type": "timeseries" }, - "time": { - "from": "now-6h", - "to": "now" + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "description": "The number of removed runners from GitHub during reconciliation.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 10, + "y": 59 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "builder", + "expr": "deleted_runners_total", + "legendFormat": "{{flavor}}", + "range": true, + "refId": "A" + } + ], + "title": "Deleted runners total", + "type": "timeseries" }, - "timepicker": {}, - "timezone": "", - "title": "GitHub Prometheus Metrics", - "uid": "bf50dbd4-24ef-48ae-bf40-ff5390c19c86", - "version": 20, - "weekStart": "" + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "description": "The number of GitHub runners that are older than maximum creation time but has not yet come online nor taken a job.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 1, + "y": 67 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "expr": "sum by(juju_application) (increase(timed_out_runners_total[$__rate_interval]))", + "legendFormat": "{{juju_model}}", + "range": true, + "refId": "A" + } + ], + "title": "Timed out runners total", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "1", + "value": "1" + }, + "description": "Define quantile (e.g. 1 = 100 % quantile, 0.75 = 75 % quantile) to be displayed in the histograms", + "hide": 0, + "label": "quantile", + "name": "quantile", + "options": [ + { + "selected": true, + "text": "1", + "value": "1" + } + ], + "query": "1", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GitHub Prometheus Metrics", + "uid": "bf50dbd4-24ef-48ae-bf40-ff5390c19c86", + "version": 45, + "weekStart": "" } \ No newline at end of file From a681018a797e43a7dc6137bea1866029eb92404e Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Tue, 11 Nov 2025 14:31:53 +0100 Subject: [PATCH 04/17] pin maturin --- base_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/base_requirements.txt b/base_requirements.txt index 4c20cc3459..49d4ab0e6c 100644 --- a/base_requirements.txt +++ b/base_requirements.txt @@ -10,3 +10,4 @@ pydantic ==1.10.22 cosl ==1.0.0 PyYAML ==6.0.* pyOpenSSL==25.1.0 +maturin==1.9.4 # issue with Cargo lock v4 on more recent version From ad3595cb18afe1141ad8206c218949f336edfa80 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Tue, 11 Nov 2025 15:00:00 +0100 Subject: [PATCH 05/17] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/grafana_dashboards/metrics_prometheus.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/grafana_dashboards/metrics_prometheus.json b/src/grafana_dashboards/metrics_prometheus.json index e042dd2fc7..45ac7e74b5 100644 --- a/src/grafana_dashboards/metrics_prometheus.json +++ b/src/grafana_dashboards/metrics_prometheus.json @@ -593,7 +593,7 @@ "uid": "PE6731821AF182E02" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, delete_runner_duration_seconds_bucket)", + "expr": "histogram_quantile($quantile, sum by (flavor, le) (rate(delete_runner_duration_seconds_bucket[10m])))", "legendFormat": "{{flavor}}", "range": true, "refId": "A" @@ -1402,7 +1402,7 @@ "uid": "PE6731821AF182E02" }, "editorMode": "builder", - "expr": "deleted_runners_total", + "expr": "sum by(juju_application) (increase(deleted_runners_total[$__rate_interval]))", "legendFormat": "{{flavor}}", "range": true, "refId": "A" From 15d3a86440314eda74cf6a98007372a685e56a68 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Tue, 11 Nov 2025 15:04:47 +0100 Subject: [PATCH 06/17] update changelog date --- docs/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index c0b89be4b3..c7301bdc6d 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,7 +2,7 @@ This changelog documents user-relevant changes to the GitHub runner charm. -## 2025-10-14 +## 2025-11-11 - Added Grafana dashboard using Prometheus data source. From 6b7f56bfb16476666e743e8d7ed3df8103234033 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Tue, 11 Nov 2025 15:44:02 +0100 Subject: [PATCH 07/17] update changelog date --- .../metrics_prometheus.json | 191 ++++++++++++++++-- 1 file changed, 171 insertions(+), 20 deletions(-) diff --git a/src/grafana_dashboards/metrics_prometheus.json b/src/grafana_dashboards/metrics_prometheus.json index 45ac7e74b5..547263f3de 100644 --- a/src/grafana_dashboards/metrics_prometheus.json +++ b/src/grafana_dashboards/metrics_prometheus.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 160, + "id": 6, "links": [], "liveNow": false, "panels": [ @@ -724,8 +724,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -791,8 +790,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -887,8 +885,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -979,8 +976,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1085,8 +1081,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1178,8 +1173,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1271,8 +1265,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1364,8 +1357,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1457,8 +1449,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1511,6 +1502,166 @@ "tags": [], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "label": "Loki datasource", + "multi": true, + "name": "lokids", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "label": "Prometheus datasource", + "multi": true, + "name": "prometheusds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", + "hide": 0, + "includeAll": true, + "label": "Juju unit", + "multi": true, + "name": "juju_unit", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", + "hide": 0, + "includeAll": true, + "label": "Juju application", + "multi": true, + "name": "juju_application", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", + "hide": 0, + "includeAll": true, + "label": "Juju model uuid", + "multi": true, + "name": "juju_model_uuid", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up,juju_model)", + "hide": 0, + "includeAll": true, + "label": "Juju model", + "multi": true, + "name": "juju_model", + "options": [], + "query": { + "query": "label_values(up,juju_model)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "current": { "selected": false, @@ -1542,6 +1693,6 @@ "timezone": "", "title": "GitHub Prometheus Metrics", "uid": "bf50dbd4-24ef-48ae-bf40-ff5390c19c86", - "version": 45, + "version": 1, "weekStart": "" } \ No newline at end of file From 76c05dd29af4fd5c801b388b0a3ba1c49dd3556f Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Tue, 11 Nov 2025 15:44:02 +0100 Subject: [PATCH 08/17] checkin updated dashboard with datasources --- .../metrics_prometheus.json | 191 ++++++++++++++++-- 1 file changed, 171 insertions(+), 20 deletions(-) diff --git a/src/grafana_dashboards/metrics_prometheus.json b/src/grafana_dashboards/metrics_prometheus.json index 45ac7e74b5..547263f3de 100644 --- a/src/grafana_dashboards/metrics_prometheus.json +++ b/src/grafana_dashboards/metrics_prometheus.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 160, + "id": 6, "links": [], "liveNow": false, "panels": [ @@ -724,8 +724,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -791,8 +790,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -887,8 +885,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -979,8 +976,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1085,8 +1081,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1178,8 +1173,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1271,8 +1265,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1364,8 +1357,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1457,8 +1449,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1511,6 +1502,166 @@ "tags": [], "templating": { "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "label": "Loki datasource", + "multi": true, + "name": "lokids", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "label": "Prometheus datasource", + "multi": true, + "name": "prometheusds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", + "hide": 0, + "includeAll": true, + "label": "Juju unit", + "multi": true, + "name": "juju_unit", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", + "hide": 0, + "includeAll": true, + "label": "Juju application", + "multi": true, + "name": "juju_application", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", + "hide": 0, + "includeAll": true, + "label": "Juju model uuid", + "multi": true, + "name": "juju_model_uuid", + "options": [], + "query": { + "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "${prometheusds}" + }, + "definition": "label_values(up,juju_model)", + "hide": 0, + "includeAll": true, + "label": "Juju model", + "multi": true, + "name": "juju_model", + "options": [], + "query": { + "query": "label_values(up,juju_model)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "current": { "selected": false, @@ -1542,6 +1693,6 @@ "timezone": "", "title": "GitHub Prometheus Metrics", "uid": "bf50dbd4-24ef-48ae-bf40-ff5390c19c86", - "version": 45, + "version": 1, "weekStart": "" } \ No newline at end of file From 597a760d86dbfd2e2cbcb0ee098105a511964974 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Tue, 11 Nov 2025 16:25:09 +0100 Subject: [PATCH 09/17] checkin uid with reference to prometheus data source --- .../metrics_prometheus.json | 64 +++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/src/grafana_dashboards/metrics_prometheus.json b/src/grafana_dashboards/metrics_prometheus.json index 547263f3de..17b1ad9343 100644 --- a/src/grafana_dashboards/metrics_prometheus.json +++ b/src/grafana_dashboards/metrics_prometheus.json @@ -38,7 +38,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -116,7 +116,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "code", "exemplar": false, @@ -135,7 +135,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "description": "The amount of time a runner is not used.", "fieldConfig": { @@ -214,7 +214,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(1, sum by(flavor, le) (rate(runner_idle_duration_seconds_bucket[10m])))", @@ -229,7 +229,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "description": "Duration of reconciliation (seconds)", "fieldConfig": { @@ -308,7 +308,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(1, sum(rate(reconcile_duration_seconds_bucket[10m])) by (flavor, le))", @@ -323,7 +323,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "description": "Time in seconds to initialize the VM and register the runner on GitHub.", "fieldConfig": { @@ -402,7 +402,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(1, sum by (flavor, le) (rate(runner_spawn_duration_seconds_bucket[10m]))) ", @@ -417,7 +417,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "description": "", "fieldConfig": { @@ -496,7 +496,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(\n $quantile,\n sum by (flavor, le) (\n rate(delete_vm_duration_seconds_bucket[10m])\n )\n)", @@ -511,7 +511,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "description": "Number of seconds taken to delete runners from GitHub for each reconcile", "fieldConfig": { @@ -590,7 +590,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile($quantile, sum by (flavor, le) (rate(delete_runner_duration_seconds_bucket[10m])))", @@ -605,7 +605,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "description": "OpenStack Server List response time", "fieldConfig": { @@ -684,7 +684,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "histogram_quantile(\n $quantile,\n sum by (le) (\n rate(openstack_http_response_time_bucket{endpoint=~\".*servers/detail\"}[10m])\n )\n)", @@ -712,7 +712,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -762,7 +762,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "sum by (flavor) (expected_runners_count)", @@ -777,7 +777,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "description": "Expected runners count - (idle runners count + busy runners count)", "fieldConfig": { @@ -828,7 +828,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "abs(expected_runners_count - (idle_runners_count + busy_runners_count))", @@ -843,7 +843,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -919,7 +919,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "sum by(flavor) (idle_runners_count)", @@ -934,7 +934,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "fieldConfig": { "defaults": { @@ -1010,7 +1010,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "sum by (flavor) (busy_runners_count)", @@ -1038,7 +1038,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "description": "The number of GitHub runners not found on GitHub but exists on cloud.", "fieldConfig": { @@ -1115,7 +1115,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "sum by(juju_application) (increase(missing_runners_total[$__rate_interval]))", @@ -1130,7 +1130,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "description": "The number of VMs deleted during reconciliation.", "fieldConfig": { @@ -1207,7 +1207,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "sum by(juju_application) (increase(deleted_vms_total[$__rate_interval]))", @@ -1222,7 +1222,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "description": "The number of GitHub runners without an underlying VM.", "fieldConfig": { @@ -1299,7 +1299,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "sum by(juju_application) (increase(dangling_runners_total[$__rate_interval]))", @@ -1314,7 +1314,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "description": "The number of removed runners from GitHub during reconciliation.", "fieldConfig": { @@ -1391,7 +1391,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "builder", "expr": "sum by(juju_application) (increase(deleted_runners_total[$__rate_interval]))", @@ -1406,7 +1406,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "description": "The number of GitHub runners that are older than maximum creation time but has not yet come online nor taken a job.", "fieldConfig": { @@ -1483,7 +1483,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "editorMode": "code", "expr": "sum by(juju_application) (increase(timed_out_runners_total[$__rate_interval]))", From a63982663a3dde0c68e5b82c939edc27eedd2675 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 12 Nov 2025 07:30:43 +0100 Subject: [PATCH 10/17] reorder maturin --- base_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_requirements.txt b/base_requirements.txt index 49d4ab0e6c..56b9663cfa 100644 --- a/base_requirements.txt +++ b/base_requirements.txt @@ -5,9 +5,9 @@ ops>=2.8 pylxd @ git+https://github.com/canonical/pylxd requests typing-extensions +maturin==1.9.4 # issue with Cargo lock v4 on more recent version cryptography==45.0.5 pydantic ==1.10.22 cosl ==1.0.0 PyYAML ==6.0.* pyOpenSSL==25.1.0 -maturin==1.9.4 # issue with Cargo lock v4 on more recent version From 1c71b208d2b5b2b30de05002d084f34fe33172b6 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 12 Nov 2025 08:17:50 +0100 Subject: [PATCH 11/17] use rustup for more recent toolchain --- base_requirements.txt | 1 - charmcraft.yaml | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/base_requirements.txt b/base_requirements.txt index 56b9663cfa..4c20cc3459 100644 --- a/base_requirements.txt +++ b/base_requirements.txt @@ -5,7 +5,6 @@ ops>=2.8 pylxd @ git+https://github.com/canonical/pylxd requests typing-extensions -maturin==1.9.4 # issue with Cargo lock v4 on more recent version cryptography==45.0.5 pydantic ==1.10.22 cosl ==1.0.0 diff --git a/charmcraft.yaml b/charmcraft.yaml index 09e6a6060e..72266be193 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -10,8 +10,12 @@ parts: - git # for installing git source of pylxd - libffi-dev # for cffi - libssl-dev # for cryptography - - rust-all # for cryptography - pkg-config # for cryptography + build-snaps: + - rustup # for cryptography + override-build: | + rustup default stable + craftctl default bases: - build-on: - name: "ubuntu" From 1982345f587d477861dc06f2a634165f9ea83421 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 12 Nov 2025 09:14:00 +0100 Subject: [PATCH 12/17] remove unused fixture --- tests/integration/test_prometheus_metrics.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index f5fa99feb6..3544456f40 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -40,19 +40,6 @@ def k8s_juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, yield juju -# juju.offer is not controller aware, we should manually switch to the microk8s controller. -@pytest.fixture(scope="function", name="switch_microk8s_controller") -def switch_microk8s_controller_fixture(k8s_juju: jubilant.Juju, juju: jubilant.Juju): - """Switch to the MicroK8s controller.""" - original_model_controller_name = juju.model - assert ( - original_model_controller_name - ), f"model & controller name not set: {original_model_controller_name}" - - yield - - -@pytest.mark.usefixtures("switch_microk8s_controller") @pytest.fixture(scope="module", name="prometheus_app") def prometheus_app_fixture(k8s_juju: jubilant.Juju): """Deploy prometheus charm.""" From 61967be532a3303e64843e5d7527925dfd5351ea Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 12 Nov 2025 09:31:06 +0100 Subject: [PATCH 13/17] remove unused fixture --- tests/integration/test_prometheus_metrics.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 3544456f40..8bf8e53eed 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -66,7 +66,6 @@ def prometheus_app_fixture(k8s_juju: jubilant.Juju): return k8s_juju.status().apps["prometheus-k8s"] -@pytest.mark.usefixtures("switch_microk8s_controller") @pytest.fixture(scope="module", name="grafana_app") def grafana_app_fixture(k8s_juju: jubilant.Juju, prometheus_app: AppStatus): """Deploy prometheus charm.""" From d38b0812a95bff6320863f558095e0639bf1cac5 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 12 Nov 2025 10:48:07 +0100 Subject: [PATCH 14/17] fix linting --- src/utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utilities.py b/src/utilities.py index ddbfb9c952..9c64a05de3 100644 --- a/src/utilities.py +++ b/src/utilities.py @@ -23,7 +23,7 @@ # Parameters of the function decorated with retry -ParamT = ParamSpec("ParamT") +ParamT = ParamSpec("ParamT") # pylint: disable=invalid-name # Return type of the function decorated with retry ReturnT = TypeVar("ReturnT") From c036f0fe4c6d61586a242317c6efda900d3c0e04 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 12 Nov 2025 11:16:52 +0100 Subject: [PATCH 15/17] lint --- .../src/github_runner_manager/github_client.py | 2 +- .../src/github_runner_manager/metrics/runner.py | 4 ++-- github-runner-manager/src/github_runner_manager/utilities.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/github-runner-manager/src/github_runner_manager/github_client.py b/github-runner-manager/src/github_runner_manager/github_client.py index 61e42532b8..b1a9ffbeb0 100644 --- a/github-runner-manager/src/github_runner_manager/github_client.py +++ b/github-runner-manager/src/github_runner_manager/github_client.py @@ -43,7 +43,7 @@ class GithubRunnerNotFoundError(Exception): # Parameters of the function decorated with retry -ParamT = ParamSpec("ParamT") +ParamT = ParamSpec("ParamT") # pylint: disable=invalid-name # Return type of the function decorated with retry ReturnT = TypeVar("ReturnT") diff --git a/github-runner-manager/src/github_runner_manager/metrics/runner.py b/github-runner-manager/src/github_runner_manager/metrics/runner.py index 163ac95bad..df31c2d380 100644 --- a/github-runner-manager/src/github_runner_manager/metrics/runner.py +++ b/github-runner-manager/src/github_runner_manager/metrics/runner.py @@ -304,14 +304,14 @@ def _parse_metrics_contents(metrics_contents_map: dict[Path, str | None]) -> _Pa try: pre_job_metrics = PreJobMetrics.parse_obj(json.loads(pre_job)) except (json.JSONDecodeError, ValidationError): - logger.warning("Corrupt pre-job metrics: %s") + logger.warning("Corrupt pre-job metrics: %s", pre_job) post_job_metrics: PostJobMetrics | None = None if post_job := metrics_contents_map.get(POST_JOB_METRICS_FILE_PATH, None): try: post_job_metrics = PostJobMetrics.parse_obj(json.loads(post_job)) except (json.JSONDecodeError, ValidationError): - logger.warning("Corrupt post-job metrics %s") + logger.warning("Corrupt post-job metrics %s", post_job) return _ParsedMetricContents( runner_installed_timestamp=runner_installed_timestamp, diff --git a/github-runner-manager/src/github_runner_manager/utilities.py b/github-runner-manager/src/github_runner_manager/utilities.py index 911bd2dd53..3e95545199 100644 --- a/github-runner-manager/src/github_runner_manager/utilities.py +++ b/github-runner-manager/src/github_runner_manager/utilities.py @@ -16,7 +16,7 @@ # Parameters of the function decorated with retry -ParamT = ParamSpec("ParamT") +ParamT = ParamSpec("ParamT") # pylint: disable=invalid-name # Return type of the function decorated with retry ReturnT = TypeVar("ReturnT") From aa94735f3798d6705b27a9b6fe49ff264fc52d38 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 13 Nov 2025 11:36:53 +0100 Subject: [PATCH 16/17] update dashboard according to review comments --- .../metrics_prometheus.json | 297 ++++++++++++++---- 1 file changed, 233 insertions(+), 64 deletions(-) diff --git a/src/grafana_dashboards/metrics_prometheus.json b/src/grafana_dashboards/metrics_prometheus.json index 17b1ad9343..528a563842 100644 --- a/src/grafana_dashboards/metrics_prometheus.json +++ b/src/grafana_dashboards/metrics_prometheus.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 6, + "id": 160, "links": [], "liveNow": false, "panels": [ @@ -102,8 +102,12 @@ "id": 11, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -200,8 +204,12 @@ "id": 10, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -294,10 +302,16 @@ "id": 6, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "sortBy": "Max", + "sortDesc": true }, "tooltip": { "mode": "single", @@ -388,8 +402,12 @@ "id": 12, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -482,8 +500,12 @@ "id": 19, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -576,8 +598,12 @@ "id": 16, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -593,7 +619,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile($quantile, sum by (flavor, le) (rate(delete_runner_duration_seconds_bucket[10m])))", + "expr": "histogram_quantile(1, sum by (flavor, le) (rate(delete_runner_duration_seconds_bucket[10m])))", "legendFormat": "{{flavor}}", "range": true, "refId": "A" @@ -602,6 +628,19 @@ "title": "GitHub runner delete duration", "type": "timeseries" }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 25, + "panels": [], + "title": "OpenStack", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -665,7 +704,7 @@ "h": 8, "w": 12, "x": 0, - "y": 25 + "y": 26 }, "id": 15, "options": { @@ -673,7 +712,7 @@ "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { "mode": "single", @@ -697,12 +736,106 @@ "type": "timeseries" }, { - "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "description": "Amount of 5xx returned by OpenStack API.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PE6731821AF182E02" + }, + "editorMode": "code", + "expr": "sum(increase(openstack_http_requests_total{status_code=~\"5.*.*\"}[$__rate_interval]))", + "legendFormat": "{{label}}", + "range": true, + "refId": "A" + } + ], + "title": "OpenStack API Internal Server Errors", + "type": "timeseries" + }, + { + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 34 }, "id": 23, "panels": [], @@ -724,7 +857,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -739,7 +873,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 35 }, "id": 3, "options": { @@ -790,7 +924,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -805,7 +940,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 35 }, "id": 14, "options": { @@ -885,7 +1020,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -900,13 +1036,17 @@ "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 43 }, "id": 20, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -976,7 +1116,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -991,13 +1132,17 @@ "h": 8, "w": 12, "x": 12, - "y": 42 + "y": 43 }, "id": 1, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -1028,7 +1173,7 @@ "h": 1, "w": 24, "x": 0, - "y": 50 + "y": 51 }, "id": 22, "panels": [], @@ -1081,7 +1226,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1094,15 +1240,19 @@ }, "gridPos": { "h": 8, - "w": 8, - "x": 1, - "y": 51 + "w": 12, + "x": 0, + "y": 52 }, "id": 7, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -1173,7 +1323,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1187,14 +1338,18 @@ "gridPos": { "h": 8, "w": 12, - "x": 10, - "y": 51 + "x": 12, + "y": 52 }, "id": 9, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -1265,7 +1420,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1278,15 +1434,19 @@ }, "gridPos": { "h": 8, - "w": 8, - "x": 1, - "y": 59 + "w": 12, + "x": 0, + "y": 60 }, "id": 4, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -1357,7 +1517,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1371,14 +1532,18 @@ "gridPos": { "h": 8, "w": 12, - "x": 10, - "y": 59 + "x": 12, + "y": 60 }, "id": 5, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -1393,7 +1558,7 @@ "type": "prometheus", "uid": "${prometheusds}" }, - "editorMode": "builder", + "editorMode": "code", "expr": "sum by(juju_application) (increase(deleted_runners_total[$__rate_interval]))", "legendFormat": "{{flavor}}", "range": true, @@ -1449,7 +1614,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1462,15 +1628,19 @@ }, "gridPos": { "h": 8, - "w": 8, - "x": 1, - "y": 67 + "w": 12, + "x": 0, + "y": 68 }, "id": 8, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -1675,8 +1845,8 @@ "options": [ { "selected": true, - "text": "1", - "value": "1" + "text": "0.5", + "value": "0.5" } ], "query": "1", @@ -1692,7 +1862,6 @@ "timepicker": {}, "timezone": "", "title": "GitHub Prometheus Metrics", - "uid": "bf50dbd4-24ef-48ae-bf40-ff5390c19c86", "version": 1, "weekStart": "" } \ No newline at end of file From 8ab83fed9f25e9fa590e53cd74e078e324880666 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Thu, 13 Nov 2025 11:54:02 +0100 Subject: [PATCH 17/17] update dashboard --- .../metrics_prometheus.json | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/grafana_dashboards/metrics_prometheus.json b/src/grafana_dashboards/metrics_prometheus.json index 528a563842..a9fe12c6a8 100644 --- a/src/grafana_dashboards/metrics_prometheus.json +++ b/src/grafana_dashboards/metrics_prometheus.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 160, + "id": 10, "links": [], "liveNow": false, "panels": [ @@ -225,7 +225,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(1, sum by(flavor, le) (rate(runner_idle_duration_seconds_bucket[10m])))", + "expr": "histogram_quantile(1, sum by(flavor, le) (rate(runner_idle_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[10m])))", "legendFormat": "{{flavor}}", "range": true, "refId": "A" @@ -325,7 +325,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(1, sum(rate(reconcile_duration_seconds_bucket[10m])) by (flavor, le))", + "expr": "histogram_quantile(1, sum by(flavor, le) (rate(reconcile_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[10m])))", "legendFormat": "{{flavor}}", "range": true, "refId": "A" @@ -423,7 +423,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(1, sum by (flavor, le) (rate(runner_spawn_duration_seconds_bucket[10m]))) ", + "expr": "histogram_quantile(1, sum by(flavor, le) (rate(runner_spawn_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[10m])))", "legendFormat": "{{flavor}}", "range": true, "refId": "A" @@ -619,7 +619,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "histogram_quantile(1, sum by (flavor, le) (rate(delete_runner_duration_seconds_bucket[10m])))", + "expr": "histogram_quantile(1, sum by(flavor, le) (rate(delete_runner_duration_seconds_bucket{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[10m])))", "legendFormat": "{{flavor}}", "range": true, "refId": "A" @@ -738,7 +738,7 @@ { "datasource": { "type": "prometheus", - "uid": "PE6731821AF182E02" + "uid": "${prometheusds}" }, "description": "Amount of 5xx returned by OpenStack API.", "fieldConfig": { @@ -891,7 +891,7 @@ "showUnfilled": true, "valueMode": "color" }, - "pluginVersion": "9.5.3", + "pluginVersion": "9.5.21", "targets": [ { "datasource": { @@ -899,7 +899,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum by (flavor) (expected_runners_count)", + "expr": "sum by(flavor) (expected_runners_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", "legendFormat": "{{flavor}}", "range": true, "refId": "A" @@ -958,7 +958,7 @@ "showUnfilled": true, "valueMode": "color" }, - "pluginVersion": "9.5.3", + "pluginVersion": "9.5.21", "targets": [ { "datasource": { @@ -966,7 +966,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "abs(expected_runners_count - (idle_runners_count + busy_runners_count))", + "expr": "abs(expected_runners_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} - (idle_runners_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} + busy_runners_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}))", "legendFormat": "{{flavor}}", "range": true, "refId": "A" @@ -1062,7 +1062,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum by(flavor) (idle_runners_count)", + "expr": "sum by(flavor) (idle_runners_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", "legendFormat": "{{flavor}}", "range": true, "refId": "A" @@ -1158,7 +1158,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum by (flavor) (busy_runners_count)", + "expr": "sum by(flavor) (busy_runners_count{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", "legendFormat": "{{flavor}}", "range": true, "refId": "A" @@ -1268,7 +1268,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum by(juju_application) (increase(missing_runners_total[$__rate_interval]))", + "expr": "sum by(juju_application) (increase(missing_runners_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", "legendFormat": "{{juju_model}}", "range": true, "refId": "A" @@ -1365,7 +1365,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum by(juju_application) (increase(deleted_vms_total[$__rate_interval]))", + "expr": "sum by(juju_application) (increase(deleted_vms_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", "legendFormat": "{{flavor}}", "range": true, "refId": "A" @@ -1462,7 +1462,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum by(juju_application) (increase(dangling_runners_total[$__rate_interval]))", + "expr": "sum by(juju_application) (increase(dangling_runners_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", "legendFormat": "{{juju_model}}", "range": true, "refId": "A" @@ -1559,7 +1559,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum by(juju_application) (increase(deleted_runners_total[$__rate_interval]))", + "expr": "sum by(juju_application) (increase(deleted_runners_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", "legendFormat": "{{flavor}}", "range": true, "refId": "A" @@ -1656,7 +1656,7 @@ "uid": "${prometheusds}" }, "editorMode": "code", - "expr": "sum by(juju_application) (increase(timed_out_runners_total[$__rate_interval]))", + "expr": "sum by(juju_application) (increase(timed_out_runners_total{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}[$__rate_interval]))", "legendFormat": "{{juju_model}}", "range": true, "refId": "A"