diff --git a/.gitignore b/.gitignore index 520f842..b9a6acc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ # exclude .env files for docker setup .env +docker-compose.override.yml /tmp/ /data/ /test/data/ +metrics/grafana/provisioning/dashboards/community/ +neops/metrics/grafana/provisioning/dashboards/community/ diff --git a/README.md b/README.md index 6f0bf68..b8788e7 100644 --- a/README.md +++ b/README.md @@ -18,3 +18,7 @@ Change directory to your `neops-docker-compose` root dir ```shell docker run -it --rm -v $(pwd)/:/app/ndc quay.io/zebbra/neops-core:dc-custom-values ``` + +## Metrics stack + +See [metrics/README.md](metrics/README.md). diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..956a8c0 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,135 @@ +services: + redis: + image: redis:7 + ports: + - "6380:6379" + + postgres: + image: postgres:15-alpine + environment: + POSTGRES_USER: postgres + POSTGRES_DB: postgres + POSTGRES_PASSWORD: unsafe + ports: + - "5433:5432" + volumes: + - ./data/dev_postgres:/var/lib/postgresql/data + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:8.9.2 + environment: + - discovery.type=single-node + - xpack.security.enabled=false + - cluster.routing.allocation.disk.threshold_enabled=false + ports: + - "9200:9200" + + frontend: + build: /home/sgr/code/neops-legacy-frontend + environment: + FRONTEND_NEOPS_URL: http://localhost:8000/graphql + ports: + - "8080:8080" + restart: on-failure + + # ── Observability stack ──────────────────────────────────────────────────── + # Start with: docker compose --profile metrics up + + celery-exporter: + profiles: [metrics] + image: danihodovic/celery-exporter:latest + command: + - "--broker-url=redis://redis:6379/0" + ports: + - "9808:9808" + depends_on: + - redis + restart: unless-stopped + + redis-exporter: + profiles: [metrics] + image: oliver006/redis_exporter:latest + environment: + - REDIS_ADDR=redis://redis:6379 + ports: + - "9121:9121" + depends_on: + - redis + restart: unless-stopped + + postgres-exporter: + profiles: [metrics] + image: prometheuscommunity/postgres-exporter:latest + environment: + - DATA_SOURCE_NAME=postgresql://postgres:unsafe@postgres:5432/postgres?sslmode=disable + ports: + - "9187:9187" + depends_on: + - postgres + restart: unless-stopped + + elasticsearch-exporter: + profiles: [metrics] + image: prometheuscommunity/elasticsearch-exporter:latest + command: + - "--es.uri=http://elasticsearch:9200" + - "--es.all" + - "--es.indices" + ports: + - "9114:9114" + depends_on: + - elasticsearch + restart: unless-stopped + + vmalert: + profiles: [metrics] + image: victoriametrics/vmalert:latest + command: + - "--datasource.url=http://victoriametrics:8428" + - "--remoteWrite.url=http://victoriametrics:8428" + - "--remoteRead.url=http://victoriametrics:8428" + - "--rule=/etc/vmalert/rules/*.yml" + - "--evaluationInterval=1m" + - "--notifier.blackhole=true" + volumes: + - ./metrics/vmalert/rules:/etc/vmalert/rules:ro + ports: + - "8880:8880" + depends_on: + - victoriametrics + restart: unless-stopped + + victoriametrics: + profiles: [metrics] + image: victoriametrics/victoria-metrics:latest + command: + - "--promscrape.config=/etc/victoriametrics/scrape_config.yml" + - "--retentionPeriod=30d" + - "--storageDataPath=/victoria-metrics-data" + volumes: + - ./metrics/scrape_config.yml:/etc/victoriametrics/scrape_config.yml:ro + - vm_data:/victoria-metrics-data + ports: + - "8428:8428" + extra_hosts: + - "host.docker.internal:host-gateway" + restart: unless-stopped + + grafana: + profiles: [metrics] + image: grafana/grafana:11.5.2 + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_USERS_ALLOW_SIGN_UP=false + volumes: + - ./metrics/grafana/provisioning:/etc/grafana/provisioning:ro + - grafana_data:/var/lib/grafana + ports: + - "3000:3000" + restart: unless-stopped + +volumes: + vm_data: + driver: local + grafana_data: + driver: local diff --git a/metrics/README.md b/metrics/README.md new file mode 100644 index 0000000..c356f97 --- /dev/null +++ b/metrics/README.md @@ -0,0 +1,82 @@ +# Metrics stack + +Observability stack for neops: VictoriaMetrics (TSDB), Grafana (dashboards), vmalert (alerting), and exporters for Celery, Redis, PostgreSQL, and Elasticsearch. + +## Usage + +### Dev + +Included in `docker-compose.yaml` under the `metrics` profile: + +```shell +# base services only +docker compose up + +# base + full observability stack +docker compose --profile metrics up +``` + +### Prod + +Add to `COMPOSE_FILE` in your `.env`: + +``` +COMPOSE_FILE=neops/docker-compose.neops.prod.yml:neops/docker-compose.neops.prod.metrics.yml +``` + +## Services + +| Service | URL | Purpose | +|---|---|---| +| Grafana | http://localhost:3000 | Dashboards (admin / admin) | +| VictoriaMetrics | http://localhost:8428 | TSDB + query UI | +| vmalert | http://localhost:8880/vmalert | Alert rules & state | +| celery-exporter | http://localhost:9808/metrics | Raw Celery metrics | +| redis-exporter | http://localhost:9121/metrics | Raw Redis metrics | +| postgres-exporter | http://localhost:9187/metrics | Raw PostgreSQL metrics | +| elasticsearch-exporter | http://localhost:9114/metrics | Raw Elasticsearch metrics | + +**Exporters:** +- `celery-exporter` — Celery task/worker metrics ([danihodovic/celery-exporter](https://github.com/danihodovic/celery-exporter)) +- `redis-exporter` — Redis metrics ([oliver006/redis_exporter](https://github.com/oliver006/redis_exporter)) +- `postgres-exporter` — PostgreSQL metrics ([prometheuscommunity/postgres_exporter](https://github.com/prometheus-community/postgres_exporter)) +- `elasticsearch-exporter` — Elasticsearch cluster/index metrics ([prometheuscommunity/elasticsearch_exporter](https://github.com/prometheus-community/elasticsearch_exporter)) +- `victoriametrics` — Prometheus-compatible TSDB, scrapes all exporters + neops backend (`/metrics`); also self-scraped for storage/ingestion metrics +- `vmalert` — Alert rule evaluation; rules live in `vmalert/rules/`; scraped for rule evaluation health and firing alert counts + +## Grafana dashboards + +Grafana is pre-provisioned with a VictoriaMetrics datasource and two sets of dashboards: + +- **`grafana/provisioning/dashboards/neops/`** — checked into the repo (neops overview, custom metrics, per-task metrics) +- **`grafana/provisioning/dashboards/community/`** — gitignored, fetched by script + +Fetch or refresh community dashboards (VictoriaMetrics, vmalert, Celery, Redis, PostgreSQL, Elasticsearch, Django): + +```shell +bash metrics/fetch-dashboards.sh +``` + +Re-run on a fresh clone before starting Grafana. + +## Scraping the local backend (dev) + +VictoriaMetrics uses `host.docker.internal` (wired via `extra_hosts: host-gateway`) to reach a backend running on the host at port 8000. Add to Django's env: + +``` +DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1,host.docker.internal +``` + +For prod, switch the neops scrape target in `scrape_config.yml` to `backend:8000`. + +## Pairs well with + +The **neops_metrics** enterprise plugin (`neops_modules/enterprise/neops_metrics`) exposes a `/metrics` endpoint on the neops backend with task execution counts, device/scope stats, facts, ES sync lag, Celery queue depth, and a Redis-backed custom metric API for use inside Jinja2 task templates. + +## Grafana 12 upgrade checklist + +- [ ] Delete `grafana_data` volume before upgrading (avoids mixed-state migration) +- [ ] Re-run `fetch-dashboards.sh` +- [ ] Migrate `grafana/provisioning/datasources/prometheus.yml` to the new Kubernetes-style provisioning format (classic file provisioning broken in Grafana 12) +- [ ] Check for plugin auto-install failures on airgapped deployments +- [ ] Verify `$datasource` template variable resolution still works in provisioned dashboards diff --git a/metrics/fetch-dashboards.sh b/metrics/fetch-dashboards.sh new file mode 100755 index 0000000..5ec3515 --- /dev/null +++ b/metrics/fetch-dashboards.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +DASHBOARDS=( + "10229:victoriametrics-overview" + "14950:vmalert" + "17508:celery-exporter" + "17509:celery-overview" + "20076:celery-tasks" + "763:redis-exporter" + "9628:postgres" + "14191:elasticsearch" + "17613:django-overview" + "17616:django-requests" + "17617:django-database" + "24933:django-models" +) + +DATASOURCE_UID="victoriametrics" +OUT="$(dirname "$0")/grafana/provisioning/dashboards/community" +mkdir -p "$OUT" + +for entry in "${DASHBOARDS[@]}"; do + id="${entry%%:*}" + name="${entry##*:}" + echo -n "Fetching $name ($id)... " + curl -sf "https://grafana.com/api/dashboards/${id}/revisions/latest/download" \ + | sed "s/\${DS_PROM}/${DATASOURCE_UID}/g" \ + | sed "s/\${DS_PROMETHEUS}/${DATASOURCE_UID}/g" \ + > "${OUT}/${name}.json" + echo "ok" +done + +echo "Done — dashboards saved to $OUT" diff --git a/metrics/grafana/provisioning/dashboards/dashboards.yml b/metrics/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..18e5e06 --- /dev/null +++ b/metrics/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,13 @@ +apiVersion: 1 +providers: + - name: neops + folder: neops + type: file + options: + path: /etc/grafana/provisioning/dashboards/neops + + - name: community + folder: Community + type: file + options: + path: /etc/grafana/provisioning/dashboards/community diff --git a/metrics/grafana/provisioning/dashboards/neops/neops.json b/metrics/grafana/provisioning/dashboards/neops/neops.json new file mode 100644 index 0000000..cb50ca3 --- /dev/null +++ b/metrics/grafana/provisioning/dashboards/neops/neops.json @@ -0,0 +1,3061 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 32, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Inventory", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(neops_devices_total)", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Devices", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_scopes_total", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Scopes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(neops_tasks_total)", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Tasks defined", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_executions_total{status=\"successful\"}", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Executions (success)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_executions_total{status=\"failed\"}", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Executions (failed)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_devices_total", + "legendFormat": "{{platform}}", + "refId": "A" + } + ], + "title": "Devices by platform", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 8, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_tasks_total", + "legendFormat": "{{scope}}", + "refId": "A" + } + ], + "title": "Tasks by scope", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 9, + "panels": [], + "title": "Task Executions", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 6 + }, + "id": 10, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(neops_task_executions_total{state=\"successful\"}[1h]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Success last 1h", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 6 + }, + "id": 11, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(neops_task_executions_total{state=~\"failed|partial_failed\"}[1h]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Failed last 1h", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 50 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 6 + }, + "id": 12, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(neops_celery_queue_depth)", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Celery queue depth", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 50 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 42, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(neops_task_last_execution_duration_seconds)", + "hide": false, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg(neops_task_last_execution_duration_seconds)", + "legendFormat": "avg", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(neops_task_last_execution_duration_seconds)", + "hide": false, + "legendFormat": "Max", + "range": true, + "refId": "C" + } + ], + "title": "Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (state) (increase(neops_task_executions_total[5m]))", + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "title": "Execution rate by state (per 5 min)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(changes(neops_task_last_execution_duration_seconds[$__rate_interval]) > 0) * neops_task_last_execution_duration_seconds", + "legendFormat": "{{task}}", + "range": true, + "refId": "A" + } + ], + "title": "Execution duration by task", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "ok" + }, + "1": { + "color": "orange", + "index": 1, + "text": "partial" + }, + "2": { + "color": "red", + "index": 2, + "text": "failed" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 18 + }, + "id": 16, + "options": { + "alignValue": "left", + "fillOpacity": 70, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_task_last_state", + "legendFormat": "{{task}}", + "refId": "A" + } + ], + "title": "Task state history", + "type": "state-timeline" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 18 + }, + "id": 17, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(increase(neops_task_execution_duration_seconds_bucket[$__rate_interval])) by (le)", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Execution duration distribution", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "state" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "failed": { + "color": "red", + "index": 1, + "text": "failed" + }, + "partial_failed": { + "color": "orange", + "index": 2, + "text": "partial failed" + }, + "successful": { + "color": "green", + "index": 0, + "text": "successful" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "last run" + }, + "properties": [ + { + "id": "unit", + "value": "dtdurationms" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "count" + }, + "properties": [ + { + "id": "displayName", + "value": "count" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Age" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 15, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Age" + } + ] + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_task_executions_total", + "instant": true, + "legendFormat": "count", + "refId": "count" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "neops_task_last_execution_age_seconds *1", + "instant": true, + "legendFormat": "last run", + "refId": "age" + } + ], + "title": "Executions by task & state", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "keepLabels": [ + "__name__", + "job", + "state", + "task" + ], + "mode": "columns" + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Age" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 18, + "panels": [], + "title": "Task Failures & Exceptions", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "red", + "value": 100 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 19, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(10, sum by (exception) (neops_task_failure_exceptions_total))", + "legendFormat": "{{exception}}", + "refId": "A" + } + ], + "title": "Top failure codes (all time)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 50 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 20, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "topk(10, sum by (exception_class) (neops_task_failure_unknown_exceptions_total))", + "legendFormat": "{{exception_class}}", + "refId": "A" + } + ], + "title": "Unknown exception classes (nornir.unknown.error drill-down)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (exception) (increase(neops_task_failure_exceptions_total[1h]))", + "legendFormat": "{{exception}}", + "refId": "A" + } + ], + "title": "New failures per 1h by exception type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 22, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_task_failure_exceptions_total", + "instant": true, + "legendFormat": "{{task}} / {{exception}}", + "refId": "A" + } + ], + "title": "Failures by task & exception code", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "columns" + } + }, + { + "id": "merge", + "options": {} + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 23, + "panels": [], + "title": "Facts", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 52 + }, + "id": 24, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum(neops_facts_current_total)", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total facts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 52 + }, + "id": 25, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_facts_current_total{type=\"device\"}", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Device facts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 52 + }, + "id": 26, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_facts_current_total{type=\"interface\"}", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Interface facts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 52 + }, + "id": 27, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_facts_current_total{type=\"group\"}", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Group facts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 52 + }, + "id": 28, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_facts_current_total{type=\"client\"}", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Client facts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 52 + }, + "id": 29, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_facts_current_total{type=\"common\"}", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Global facts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 56 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_facts_current_total", + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Current facts by type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 56 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_fact_revisions_total", + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Fact revisions by type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 56 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_fact_last_update_age_seconds", + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Fact staleness — age since last write", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 33, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_facts_by_key_total", + "instant": true, + "legendFormat": "{{type}} / {{key}}", + "refId": "A" + } + ], + "title": "Facts by key (5-min cache)", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "columns" + } + }, + { + "id": "merge", + "options": {} + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 34, + "panels": [], + "title": "Elasticsearch", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "0": { + "color": "green", + "text": "green" + }, + "1": { + "color": "yellow", + "text": "yellow" + }, + "2": { + "color": "red", + "text": "red" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 73 + }, + "id": 35, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_elasticsearch_cluster_status", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Cluster status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 4, + "y": 73 + }, + "id": 36, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_elasticsearch_nodes_total", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 7, + "y": 73 + }, + "id": 37, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_elasticsearch_unassigned_shards", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Unassigned shards", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 10, + "y": 73 + }, + "id": 38, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_elasticsearch_docs_total", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Docs total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 13, + "y": 73 + }, + "id": 39, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_elasticsearch_store_size_bytes", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Store size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_elasticsearch_sync_lag_total", + "legendFormat": "{{model}}", + "refId": "A" + } + ], + "title": "ES sync lag (unsynced objects)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 50 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 77 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "neops_celery_queue_depth", + "legendFormat": "{{queue}}", + "refId": "A" + } + ], + "title": "Celery queue depth", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [ + "neops" + ], + "templating": { + "list": [ + { + "current": {}, + "includeAll": false, + "label": "Datasource", + "name": "datasource", + "query": "prometheus", + "refresh": 1, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "neops / Overview", + "uid": "neops-overview", + "version": 17, + "weekStart": "" +} \ No newline at end of file diff --git a/metrics/grafana/provisioning/dashboards/neops/neops_custom.json b/metrics/grafana/provisioning/dashboards/neops/neops_custom.json new file mode 100644 index 0000000..f6e5f43 --- /dev/null +++ b/metrics/grafana/provisioning/dashboards/neops/neops_custom.json @@ -0,0 +1,143 @@ +{ + "title": "neops / Custom Metrics", + "uid": "neops-custom", + "tags": ["neops"], + "timezone": "browser", + "refresh": "30s", + "schemaVersion": 38, + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource" + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Global Metrics", + "gridPos": { "x": 0, "y": 0, "w": 24, "h": 1 }, + "collapsed": false + }, + { + "type": "bargauge", + "title": "Global metrics — latest values", + "description": "Metrics written via set_global_metric / inc_global_metric (no task label).", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "x": 0, "y": 1, "w": 24, "h": 8 }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true + }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, + "noValue": "no global metrics yet" + } + }, + "targets": [{ + "expr": "{__name__=~\"neops_custom_.*\", task=\"\"}", + "legendFormat": "{{__name__}}", + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }] + }, + { + "type": "timeseries", + "title": "Global metrics — history", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "x": 0, "y": 9, "w": 24, "h": 9 }, + "options": { + "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull", "max"] }, + "tooltip": { "mode": "multi" } + }, + "fieldConfig": { "defaults": { "custom": { "lineWidth": 2, "fillOpacity": 8 } } }, + "targets": [{ + "expr": "{__name__=~\"neops_custom_.*\", task=\"\"}", + "legendFormat": "{{__name__}}", + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }] + }, + { + "type": "row", + "title": "Tasks Overview", + "gridPos": { "x": 0, "y": 18, "w": 24, "h": 1 }, + "collapsed": false + }, + { + "type": "table", + "title": "Tasks with custom metrics", + "description": "Click a task name to open the per-task dashboard.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "x": 0, "y": 19, "w": 12, "h": 10 }, + "options": { "sortBy": [{ "displayName": "metrics", "desc": true }] }, + "fieldConfig": { + "defaults": { "custom": { "filterable": true } }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "task" }, + "properties": [ + { + "id": "links", + "value": [{ + "title": "Open task dashboard →", + "url": "/d/neops-custom-task?var-task=${__data.fields.task}", + "targetBlank": false + }] + } + ] + }, + { + "matcher": { "id": "byName", "options": "Value" }, + "properties": [{ "id": "displayName", "value": "metrics" }] + }, + { "matcher": { "id": "byName", "options": "__name__" }, "properties": [{ "id": "custom.hidden", "value": true }] }, + { "matcher": { "id": "byName", "options": "job" }, "properties": [{ "id": "custom.hidden", "value": true }] } + ] + }, + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } }, + { "id": "merge", "options": {} } + ], + "targets": [{ + "expr": "count by (task) ({__name__=~\"neops_custom_.*\", task!=\"\"})", + "legendFormat": "{{task}}", + "instant": true, + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }] + }, + { + "type": "bargauge", + "title": "Metric series per task", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "x": 12, "y": 19, "w": 12, "h": 10 }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true + }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, + "unit": "short", + "noValue": "0" + } + }, + "targets": [{ + "expr": "count by (task) ({__name__=~\"neops_custom_.*\", task!=\"\"})", + "legendFormat": "{{task}}", + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }] + } + ], + "time": { "from": "now-3h", "to": "now" }, + "timepicker": {} +} diff --git a/metrics/grafana/provisioning/dashboards/neops/neops_custom_task.json b/metrics/grafana/provisioning/dashboards/neops/neops_custom_task.json new file mode 100644 index 0000000..22514a9 --- /dev/null +++ b/metrics/grafana/provisioning/dashboards/neops/neops_custom_task.json @@ -0,0 +1,115 @@ +{ + "title": "neops / Custom Metrics — by Task", + "uid": "neops-custom-task", + "tags": ["neops"], + "timezone": "browser", + "refresh": "30s", + "schemaVersion": 38, + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource" + }, + { + "name": "task", + "type": "query", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "query": "label_values(neops_task_executions_total, task)", + "refresh": 2, + "includeAll": false, + "multi": false, + "label": "Task", + "hide": 0, + "sort": 1 + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Latest Values", + "gridPos": { "x": 0, "y": 0, "w": 24, "h": 1 }, + "collapsed": false + }, + { + "type": "bargauge", + "title": "$task — current values", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "x": 0, "y": 1, "w": 24, "h": 10 }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true + }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, + "noValue": "no data — run the task first" + } + }, + "targets": [{ + "expr": "{__name__=~\"neops_custom_.*\", task=\"$task\"}", + "legendFormat": "{{__name__}}", + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }] + }, + { + "type": "row", + "title": "History", + "gridPos": { "x": 0, "y": 11, "w": 24, "h": 1 }, + "collapsed": false + }, + { + "type": "timeseries", + "title": "$task — metric history", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "x": 0, "y": 12, "w": 24, "h": 10 }, + "options": { + "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull", "max", "mean"] }, + "tooltip": { "mode": "multi" } + }, + "fieldConfig": { "defaults": { "custom": { "lineWidth": 2, "fillOpacity": 8 } } }, + "targets": [{ + "expr": "{__name__=~\"neops_custom_.*\", task=\"$task\"}", + "legendFormat": "{{__name__}}", + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }] + }, + { + "type": "row", + "title": "Snapshot", + "gridPos": { "x": 0, "y": 22, "w": 24, "h": 1 }, + "collapsed": false + }, + { + "type": "table", + "title": "$task — latest metric values", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "x": 0, "y": 23, "w": 24, "h": 8 }, + "options": { "sortBy": [{ "displayName": "Value", "desc": true }] }, + "fieldConfig": { "defaults": { "custom": { "filterable": true } }, "overrides": [ + { "matcher": { "id": "byName", "options": "__name__" }, "properties": [{ "id": "custom.hidden", "value": true }] }, + { "matcher": { "id": "byName", "options": "job" }, "properties": [{ "id": "custom.hidden", "value": true }] } + ]}, + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } }, + { "id": "merge", "options": {} } + ], + "targets": [{ + "expr": "{__name__=~\"neops_custom_.*\", task=\"$task\"}", + "legendFormat": "{{__name__}}", + "instant": true, + "datasource": { "type": "prometheus", "uid": "${datasource}" } + }] + } + ], + "time": { "from": "now-3h", "to": "now" }, + "timepicker": {} +} diff --git a/metrics/grafana/provisioning/datasources/prometheus.yml b/metrics/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..9581899 --- /dev/null +++ b/metrics/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,8 @@ +apiVersion: 1 +datasources: + - name: VictoriaMetrics + uid: victoriametrics + type: prometheus + access: proxy + url: http://victoriametrics:8428 + isDefault: true diff --git a/metrics/scrape_config.yml b/metrics/scrape_config.yml new file mode 100644 index 0000000..db55423 --- /dev/null +++ b/metrics/scrape_config.yml @@ -0,0 +1,35 @@ +global: + scrape_interval: 30s + +scrape_configs: + - job_name: neops + static_configs: + # dev: backend running on host (works on Linux via extra_hosts: host-gateway) + - targets: [host.docker.internal:8000] + # prod: backend running as docker service + # - targets: [backend:8000] + metrics_path: /metrics + + - job_name: celery + static_configs: + - targets: [celery-exporter:9808] + + - job_name: redis + static_configs: + - targets: [redis-exporter:9121] + + - job_name: postgres + static_configs: + - targets: [postgres-exporter:9187] + + - job_name: elasticsearch + static_configs: + - targets: [elasticsearch-exporter:9114] + + - job_name: victoriametrics + static_configs: + - targets: [victoriametrics:8428] + + - job_name: vmalert + static_configs: + - targets: [vmalert:8880] diff --git a/metrics/vmalert/rules/celery.yml b/metrics/vmalert/rules/celery.yml new file mode 100644 index 0000000..e6acaa4 --- /dev/null +++ b/metrics/vmalert/rules/celery.yml @@ -0,0 +1,41 @@ +groups: + - name: celery + interval: 1m + rules: + - alert: CeleryWorkerDown + expr: celery_worker_up == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Celery worker {{ $labels.hostname }} is down" + description: "Worker {{ $labels.hostname }} has not been seen for 2 minutes." + + - alert: CeleryNoActiveWorkers + expr: sum(celery_worker_up) == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "No active Celery workers" + description: "All Celery workers are offline — task execution is halted." + + - alert: CeleryHighFailureRate + expr: | + rate(celery_task_failed_total[15m]) / + (rate(celery_task_succeeded_total[15m]) + rate(celery_task_failed_total[15m]) + 1e-9) > 0.1 + for: 5m + labels: + severity: warning + annotations: + summary: "Celery task failure rate high ({{ $value | humanizePercentage }})" + description: "More than 10% of Celery tasks have failed over the last 15 minutes." + + - alert: CeleryQueueLengthHigh + expr: celery_queue_length > 100 + for: 5m + labels: + severity: warning + annotations: + summary: "Celery queue {{ $labels.queue_name }} length high ({{ $value }})" + description: "Queue {{ $labels.queue_name }} has more than 100 pending tasks." diff --git a/metrics/vmalert/rules/elasticsearch.yml b/metrics/vmalert/rules/elasticsearch.yml new file mode 100644 index 0000000..306d704 --- /dev/null +++ b/metrics/vmalert/rules/elasticsearch.yml @@ -0,0 +1,83 @@ +groups: + - name: elasticsearch + interval: 1m + rules: + - alert: ElasticsearchDown + expr: up{job="elasticsearch"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Elasticsearch exporter is unreachable" + description: "{{ $labels.instance }} has been down for more than 2 minutes." + + - alert: ElasticsearchClusterRed + expr: elasticsearch_cluster_health_status{color="red"} == 1 + for: 1m + labels: + severity: critical + annotations: + summary: "Elasticsearch cluster {{ $labels.cluster }} is RED" + description: "Primary shards are unavailable — cluster is not serving requests." + + - alert: ElasticsearchClusterYellow + expr: elasticsearch_cluster_health_status{color="yellow"} == 1 + for: 10m + labels: + severity: warning + annotations: + summary: "Elasticsearch cluster {{ $labels.cluster }} is YELLOW" + description: "Replica shards are unassigned. Cluster is functional but not fully redundant." + + - alert: ElasticsearchUnassignedShards + expr: elasticsearch_cluster_health_unassigned_shards > 0 + for: 10m + labels: + severity: warning + annotations: + summary: "{{ $value }} unassigned shards in cluster {{ $labels.cluster }}" + description: "Elasticsearch has unassigned shards for more than 10 minutes." + + - alert: ElasticsearchHighHeapUsage + expr: | + elasticsearch_jvm_memory_used_bytes{area="heap"} + / elasticsearch_jvm_memory_max_bytes{area="heap"} > 0.85 + for: 5m + labels: + severity: warning + annotations: + summary: "Elasticsearch JVM heap usage high on {{ $labels.name }} ({{ $value | humanizePercentage }})" + description: "JVM heap is above 85% — GC pressure may be building up." + + - alert: ElasticsearchHeapCritical + expr: | + elasticsearch_jvm_memory_used_bytes{area="heap"} + / elasticsearch_jvm_memory_max_bytes{area="heap"} > 0.95 + for: 2m + labels: + severity: critical + annotations: + summary: "Elasticsearch JVM heap critical on {{ $labels.name }} ({{ $value | humanizePercentage }})" + description: "JVM heap is above 95% — OutOfMemoryError risk." + + - alert: ElasticsearchLowDiskSpace + expr: | + elasticsearch_filesystem_data_available_bytes + / elasticsearch_filesystem_data_size_bytes < 0.15 + for: 5m + labels: + severity: warning + annotations: + summary: "Elasticsearch low disk space on {{ $labels.name }} ({{ $value | humanizePercentage }} free)" + description: "Less than 15% disk space remaining — ES may trigger read-only index protection." + + - alert: ElasticsearchDiskCritical + expr: | + elasticsearch_filesystem_data_available_bytes + / elasticsearch_filesystem_data_size_bytes < 0.05 + for: 2m + labels: + severity: critical + annotations: + summary: "Elasticsearch disk space critical on {{ $labels.name }}" + description: "Less than 5% disk space — indices likely locked read-only." diff --git a/metrics/vmalert/rules/neops.yml b/metrics/vmalert/rules/neops.yml new file mode 100644 index 0000000..27ea495 --- /dev/null +++ b/metrics/vmalert/rules/neops.yml @@ -0,0 +1,59 @@ +groups: + - name: neops + interval: 1m + rules: + - alert: NeopsBackendDown + expr: up{job="neops"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "neops backend is unreachable" + description: "{{ $labels.instance }} has been down for more than 2 minutes." + + - alert: NeopsHighTaskFailureRate + expr: | + sum(rate(neops_task_executions_total{state="FAILURE"}[15m])) / + sum(rate(neops_task_executions_total[15m])) > 0.1 + for: 5m + labels: + severity: warning + annotations: + summary: "High neops task failure rate ({{ $value | humanizePercentage }})" + description: "More than 10% of task executions have failed over the last 15 minutes." + + - alert: NeopsCeleryQueueHigh + expr: sum(neops_celery_queue_depth) > 50 + for: 5m + labels: + severity: warning + annotations: + summary: "Celery queue depth high ({{ $value }})" + description: "Combined Celery queue depth has exceeded 50 for 5 minutes." + + - alert: NeopsCeleryQueueCritical + expr: sum(neops_celery_queue_depth) > 200 + for: 2m + labels: + severity: critical + annotations: + summary: "Celery queue depth critical ({{ $value }})" + description: "Combined Celery queue depth has exceeded 200 — tasks are stacking up." + + - alert: NeopsESSyncLagHigh + expr: sum(neops_elasticsearch_sync_lag_total) > 500 + for: 5m + labels: + severity: warning + annotations: + summary: "Elasticsearch sync lag high ({{ $value }} objects)" + description: "More than 500 neops objects are pending ES sync for over 5 minutes." + + - alert: NeopsESClusterNotGreen + expr: neops_elasticsearch_cluster_status > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "neops reports ES cluster not green (status={{ $value }})" + description: "neops_elasticsearch_cluster_status is non-zero (1=yellow, 2=red)." diff --git a/metrics/vmalert/rules/postgres.yml b/metrics/vmalert/rules/postgres.yml new file mode 100644 index 0000000..ccd2289 --- /dev/null +++ b/metrics/vmalert/rules/postgres.yml @@ -0,0 +1,39 @@ +groups: + - name: postgres + interval: 1m + rules: + - alert: PostgresDown + expr: up{job="postgres"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "PostgreSQL is unreachable" + description: "{{ $labels.instance }} has been down for more than 2 minutes." + + - alert: PostgresExclusiveLocks + expr: sum by (datname) (pg_locks_count{mode="exclusivelock"}) > 10 + for: 5m + labels: + severity: warning + annotations: + summary: "High exclusive lock count on {{ $labels.datname }} ({{ $value }})" + description: "More than 10 exclusive locks held in database {{ $labels.datname }} for 5+ minutes — possible deadlock or long-running transaction." + + - alert: PostgresReplicationLagHigh + expr: pg_replication_lag_seconds > 30 + for: 5m + labels: + severity: warning + annotations: + summary: "PostgreSQL replication lag high ({{ $value | humanizeDuration }})" + description: "Replication is lagging behind the primary by more than 30 seconds." + + - alert: PostgresScrapeFailing + expr: pg_exporter_last_scrape_error > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "postgres-exporter scrape failing" + description: "postgres-exporter cannot collect metrics from {{ $labels.instance }}." diff --git a/metrics/vmalert/rules/redis.yml b/metrics/vmalert/rules/redis.yml new file mode 100644 index 0000000..b3ef9b4 --- /dev/null +++ b/metrics/vmalert/rules/redis.yml @@ -0,0 +1,51 @@ +groups: + - name: redis + interval: 1m + rules: + - alert: RedisDown + expr: up{job="redis"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Redis is unreachable" + description: "{{ $labels.instance }} has been down for more than 2 minutes." + + - alert: RedisHighMemoryUsage + # only fires when maxmemory is configured (> 0) + expr: | + redis_memory_used_bytes / redis_memory_max_bytes > 0.9 + and redis_memory_max_bytes > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Redis memory usage high ({{ $value | humanizePercentage }})" + description: "Redis is using more than 90% of its configured maxmemory." + + - alert: RedisHighEvictionRate + expr: rate(redis_evicted_keys_total[5m]) > 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Redis evicting keys ({{ $value | humanize }}/s)" + description: "Redis is evicting more than 10 keys per second — maxmemory may be too low." + + - alert: RedisManyBlockedClients + expr: redis_blocked_clients > 20 + for: 5m + labels: + severity: warning + annotations: + summary: "Redis has {{ $value }} blocked clients" + description: "A large number of clients are blocked waiting on Redis commands (BLPOP, BRPOP, etc.)." + + - alert: RedisRDBSaveFailed + expr: redis_rdb_last_bgsave_status == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Redis last RDB save failed" + description: "The last Redis background save did not complete successfully." diff --git a/neops/.env.dist b/neops/.env.dist index 82a2fa2..5d6434c 100644 --- a/neops/.env.dist +++ b/neops/.env.dist @@ -48,6 +48,18 @@ ELASTICSEARCH_DSL_AUTOSYNC=True ELASTICSEARCH_DSL_SIGNAL_PROCESSOR=neops.core.documents.neops_signal_processor.NeopsCacheSignalProcessor NEOPS_ELASTIC_CONVERT_FACTS=False +# data retention +EXECUTION_LOG_RETENTION_ENABLED=False +EXECUTION_LOG_RETENTION_DAYS=90 +FACT_CLEANUP_ENABLED=False +FACT_RETENTION_MAX_ENTRIES={"device":20,"interface":20,"device_group":20,"client":20,"common":20} +NEOPS_EXECUTION_CLEANUP_ENABLED=False + +# neops logger configuration (current default is DbLogger) +#NEOPS_LOGGER_CLASS="neops.core.log.loggers.loki.LokiLogger" +NEOPS_LOGGER_CLASS="neops.core.log.loggers.db.DbLogger" +# NEOPS_LOGGER_CLASS="neops.core.log.loggers.optimized_db.OptimizedDbLogger" + # Email EMAIL_URL=smtp+tls:// diff --git a/neops/docker-compose.neops.prod.metrics.yml b/neops/docker-compose.neops.prod.metrics.yml new file mode 100644 index 0000000..781eedc --- /dev/null +++ b/neops/docker-compose.neops.prod.metrics.yml @@ -0,0 +1,105 @@ +services: + celery-exporter: + image: danihodovic/celery-exporter:latest + command: + - "--broker-url=redis://redis:6379/0" + networks: + - private + restart: unless-stopped + + redis-exporter: + image: oliver006/redis_exporter:latest + environment: + - REDIS_ADDR=redis://redis:6379 + networks: + - private + restart: unless-stopped + + postgres-exporter: + image: prometheuscommunity/postgres-exporter:latest + environment: + - DATA_SOURCE_NAME=postgresql://neops:${POSTGRES_PASSWORD}@postgres:5432/neops?sslmode=disable + networks: + - private + restart: unless-stopped + + elasticsearch-exporter: + image: prometheuscommunity/elasticsearch-exporter:latest + command: + - "--es.uri=http://elasticsearch:9200" + - "--es.all" + - "--es.indices" + networks: + - private + restart: unless-stopped + + vmalert: + image: victoriametrics/vmalert:latest + command: + - "--datasource.url=http://victoriametrics:8428" + - "--remoteWrite.url=http://victoriametrics:8428" + - "--remoteRead.url=http://victoriametrics:8428" + - "--rule=/etc/vmalert/rules/*.yml" + - "--evaluationInterval=1m" + - "--notifier.blackhole=true" + volumes: + - ../metrics/vmalert/rules:/etc/vmalert/rules:ro + networks: + - private + depends_on: + - victoriametrics + restart: unless-stopped + + victoriametrics: + image: victoriametrics/victoria-metrics:latest + command: + - "--promscrape.config=/etc/victoriametrics/scrape_config.yml" + - "--retentionPeriod=30d" + - "--storageDataPath=/victoria-metrics-data" + volumes: + - ../metrics/scrape_config.yml:/etc/victoriametrics/scrape_config.yml:ro + - vm_data:/victoria-metrics-data + networks: + - private + extra_hosts: + - "host.docker.internal:host-gateway" + restart: unless-stopped + + # prometheus: + # image: prom/prometheus:latest + # command: + # - "--config.file=/etc/prometheus/prometheus.yml" + # - "--storage.tsdb.path=/prometheus" + # - "--storage.tsdb.retention.time=30d" + # volumes: + # - ../metrics/scrape_config.yml:/etc/prometheus/prometheus.yml:ro + # - prometheus_data:/prometheus + # networks: + # - private + # restart: unless-stopped + + grafana: + image: grafana/grafana:11.5.2 + environment: + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:-admin} + - GF_USERS_ALLOW_SIGN_UP=false + + volumes: + - ../metrics/grafana/provisioning:/etc/grafana/provisioning:ro + - grafana_data:/var/lib/grafana + networks: + - private + restart: unless-stopped + +volumes: + vm_data: + driver: local + # prometheus_data: + # driver: local + grafana_data: + driver: local + +networks: + private: + external: true + name: neops_private diff --git a/neops/docker-compose.neops.prod.yml b/neops/docker-compose.neops.prod.yml index 2366d00..b617dd1 100644 --- a/neops/docker-compose.neops.prod.yml +++ b/neops/docker-compose.neops.prod.yml @@ -37,6 +37,12 @@ services: SENTRY_DSN: "${SENTRY_DSN}" SENTRY_ENVIRONMENT: "${SENTRY_ENVIRONMENT}" EMAIL_URL: "${EMAIL_URL}" + NEOPS_LOGGER_CLASS: "${NEOPS_LOGGER_CLASS}" + EXECUTION_LOG_RETENTION_ENABLED: "${EXECUTION_LOG_RETENTION_ENABLED}" + EXECUTION_LOG_RETENTION_DAYS: "${EXECUTION_LOG_RETENTION_DAYS}" + FACT_CLEANUP_ENABLED: "${FACT_CLEANUP_ENABLED}" + FACT_RETENTION_MAX_ENTRIES: "${FACT_RETENTION_MAX_ENTRIES}" + NEOPS_EXECUTION_CLEANUP_ENABLED: "${NEOPS_EXECUTION_CLEANUP_ENABLED}" IMPORT_DEVICE_DEFAULT_PLATFORM: "${IMPORT_DEVICE_DEFAULT_PLATFORM}" IMPORT_DEVICE_DEFAULT_USERNAME: "${IMPORT_DEVICE_DEFAULT_USERNAME}" IMPORT_DEVICE_DEFAULT_PASSWORD: "${IMPORT_DEVICE_DEFAULT_PASSWORD}" @@ -78,6 +84,12 @@ services: SENTRY_DSN: "${SENTRY_DSN}" SENTRY_ENVIRONMENT: "${SENTRY_ENVIRONMENT}" EMAIL_URL: "${EMAIL_URL}" + NEOPS_LOGGER_CLASS: "${NEOPS_LOGGER_CLASS}" + EXECUTION_LOG_RETENTION_ENABLED: "${EXECUTION_LOG_RETENTION_ENABLED}" + EXECUTION_LOG_RETENTION_DAYS: "${EXECUTION_LOG_RETENTION_DAYS}" + FACT_CLEANUP_ENABLED: "${FACT_CLEANUP_ENABLED}" + FACT_RETENTION_MAX_ENTRIES: "${FACT_RETENTION_MAX_ENTRIES}" + NEOPS_EXECUTION_CLEANUP_ENABLED: "${NEOPS_EXECUTION_CLEANUP_ENABLED}" IMPORT_DEVICE_DEFAULT_PLATFORM: "${IMPORT_DEVICE_DEFAULT_PLATFORM}" IMPORT_DEVICE_DEFAULT_USERNAME: "${IMPORT_DEVICE_DEFAULT_USERNAME}" IMPORT_DEVICE_DEFAULT_PASSWORD: "${IMPORT_DEVICE_DEFAULT_PASSWORD}" @@ -130,6 +142,12 @@ services: SENTRY_DSN: "${SENTRY_DSN}" SENTRY_ENVIRONMENT: "${SENTRY_ENVIRONMENT}" EMAIL_URL: "${EMAIL_URL}" + NEOPS_LOGGER_CLASS: "${NEOPS_LOGGER_CLASS}" + EXECUTION_LOG_RETENTION_ENABLED: "${EXECUTION_LOG_RETENTION_ENABLED}" + EXECUTION_LOG_RETENTION_DAYS: "${EXECUTION_LOG_RETENTION_DAYS}" + FACT_CLEANUP_ENABLED: "${FACT_CLEANUP_ENABLED}" + FACT_RETENTION_MAX_ENTRIES: "${FACT_RETENTION_MAX_ENTRIES}" + NEOPS_EXECUTION_CLEANUP_ENABLED: "${NEOPS_EXECUTION_CLEANUP_ENABLED}" IMPORT_DEVICE_DEFAULT_PLATFORM: "${IMPORT_DEVICE_DEFAULT_PLATFORM}" IMPORT_DEVICE_DEFAULT_USERNAME: "${IMPORT_DEVICE_DEFAULT_USERNAME}" IMPORT_DEVICE_DEFAULT_PASSWORD: "${IMPORT_DEVICE_DEFAULT_PASSWORD}" @@ -167,6 +185,12 @@ services: NEOPS_PLUGINS: "${NEOPS_PLUGINS}" SENTRY_DSN: "${SENTRY_DSN}" SENTRY_ENVIRONMENT: "${SENTRY_ENVIRONMENT}" + NEOPS_LOGGER_CLASS: "${NEOPS_LOGGER_CLASS}" + EXECUTION_LOG_RETENTION_ENABLED: "${EXECUTION_LOG_RETENTION_ENABLED}" + EXECUTION_LOG_RETENTION_DAYS: "${EXECUTION_LOG_RETENTION_DAYS}" + FACT_CLEANUP_ENABLED: "${FACT_CLEANUP_ENABLED}" + FACT_RETENTION_MAX_ENTRIES: "${FACT_RETENTION_MAX_ENTRIES}" + NEOPS_EXECUTION_CLEANUP_ENABLED: "${NEOPS_EXECUTION_CLEANUP_ENABLED}" DJANGO_ALLOW_ASYNC_UNSAFE: "True" restart: "on-failure"