feat: consolidate metrics stack into docker-compose.yaml using profiles

.gitignore

@@ -1,5 +1,8 @@
 # exclude .env files for docker setup
 .env
+docker-compose.override.yml
 /tmp/
 /data/
 /test/data/
+metrics/grafana/provisioning/dashboards/community/
+neops/metrics/grafana/provisioning/dashboards/community/

README.md

@@ -18,3 +18,7 @@ Change directory to your `neops-docker-compose` root dir
 ```shell
 docker run -it --rm -v $(pwd)/:/app/ndc quay.io/zebbra/neops-core:dc-custom-values
 ```
+
+## Metrics stack
+
+See [metrics/README.md](metrics/README.md).

docker-compose.yaml

@@ -0,0 +1,135 @@
+services:
+  redis:
+    image: redis:7
+    ports:
+      - "6380:6379"
+
+  postgres:
+    image: postgres:15-alpine
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_DB: postgres
+      POSTGRES_PASSWORD: unsafe
+    ports:
+      - "5433:5432"
+    volumes:
+      - ./data/dev_postgres:/var/lib/postgresql/data
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.9.2
+    environment:
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+      - cluster.routing.allocation.disk.threshold_enabled=false
+    ports:
+      - "9200:9200"
+
+  frontend:
+    build: /home/sgr/code/neops-legacy-frontend
+    environment:
+      FRONTEND_NEOPS_URL: http://localhost:8000/graphql
+    ports:
+      - "8080:8080"
+    restart: on-failure
+
+  # ── Observability stack ────────────────────────────────────────────────────
+  # Start with: docker compose --profile metrics up
+
+  celery-exporter:
+    profiles: [metrics]
+    image: danihodovic/celery-exporter:latest
+    command:
+      - "--broker-url=redis://redis:6379/0"
+    ports:
+      - "9808:9808"
+    depends_on:
+      - redis
+    restart: unless-stopped
+
+  redis-exporter:
+    profiles: [metrics]
+    image: oliver006/redis_exporter:latest
+    environment:
+      - REDIS_ADDR=redis://redis:6379
+    ports:
+      - "9121:9121"
+    depends_on:
+      - redis
+    restart: unless-stopped
+
+  postgres-exporter:
+    profiles: [metrics]
+    image: prometheuscommunity/postgres-exporter:latest
+    environment:
+      - DATA_SOURCE_NAME=postgresql://postgres:unsafe@postgres:5432/postgres?sslmode=disable
+    ports:
+      - "9187:9187"
+    depends_on:
+      - postgres
+    restart: unless-stopped
+
+  elasticsearch-exporter:
+    profiles: [metrics]
+    image: prometheuscommunity/elasticsearch-exporter:latest
+    command:
+      - "--es.uri=http://elasticsearch:9200"
+      - "--es.all"
+      - "--es.indices"
+    ports:
+      - "9114:9114"
+    depends_on:
+      - elasticsearch
+    restart: unless-stopped
+
+  vmalert:
+    profiles: [metrics]
+    image: victoriametrics/vmalert:latest
+    command:
+      - "--datasource.url=http://victoriametrics:8428"
+      - "--remoteWrite.url=http://victoriametrics:8428"
+      - "--remoteRead.url=http://victoriametrics:8428"
+      - "--rule=/etc/vmalert/rules/*.yml"
+      - "--evaluationInterval=1m"
+      - "--notifier.blackhole=true"
+    volumes:
+      - ./metrics/vmalert/rules:/etc/vmalert/rules:ro
+    ports:
+      - "8880:8880"
+    depends_on:
+      - victoriametrics
+    restart: unless-stopped
+
+  victoriametrics:
+    profiles: [metrics]
+    image: victoriametrics/victoria-metrics:latest
+    command:
+      - "--promscrape.config=/etc/victoriametrics/scrape_config.yml"
+      - "--retentionPeriod=30d"
+      - "--storageDataPath=/victoria-metrics-data"
+    volumes:
+      - ./metrics/scrape_config.yml:/etc/victoriametrics/scrape_config.yml:ro
+      - vm_data:/victoria-metrics-data
+    ports:
+      - "8428:8428"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    restart: unless-stopped
+
+  grafana:
+    profiles: [metrics]
+    image: grafana/grafana:11.5.2
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_USERS_ALLOW_SIGN_UP=false
+    volumes:
+      - ./metrics/grafana/provisioning:/etc/grafana/provisioning:ro
+      - grafana_data:/var/lib/grafana
+    ports:
+      - "3000:3000"
+    restart: unless-stopped
+
+volumes:
+  vm_data:
+    driver: local
+  grafana_data:
+    driver: local

metrics/README.md

@@ -0,0 +1,82 @@
+# Metrics stack
+
+Observability stack for neops: VictoriaMetrics (TSDB), Grafana (dashboards), vmalert (alerting), and exporters for Celery, Redis, PostgreSQL, and Elasticsearch.
+
+## Usage
+
+### Dev
+
+Included in `docker-compose.yaml` under the `metrics` profile:
+
+```shell
+# base services only
+docker compose up
+
+# base + full observability stack
+docker compose --profile metrics up
+```
+
+### Prod
+
+Add to `COMPOSE_FILE` in your `.env`:
+
+```
+COMPOSE_FILE=neops/docker-compose.neops.prod.yml:neops/docker-compose.neops.prod.metrics.yml
+```
+
+## Services
+
+| Service | URL | Purpose |
+|---|---|---|
+| Grafana | http://localhost:3000 | Dashboards (admin / admin) |
+| VictoriaMetrics | http://localhost:8428 | TSDB + query UI |
+| vmalert | http://localhost:8880/vmalert | Alert rules & state |
+| celery-exporter | http://localhost:9808/metrics | Raw Celery metrics |
+| redis-exporter | http://localhost:9121/metrics | Raw Redis metrics |
+| postgres-exporter | http://localhost:9187/metrics | Raw PostgreSQL metrics |
+| elasticsearch-exporter | http://localhost:9114/metrics | Raw Elasticsearch metrics |
+
+**Exporters:**
+- `celery-exporter` — Celery task/worker metrics ([danihodovic/celery-exporter](https://github.com/danihodovic/celery-exporter))
+- `redis-exporter` — Redis metrics ([oliver006/redis_exporter](https://github.com/oliver006/redis_exporter))
+- `postgres-exporter` — PostgreSQL metrics ([prometheuscommunity/postgres_exporter](https://github.com/prometheus-community/postgres_exporter))
+- `elasticsearch-exporter` — Elasticsearch cluster/index metrics ([prometheuscommunity/elasticsearch_exporter](https://github.com/prometheus-community/elasticsearch_exporter))
+- `victoriametrics` — Prometheus-compatible TSDB, scrapes all exporters + neops backend (`/metrics`); also self-scraped for storage/ingestion metrics
+- `vmalert` — Alert rule evaluation; rules live in `vmalert/rules/`; scraped for rule evaluation health and firing alert counts
+
+## Grafana dashboards
+
+Grafana is pre-provisioned with a VictoriaMetrics datasource and two sets of dashboards:
+
+- **`grafana/provisioning/dashboards/neops/`** — checked into the repo (neops overview, custom metrics, per-task metrics)
+- **`grafana/provisioning/dashboards/community/`** — gitignored, fetched by script
+
+Fetch or refresh community dashboards (VictoriaMetrics, vmalert, Celery, Redis, PostgreSQL, Elasticsearch, Django):
+
+```shell
+bash metrics/fetch-dashboards.sh
+```
+
+Re-run on a fresh clone before starting Grafana.
+
+## Scraping the local backend (dev)
+
+VictoriaMetrics uses `host.docker.internal` (wired via `extra_hosts: host-gateway`) to reach a backend running on the host at port 8000. Add to Django's env:
+
+```
+DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1,host.docker.internal
+```
+
+For prod, switch the neops scrape target in `scrape_config.yml` to `backend:8000`.
+
+## Pairs well with
+
+The **neops_metrics** enterprise plugin (`neops_modules/enterprise/neops_metrics`) exposes a `/metrics` endpoint on the neops backend with task execution counts, device/scope stats, facts, ES sync lag, Celery queue depth, and a Redis-backed custom metric API for use inside Jinja2 task templates.
+
+## Grafana 12 upgrade checklist
+
+- [ ] Delete `grafana_data` volume before upgrading (avoids mixed-state migration)
+- [ ] Re-run `fetch-dashboards.sh`
+- [ ] Migrate `grafana/provisioning/datasources/prometheus.yml` to the new Kubernetes-style provisioning format (classic file provisioning broken in Grafana 12)
+- [ ] Check for plugin auto-install failures on airgapped deployments
+- [ ] Verify `$datasource` template variable resolution still works in provisioned dashboards

metrics/fetch-dashboards.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+DASHBOARDS=(
+  "10229:victoriametrics-overview"
+  "14950:vmalert"
+  "17508:celery-exporter"
+  "17509:celery-overview"
+  "20076:celery-tasks"
+  "763:redis-exporter"
+  "9628:postgres"
+  "14191:elasticsearch"
+  "17613:django-overview"
+  "17616:django-requests"
+  "17617:django-database"
+  "24933:django-models"
+)
+
+DATASOURCE_UID="victoriametrics"
+OUT="$(dirname "$0")/grafana/provisioning/dashboards/community"
+mkdir -p "$OUT"
+
+for entry in "${DASHBOARDS[@]}"; do
+  id="${entry%%:*}"
+  name="${entry##*:}"
+  echo -n "Fetching $name ($id)... "
+  curl -sf "https://grafana.com/api/dashboards/${id}/revisions/latest/download" \
+    | sed "s/\${DS_PROM}/${DATASOURCE_UID}/g" \
+    | sed "s/\${DS_PROMETHEUS}/${DATASOURCE_UID}/g" \
+    > "${OUT}/${name}.json"
+  echo "ok"
+done
+
+echo "Done — dashboards saved to $OUT"

metrics/grafana/provisioning/dashboards/dashboards.yml

@@ -0,0 +1,13 @@
+apiVersion: 1
+providers:
+  - name: neops
+    folder: neops
+    type: file
+    options:
+      path: /etc/grafana/provisioning/dashboards/neops
+
+  - name: community
+    folder: Community
+    type: file
+    options:
+      path: /etc/grafana/provisioning/dashboards/community