feat(monitoring): add Prometheus, Grafana, and exporters configuratio…

.github/workflows/devops.yml

@@ -223,11 +223,6 @@ jobs:
       - name: Run migrations
         run: docker exec rythmify_backend sh -c "npm run migrate:up"
 
-      - name: Run seeds
-        run: |
-          docker cp $GITHUB_WORKSPACE/backend/seeds/seed.sql rythmify_db:/seed.sql
-          docker exec rythmify_db psql -U dev -d rythmify_dev -f /seed.sql
-
       - name: Run backend smoke check
         run: |
           echo "Running backend smoke check..."

.gitignore

@@ -4,6 +4,7 @@
 *.env
 *.env.*
 !.env.example
+!monitoring/.env.monitoring.example
 environments/dev.env
 environments/staging.env
 environments/production.env
@@ -80,4 +81,4 @@ coverage/
 .terraform/
 *.tfstate
 *.tfstate.*
-crash.log
+crash.log

docker-compose.monitoring.yml

@@ -0,0 +1,80 @@
+services:
+  prometheus:
+    image: prom/prometheus:v2.55.1
+    container_name: rythmify_prometheus
+    restart: unless-stopped
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --web.enable-lifecycle
+    extra_hosts:
+      - host.docker.internal:host-gateway
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus_data:/prometheus
+    depends_on:
+      - node_exporter
+      - nginx_exporter
+      - blackbox_exporter
+
+  grafana:
+    image: grafana/grafana:11.4.0
+    container_name: rythmify_grafana
+    restart: unless-stopped
+    ports:
+      - "3000:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
+      GF_SMTP_ENABLED: ${GF_SMTP_ENABLED}
+      GF_SMTP_HOST: ${SMTP_HOST}
+      GF_SMTP_USER: ${SMTP_USER}
+      GF_SMTP_PASSWORD: ${SMTP_PASSWORD}
+      GF_SMTP_FROM_ADDRESS: ${GF_SMTP_FROM_ADDRESS}
+      GF_SMTP_FROM_NAME: ${GF_SMTP_FROM_NAME}
+      ALERT_EMAIL_TO: ${ALERT_EMAIL_TO}
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
+    depends_on:
+      - prometheus
+
+  node_exporter:
+    image: prom/node-exporter:v1.8.2
+    container_name: rythmify_node_exporter
+    restart: unless-stopped
+    command:
+      - --path.rootfs=/host
+    pid: host
+    ports:
+      - "9100:9100"
+    volumes:
+      - /:/host:ro,rslave
+
+  nginx_exporter:
+    image: nginx/nginx-prometheus-exporter:1.3.0
+    container_name: rythmify_nginx_exporter
+    restart: unless-stopped
+    command:
+      - --nginx.scrape-uri=http://host.docker.internal/nginx_status
+    extra_hosts:
+      - host.docker.internal:host-gateway
+    ports:
+      - "9113:9113"
+
+  blackbox_exporter:
+    image: prom/blackbox-exporter:v0.25.0
+    container_name: rythmify_blackbox_exporter
+    restart: unless-stopped
+    command:
+      - --config.file=/etc/blackbox_exporter/blackbox.yml
+    ports:
+      - "9115:9115"
+    volumes:
+      - ./monitoring/blackbox/blackbox.yml:/etc/blackbox_exporter/blackbox.yml:ro
+
+volumes:
+  prometheus_data:
+  grafana_data:

monitoring/README.md

@@ -0,0 +1,122 @@
+# Rythmify Monitoring
+
+This stack runs Prometheus, Grafana, node_exporter, nginx_exporter, and blackbox_exporter on the Dir 2 VM.
+
+## Architecture
+
+- Prometheus scrapes every 15 seconds.
+- Backend app metrics are scraped from `host.docker.internal:8080/metrics`.
+- VM CPU, memory, and disk metrics come from `node_exporter`.
+- Nginx metrics come from `nginx_exporter`, which reads `http://host.docker.internal/nginx_status`.
+- Public uptime checks are run through `blackbox_exporter` for:
+  - `http://rythmify.duckdns.org/`
+  - `http://rythmify-back.duckdns.org/`
+  - `http://rythmify-back.duckdns.org/metrics`
+- Grafana provisions Prometheus as the default data source and email alerts for core health signals.
+
+## VM Access
+
+```powershell
+ssh -i "C:\Users\alaba\OneDrive\Desktop\rythmify-app-vm_key.pem" rythmify@20.196.3.253
+```
+
+## Nginx stub_status
+
+Add this location to the Nginx server block that listens on localhost:
+
+```nginx
+location /nginx_status {
+    stub_status;
+    allow 127.0.0.1;
+    allow 172.16.0.0/12;
+    allow 172.17.0.0/16;
+    deny all;
+}
+```
+
+Validate and reload Nginx:
+
+```bash
+sudo nginx -t
+sudo systemctl reload nginx
+curl http://localhost/nginx_status
+```
+
+## Environment
+
+Create a runtime environment file from the example:
+
+```bash
+cp monitoring/.env.monitoring.example .env.monitoring
+```
+
+Edit `.env.monitoring` and replace `SMTP_PASSWORD` with a Gmail app password. Do not commit `.env.monitoring`.
+
+## Docker Commands
+
+Start the stack:
+
+```bash
+docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml up -d
+```
+
+Check containers:
+
+```bash
+docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml ps
+```
+
+View logs:
+
+```bash
+docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml logs -f prometheus grafana
+```
+
+Restart after config changes:
+
+```bash
+docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml restart prometheus grafana
+```
+
+Stop the stack:
+
+```bash
+docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml down
+```
+
+## Verification
+
+```bash
+curl http://localhost:9090/-/ready
+curl http://localhost:9115/-/healthy
+curl http://localhost/nginx_status
+curl http://localhost:9090/api/v1/targets
+```
+
+Open Grafana:
+
+```text
+http://20.196.3.253:3000
+```
+
+Default credentials are configured through `.env.monitoring`.
+
+## Alerts
+
+Grafana provisions alerts for:
+
+- backend down
+- high CPU
+- high memory
+- disk full
+- high backend error rate
+- high backend latency
+- public endpoint down
+
+## Troubleshooting
+
+- If `backend_metrics` is down, verify the backend is listening on VM port `8080` and `curl http://localhost:8080/metrics` works on the VM.
+- If `nginx_exporter` is down, verify `curl http://localhost/nginx_status` works on the VM and that the Nginx allow list includes Docker bridge ranges.
+- If blackbox targets are down, run `curl -I` against each public URL from the VM.
+- If Grafana email alerts do not send, verify `GF_SMTP_ENABLED=true`, `SMTP_HOST`, `SMTP_USER`, and the app password in `.env.monitoring`.
+- If Prometheus cannot resolve `host.docker.internal`, confirm the compose services include `extra_hosts: host.docker.internal:host-gateway` and the host has a recent Docker Engine.

monitoring/blackbox/blackbox.yml

@@ -0,0 +1,11 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      method: GET
+      preferred_ip_protocol: ip4
+      valid_http_versions:
+        - HTTP/1.1
+        - HTTP/2.0
+      valid_status_codes: []