diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 0000000..72e2bb2 --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,11 @@ +{ + "permissions": { + "allow": [ + "Bash(Remove-Item \"C:\\\\Users\\\\alaba\\\\OneDrive\\\\Desktop\\\\Rythmify\\\\devops\\\\monitoring\\\\grafana\\\\provisioning\\\\alerting\\\\alerts.yml\" -Force)", + "Bash(Get-ChildItem \"C:\\\\Users\\\\alaba\\\\OneDrive\\\\Desktop\\\\Rythmify\\\\devops\\\\monitoring\\\\grafana\\\\provisioning\\\\alerting\" -ErrorAction SilentlyContinue)", + "Bash(Measure-Object)", + "Bash(Select-Object -ExpandProperty Count)", + "PowerShell(Remove-Item *)" + ] + } +} diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml index ebe739e..29d0f72 100644 --- a/docker-compose.monitoring.yml +++ b/docker-compose.monitoring.yml @@ -30,6 +30,7 @@ services: - nginx-exporter - blackbox_exporter - postgres_exporter + - redis_exporter alertmanager: image: prom/alertmanager:v0.26.0 @@ -99,7 +100,7 @@ services: container_name: rythmify_nginx_exporter restart: unless-stopped command: - --nginx.scrape-uri=http://host.docker.internal:8081/nginx_status + - '--nginx.scrape-uri=http://host.docker.internal:8081/nginx_status' extra_hosts: - host.docker.internal:host-gateway ports: @@ -134,6 +135,19 @@ services: networks: - monitoring + redis_exporter: + image: oliver006/redis_exporter:latest + container_name: rythmify_redis_exporter + restart: unless-stopped + environment: + REDIS_ADDR: redis://host.docker.internal:6379 + extra_hosts: + - host.docker.internal:host-gateway + ports: + - "9121:9121" + networks: + - monitoring + volumes: prometheus_data: grafana_data: diff --git a/monitoring/README.md b/monitoring/README.md index 81efb3e..8b21184 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -12,26 +12,26 @@ Components: **Prometheus** · **Grafana** · **Alertmanager** · **Blackbox Expo ``` ┌─────────────────────────────────────────┐ - │ VM1 — Docker monitoring stack │ - │ │ + │ VM1 — Docker monitoring stack │ + │ │ ┌──────────────┐ │ ┌───────────┐ ┌──────────────────┐ │ │ Backend │◄─────┼──│ Prometheus│───►│ Grafana │ │ │ :8080/metrics│ │ │ :9090 │ │ :3000 │ │ └──────────────┘ │ └─────┬─────┘ │ /grafana/ proxy │ │ - │ │ └──────────────────┘ │ - ┌──────────────┐ │ │ ┌──────────────────┐ │ - │ Node Exporter│◄─────┼────────┤ │ Alertmanager │ │ - │ :9100 │ │ │ │ :9093 │ │ - └──────────────┘ │ │ │ Gmail SMTP │ │ - │ │ └──────────────────┘ │ - ┌──────────────┐ │ │ │ + │ │ └──────────────────┘ │ + ┌──────────────┐ │ │ ┌──────────────────┐ │ + │ Node Exporter│◄─────┼────────┤ │ Alertmanager │ │ + │ :9100 │ │ │ │ :9093 │ │ + └──────────────┘ │ │ │ Gmail SMTP │ │ + │ │ └──────────────────┘ │ + ┌──────────────┐ │ │ │ │ Nginx │◄─────┼────────┤ ┌──────────────────────────┐ │ - │ stub_status │ │ │ │ Blackbox Exporter :9115 │ │ - │ 172.17.0.1 │ │ ├──┤ probes public HTTPS URLs │ │ + │ stub_status │ │ │ │ Blackbox Exporter :9115 │ │ + │ 172.17.0.1 │ │ ├──┤ probes public HTTPS URLs │ │ │ :8081 │ │ │ └──────────────────────────┘ │ - └──────────────┘ │ │ │ + └──────────────┘ │ │ │ │ │ ┌──────────────────────────┐ │ - ┌──────────────┐ │ └──┤ Postgres Exporter :9187 │ │ + ┌──────────────┐ │ └──┤ Postgres Exporter :9187 │ │ │ PostgreSQL │◄─────┼───────────┤ connects to VM2 :5432 │ │ │ VM2:5432 │ │ └──────────────────────────┘ │ └──────────────┘ └─────────────────────────────────────────┘ @@ -223,3 +223,15 @@ Verify `GF_SMTP_ENABLED=true`, `SMTP_HOST=smtp.gmail.com:587`, `SMTP_USER`, and **`host.docker.internal` not resolving in Prometheus** Confirm `docker-compose.monitoring.yml` includes `extra_hosts: - "host.docker.internal:host-gateway"` on the Prometheus service. + +--- + +## Grafana Community Dashboards + +Import via Grafana UI → Dashboards → Import → Enter ID → Load → Save. + +| Dashboard | ID | What it shows | +|---|---|---| +| Node Exporter Full | 1860 | VM CPU, memory, disk, network (full detail) | +| NGINX | 12708 | Nginx connections, requests, error rates | +| PostgreSQL | 9628 | DB health (alternative to provisioned dashboard) | diff --git a/monitoring/alerts.md b/monitoring/alerts.md deleted file mode 100644 index e69de29..0000000 diff --git a/monitoring/blackbox/blackbox.yml b/monitoring/blackbox/blackbox.yml index 516c7cf..238a21d 100644 --- a/monitoring/blackbox/blackbox.yml +++ b/monitoring/blackbox/blackbox.yml @@ -1,5 +1,4 @@ modules: - # Used internally or for plain HTTP endpoints http_2xx: prober: http timeout: 5s @@ -11,7 +10,6 @@ modules: - HTTP/2.0 valid_status_codes: [] - # Used for all public HTTPS endpoints — fails if TLS absent or cert invalid https_2xx: prober: http timeout: 10s diff --git a/monitoring/grafana.md b/monitoring/grafana.md deleted file mode 100644 index 1ba40bf..0000000 --- a/monitoring/grafana.md +++ /dev/null @@ -1,27 +0,0 @@ -# Grafana Monitoring Plan - -## Purpose -Grafana will be used in later phases to monitor backend health, infrastructure metrics, and service availability. - -## Planned Monitoring Scope -- Backend uptime -- API response times -- Error rates -- Database health -- Resource usage (CPU, memory, disk) -- Storage-related metrics - -## Planned Data Sources -- Azure Monitor -- PostgreSQL metrics -- Application logs -- Container metrics - -## Planned Dashboards -- System Health Dashboard -- Backend API Dashboard -- Database Dashboard -- Storage & Usage Dashboard - -## Status -Planned for deployment phase. Current phase focuses on local setup and CI/CD preparation. diff --git a/monitoring/grafana/provisioning/alerting/alerts.yml b/monitoring/grafana/provisioning/alerting/alerts.yml deleted file mode 100644 index a383b7f..0000000 --- a/monitoring/grafana/provisioning/alerting/alerts.yml +++ /dev/null @@ -1,311 +0,0 @@ -apiVersion: 1 - -contactPoints: - - orgId: 1 - name: Rythmify Email - receivers: - - uid: rythmify-email - type: email - settings: - addresses: ${ALERT_EMAIL_TO} - singleEmail: true - -policies: - - orgId: 1 - receiver: Rythmify Email - group_by: - - grafana_folder - - alertname - group_wait: 30s - group_interval: 5m - repeat_interval: 4h - routes: - - receiver: Rythmify Email - matchers: - - severity = critical - group_wait: 10s - repeat_interval: 1h - -groups: - - orgId: 1 - name: Rythmify Monitoring Alerts - folder: Rythmify Monitoring - interval: 1m - rules: - # FIX: was `up{job="backend_metrics"} == 0` with `type: gt, params: [0]` - # That logic was inverted — fired when UP (no-data), silent when DOWN (0 > 0 = false). - # Fixed: query returns raw `up` value; condition fires when up < 1. - - uid: backend-down - title: Backend Down - condition: C - data: - - refId: A - datasourceUid: prometheus - relativeTimeRange: - from: 300 - to: 0 - model: - expr: up{job="backend_metrics"} - instant: true - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 1 - type: lt - operator: - type: and - query: - params: - - A - reducer: - type: last - type: query - refId: C - type: classic_conditions - noDataState: Alerting - execErrState: Alerting - for: 2m - annotations: - summary: Backend metrics endpoint is down - labels: - severity: critical - - - uid: high-cpu - title: High CPU - condition: C - data: - - refId: A - datasourceUid: prometheus - relativeTimeRange: - from: 300 - to: 0 - model: - expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85 - instant: true - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - A - reducer: - type: last - type: query - refId: C - type: classic_conditions - noDataState: NoData - execErrState: Error - for: 5m - annotations: - summary: VM CPU usage is above 85% - labels: - severity: warning - - - uid: high-memory - title: High Memory - condition: C - data: - - refId: A - datasourceUid: prometheus - relativeTimeRange: - from: 300 - to: 0 - model: - expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85 - instant: true - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - A - reducer: - type: last - type: query - refId: C - type: classic_conditions - noDataState: NoData - execErrState: Error - for: 5m - annotations: - summary: VM memory usage is above 85% - labels: - severity: warning - - - uid: disk-full - title: Disk Full - condition: C - data: - - refId: A - datasourceUid: prometheus - relativeTimeRange: - from: 300 - to: 0 - model: - expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"})) * 100 > 85 - instant: true - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - A - reducer: - type: last - type: query - refId: C - type: classic_conditions - noDataState: NoData - execErrState: Error - for: 5m - annotations: - summary: VM disk usage is above 85% - labels: - severity: warning - - - uid: high-error-rate - title: High Error Rate - condition: C - data: - - refId: A - datasourceUid: prometheus - relativeTimeRange: - from: 300 - to: 0 - model: - expr: sum(rate(http_errors_total[5m])) / clamp_min(sum(rate(http_requests_total[5m])), 1) > 0.05 - instant: true - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - A - reducer: - type: last - type: query - refId: C - type: classic_conditions - noDataState: NoData - execErrState: Error - for: 5m - annotations: - summary: Backend HTTP error rate is above 5% - labels: - severity: warning - - - uid: high-latency - title: High Latency - condition: C - data: - - refId: A - datasourceUid: prometheus - relativeTimeRange: - from: 300 - to: 0 - model: - expr: histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m]))) > 1 - instant: true - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - type: gt - operator: - type: and - query: - params: - - A - reducer: - type: last - type: query - refId: C - type: classic_conditions - noDataState: NoData - execErrState: Error - for: 5m - annotations: - summary: Backend p95 latency is above 1 second - labels: - severity: warning - - # FIX: same inverted-logic bug as backend-down. - # Fixed: query returns raw `probe_success`; condition fires when probe_success < 1. - - uid: public-endpoint-down - title: Public Endpoint Down - condition: C - data: - - refId: A - datasourceUid: prometheus - relativeTimeRange: - from: 300 - to: 0 - model: - expr: probe_success{job="blackbox"} - instant: true - refId: A - - refId: C - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 1 - type: lt - operator: - type: and - query: - params: - - A - reducer: - type: last - type: query - refId: C - type: classic_conditions - noDataState: Alerting - execErrState: Alerting - for: 2m - annotations: - summary: One or more public Rythmify endpoints are down - labels: - severity: critical diff --git a/monitoring/logs.md b/monitoring/logs.md deleted file mode 100644 index e69de29..0000000 diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index bae16df..d60a675 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -38,6 +38,11 @@ scrape_configs: - targets: - postgres_exporter:9187 + - job_name: redis + static_configs: + - targets: + - redis_exporter:9121 + - job_name: blackbox metrics_path: /probe params: diff --git a/nginx/rythmify-back.duckdns.org.conf b/nginx/rythmify-back.duckdns.org.conf index 89d872c..3c187ef 100644 --- a/nginx/rythmify-back.duckdns.org.conf +++ b/nginx/rythmify-back.duckdns.org.conf @@ -1,12 +1,7 @@ -# GeoIP2 — resolves client IP to ISO 3166-1 alpha-2 country code. -# Requires: libnginx-mod-http-geoip2 and /var/lib/GeoIP/GeoLite2-Country.mmdb -# $geoip2_data_country_code is injected as X-Country-Code on every proxied -# request. Backend reads it via req.headers['x-country-code'] (e.g. "EG", "SA"). geoip2 /var/lib/GeoIP/GeoLite2-Country.mmdb { $geoip2_data_country_code country iso_code; } -# ── HTTP → HTTPS redirect ────────────────────────────────────────────────────── server { listen 80; listen [::]:80; @@ -15,23 +10,16 @@ server { return 301 https://$host$request_uri; } -# ── HTTPS — backend API + WebSocket (Socket.IO) ──────────────────────────────── server { listen 443 ssl; listen [::]:443 ssl; server_name rythmify-back.duckdns.org; - # Certificate covers both rythmify.duckdns.org and rythmify-back.duckdns.org - # as Subject Alternative Names. Issued with: - # sudo certbot certonly --nginx \ - # -d rythmify.duckdns.org -d rythmify-back.duckdns.org ssl_certificate /etc/letsencrypt/live/rythmify.duckdns.org/fullchain.pem; ssl_certificate_key /etc/letsencrypt/live/rythmify.duckdns.org/privkey.pem; include /etc/letsencrypt/options-ssl-nginx.conf; ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - # ── /health — liveness probe used by Blackbox exporter ────────────────────── - # No WebSocket headers — this is a plain HTTP probe. location /health { proxy_pass http://localhost:8080; proxy_http_version 1.1; @@ -42,10 +30,6 @@ server { proxy_set_header X-Forwarded-Proto $scheme; } - # ── /api/ — explicit REST API path ────────────────────────────────────────── - # Mirrors the / block so REST calls to /api/v1/... are handled explicitly. - # WebSocket upgrade headers are kept so Socket.IO clients using a custom - # path (e.g. /api/socket.io) are not broken. location /api/ { proxy_pass http://localhost:8080; proxy_http_version 1.1; @@ -63,7 +47,6 @@ server { proxy_cache_bypass $http_upgrade; } - # ── /grafana/ — Grafana dashboard (proxied from port 3000) ────────────────── location /grafana/ { proxy_pass http://localhost:3000/; proxy_http_version 1.1; @@ -74,12 +57,10 @@ server { proxy_set_header X-Forwarded-Proto $scheme; } - # ── / — all API and WebSocket traffic ─────────────────────────────────────── location / { proxy_pass http://localhost:8080; proxy_http_version 1.1; - # WebSocket upgrade — required for Socket.IO long-polling and ws transports proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; @@ -88,22 +69,12 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - # Country resolved from client IP via MaxMind GeoLite2-Country. - # Empty string when IP is unresolvable (private ranges, localhost). proxy_set_header X-Country-Code $geoip2_data_country_code; proxy_cache_bypass $http_upgrade; } } -# ── Nginx stub_status — for prometheus nginx_exporter ───────────────────────── -# -# Binds to the Docker bridge host IP (172.17.0.1) on a dedicated port 8081. -# nginx_exporter in docker-compose.monitoring.yml scrapes: -# http://172.17.0.1:8081/nginx_status -# Binding to 172.17.0.1 means this address is unreachable from the public -# internet — only Docker containers on the bridge network can reach it, so -# allow all is safe. server { listen 172.17.0.1:8081; server_name _; diff --git a/nginx/rythmify.duckdns.org.conf b/nginx/rythmify.duckdns.org.conf index acd9db4..be135fa 100644 --- a/nginx/rythmify.duckdns.org.conf +++ b/nginx/rythmify.duckdns.org.conf @@ -1,4 +1,3 @@ -# ── HTTP → HTTPS redirect ────────────────────────────────────────────────────── server { listen 80; listen [::]:80; @@ -7,12 +6,6 @@ server { return 301 https://$host$request_uri; } -# ── HTTPS — frontend static files (React SPA) ───────────────────────────────── -# -# Serves pre-built assets from /var/www/rythmify (deployed by the Frontend CD -# pipeline via rsync). All unknown routes fall back to index.html so -# client-side routing works. API calls go directly to -# rythmify-back.duckdns.org — no proxy on this domain. server { listen 443 ssl; listen [::]:443 ssl;