From c643c52355eac13ab70edc5125cb2e48cf1d1aeb Mon Sep 17 00:00:00 2001 From: Mohamed jameel alabasy Date: Fri, 1 May 2026 17:45:52 +0300 Subject: [PATCH] feat(monitoring): enhance blackbox and alerting configurations --- docker-compose.monitoring.yml | 80 ++++++-- monitoring/.env.monitoring.example | 32 +++ monitoring/alertmanager/alertmanager.yml.tmpl | 36 ++++ monitoring/blackbox/blackbox.yml | 16 ++ .../grafana/provisioning/alerting/alerts.yml | 26 ++- .../provisioning/dashboards/backend_api.json | 135 ++++++++++++ .../provisioning/dashboards/dashboards.yml | 12 ++ .../dashboards/postgresql_health.json | 116 +++++++++++ .../provisioning/dashboards/uptime.json | 92 +++++++++ monitoring/prometheus/alert_rules.yml | 75 +++++++ monitoring/prometheus/prometheus.yml | 26 ++- nginx/README.md | 192 ++++++++++++++++++ nginx/rythmify-back.duckdns.org.conf | 64 ++++++ nginx/rythmify.duckdns.org.conf | 76 +++++++ 14 files changed, 954 insertions(+), 24 deletions(-) create mode 100644 monitoring/.env.monitoring.example create mode 100644 monitoring/alertmanager/alertmanager.yml.tmpl create mode 100644 monitoring/grafana/provisioning/dashboards/backend_api.json create mode 100644 monitoring/grafana/provisioning/dashboards/dashboards.yml create mode 100644 monitoring/grafana/provisioning/dashboards/postgresql_health.json create mode 100644 monitoring/grafana/provisioning/dashboards/uptime.json create mode 100644 monitoring/prometheus/alert_rules.yml create mode 100644 nginx/README.md create mode 100644 nginx/rythmify-back.duckdns.org.conf create mode 100644 nginx/rythmify.duckdns.org.conf diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml index 2aca50e..e8620d7 100644 --- a/docker-compose.monitoring.yml +++ b/docker-compose.monitoring.yml @@ -1,3 +1,9 @@ +# Run with: +# docker compose --env-file ./environments/.env.monitoring -f docker-compose.monitoring.yml up -d + +networks: + monitoring: + services: prometheus: image: prom/prometheus:v2.55.1 @@ -6,6 +12,7 @@ services: command: - --config.file=/etc/prometheus/prometheus.yml - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=30d - --web.enable-lifecycle extra_hosts: - host.docker.internal:host-gateway @@ -13,11 +20,36 @@ services: - "9090:9090" volumes: - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./monitoring/prometheus/alert_rules.yml:/etc/prometheus/alert_rules.yml:ro - prometheus_data:/prometheus + networks: + - monitoring depends_on: + - alertmanager - node_exporter - - nginx_exporter + - nginx-exporter - blackbox_exporter + - postgres_exporter + + alertmanager: + image: prom/alertmanager:v0.27.0 + container_name: rythmify_alertmanager + restart: unless-stopped + entrypoint: ["/bin/sh", "-c"] + command: + - | + apk add --no-cache gettext && + envsubst < /etc/alertmanager/alertmanager.yml.tmpl > /tmp/alertmanager.yml && + /bin/alertmanager --config.file=/tmp/alertmanager.yml --storage.path=/alertmanager + environment: + ALERTMANAGER_SMTP_PASSWORD: ${ALERTMANAGER_SMTP_PASSWORD} + volumes: + - alertmanager_data:/alertmanager + - ./monitoring/alertmanager/alertmanager.yml.tmpl:/etc/alertmanager/alertmanager.yml.tmpl + ports: + - "9093:9093" + networks: + - monitoring grafana: image: grafana/grafana:11.4.0 @@ -26,18 +58,21 @@ services: ports: - "3000:3000" environment: - GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER} + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} - GF_SMTP_ENABLED: ${GF_SMTP_ENABLED} - GF_SMTP_HOST: ${SMTP_HOST} - GF_SMTP_USER: ${SMTP_USER} - GF_SMTP_PASSWORD: ${SMTP_PASSWORD} - GF_SMTP_FROM_ADDRESS: ${GF_SMTP_FROM_ADDRESS} - GF_SMTP_FROM_NAME: ${GF_SMTP_FROM_NAME} - ALERT_EMAIL_TO: ${ALERT_EMAIL_TO} + GF_SMTP_ENABLED: ${GF_SMTP_ENABLED:-false} + GF_SMTP_HOST: ${SMTP_HOST:-smtp.gmail.com:587} + GF_SMTP_USER: ${SMTP_USER:-} + GF_SMTP_PASSWORD: ${SMTP_PASSWORD:-} + GF_SMTP_FROM_ADDRESS: ${GF_SMTP_FROM_ADDRESS:-} + GF_SMTP_FROM_NAME: ${GF_SMTP_FROM_NAME:-Rythmify Monitoring} + GF_SMTP_STARTTLS_POLICY: MandatoryStartTLS + ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-} volumes: - grafana_data:/var/lib/grafana - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + networks: + - monitoring depends_on: - prometheus @@ -52,17 +87,21 @@ services: - "9100:9100" volumes: - /:/host:ro,rslave + networks: + - monitoring - nginx_exporter: - image: nginx/nginx-prometheus-exporter:1.3.0 + nginx-exporter: + image: nginx/nginx-prometheus-exporter:latest container_name: rythmify_nginx_exporter restart: unless-stopped command: - - --nginx.scrape-uri=http://host.docker.internal/nginx_status + - '--nginx.scrape-uri=http://host.docker.internal/nginx_status' extra_hosts: - host.docker.internal:host-gateway ports: - "9113:9113" + networks: + - monitoring blackbox_exporter: image: prom/blackbox-exporter:v0.25.0 @@ -74,7 +113,24 @@ services: - "9115:9115" volumes: - ./monitoring/blackbox/blackbox.yml:/etc/blackbox_exporter/blackbox.yml:ro + networks: + - monitoring + + # Reaches PostgreSQL on VM2 private IP 10.0.1.4. + # Pre-requisite on VM2: + # GRANT pg_monitor TO rythmify_admin; + postgres_exporter: + image: quay.io/prometheuscommunity/postgres-exporter:v0.15.0 + container_name: rythmify_postgres_exporter + restart: unless-stopped + environment: + DATA_SOURCE_NAME: "postgresql://rythmify_admin:${POSTGRES_PASSWORD}@10.0.1.4:5432/rythmify_db?sslmode=disable" + ports: + - "9187:9187" + networks: + - monitoring volumes: prometheus_data: grafana_data: + alertmanager_data: diff --git a/monitoring/.env.monitoring.example b/monitoring/.env.monitoring.example new file mode 100644 index 0000000..f2ff1a2 --- /dev/null +++ b/monitoring/.env.monitoring.example @@ -0,0 +1,32 @@ +# Copy this file to devops/environments/.env.monitoring and fill in all values. +# Start the stack with: +# docker compose --env-file ./environments/.env.monitoring -f docker-compose.monitoring.yml up -d + +# ── Grafana ──────────────────────────────────────────────────────────────────── +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=your_grafana_password_here + +# ── SMTP (Gmail) ─────────────────────────────────────────────────────────────── +# Generate an App Password at https://myaccount.google.com/apppasswords +# Use port 587 (STARTTLS). Port 465 (SSL) is not used here. +GF_SMTP_ENABLED=true +SMTP_HOST=smtp.gmail.com:587 +SMTP_USER=your-sender@gmail.com +SMTP_PASSWORD=your-16-char-app-password +GF_SMTP_FROM_ADDRESS=your-sender@gmail.com +GF_SMTP_FROM_NAME=Rythmify Monitoring + +# ── Alert recipient ──────────────────────────────────────────────────────────── +ALERT_EMAIL_TO=oncall@example.com + +# ── Alertmanager SMTP ───────────────────────────────────────────────────────── +# Generate a Gmail App Password at: https://myaccount.google.com/apppasswords +# WARNING: do NOT use special characters in this value — envsubst will break +# if the password contains /, &, @, or $. Gmail App Passwords (16 lowercase +# letters, no punctuation) are safe by default. +ALERTMANAGER_SMTP_PASSWORD=your_gmail_app_password_here + +# ── PostgreSQL exporter ──────────────────────────────────────────────────────── +# rythmify_admin must have the pg_monitor role on VM2: +# GRANT pg_monitor TO rythmify_admin; +POSTGRES_PASSWORD=your_postgres_password_here diff --git a/monitoring/alertmanager/alertmanager.yml.tmpl b/monitoring/alertmanager/alertmanager.yml.tmpl new file mode 100644 index 0000000..7bbeaa2 --- /dev/null +++ b/monitoring/alertmanager/alertmanager.yml.tmpl @@ -0,0 +1,36 @@ +# Alertmanager config template — ${ALERTMANAGER_SMTP_PASSWORD} is substituted at +# container start via envsubst in docker-compose.monitoring.yml. The raw template +# is never executed by Alertmanager directly. + +global: + smtp_smarthost: 'smtp.gmail.com:587' + smtp_from: 'rythmify.noreply@gmail.com' + smtp_auth_username: 'rythmify.noreply@gmail.com' + smtp_auth_password: '${ALERTMANAGER_SMTP_PASSWORD}' + smtp_require_tls: true + +route: + receiver: email + group_by: [alertname, severity] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + routes: + - matchers: + - severity = critical + receiver: email + group_wait: 10s + repeat_interval: 1h + +receivers: + - name: email + email_configs: + - to: 'alabasy@live.com' + send_resolved: true + +inhibit_rules: + - source_matchers: + - severity = critical + target_matchers: + - severity = warning + equal: [alertname] diff --git a/monitoring/blackbox/blackbox.yml b/monitoring/blackbox/blackbox.yml index 383a16f..516c7cf 100644 --- a/monitoring/blackbox/blackbox.yml +++ b/monitoring/blackbox/blackbox.yml @@ -1,4 +1,5 @@ modules: + # Used internally or for plain HTTP endpoints http_2xx: prober: http timeout: 5s @@ -9,3 +10,18 @@ modules: - HTTP/1.1 - HTTP/2.0 valid_status_codes: [] + + # Used for all public HTTPS endpoints — fails if TLS absent or cert invalid + https_2xx: + prober: http + timeout: 10s + http: + method: GET + preferred_ip_protocol: ip4 + valid_http_versions: + - HTTP/1.1 + - HTTP/2.0 + valid_status_codes: [] + tls_config: + insecure_skip_verify: false + fail_if_not_ssl: true diff --git a/monitoring/grafana/provisioning/alerting/alerts.yml b/monitoring/grafana/provisioning/alerting/alerts.yml index 7135484..a383b7f 100644 --- a/monitoring/grafana/provisioning/alerting/alerts.yml +++ b/monitoring/grafana/provisioning/alerting/alerts.yml @@ -16,6 +16,15 @@ policies: group_by: - grafana_folder - alertname + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + routes: + - receiver: Rythmify Email + matchers: + - severity = critical + group_wait: 10s + repeat_interval: 1h groups: - orgId: 1 @@ -23,6 +32,9 @@ groups: folder: Rythmify Monitoring interval: 1m rules: + # FIX: was `up{job="backend_metrics"} == 0` with `type: gt, params: [0]` + # That logic was inverted — fired when UP (no-data), silent when DOWN (0 > 0 = false). + # Fixed: query returns raw `up` value; condition fires when up < 1. - uid: backend-down title: Backend Down condition: C @@ -33,7 +45,7 @@ groups: from: 300 to: 0 model: - expr: up{job="backend_metrics"} == 0 + expr: up{job="backend_metrics"} instant: true refId: A - refId: C @@ -42,8 +54,8 @@ groups: conditions: - evaluator: params: - - 0 - type: gt + - 1 + type: lt operator: type: and query: @@ -257,6 +269,8 @@ groups: labels: severity: warning + # FIX: same inverted-logic bug as backend-down. + # Fixed: query returns raw `probe_success`; condition fires when probe_success < 1. - uid: public-endpoint-down title: Public Endpoint Down condition: C @@ -267,7 +281,7 @@ groups: from: 300 to: 0 model: - expr: probe_success{job="blackbox"} == 0 + expr: probe_success{job="blackbox"} instant: true refId: A - refId: C @@ -276,8 +290,8 @@ groups: conditions: - evaluator: params: - - 0 - type: gt + - 1 + type: lt operator: type: and query: diff --git a/monitoring/grafana/provisioning/dashboards/backend_api.json b/monitoring/grafana/provisioning/dashboards/backend_api.json new file mode 100644 index 0000000..27a2358 --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/backend_api.json @@ -0,0 +1,135 @@ +{ + "annotations": { "list": [] }, + "description": "Rythmify backend API — request rate, latency, errors, Node.js process metrics", + "editable": true, + "graphTooltip": 1, + "id": null, + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 1, "fillOpacity": 8 }, "unit": "reqps" }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "id": 1, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(http_requests_total[1m]))", "legendFormat": "Requests/s", "refId": "A" } + ], + "title": "Request Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 1, "fillOpacity": 8 }, "unit": "percentunit", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 0.05 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "id": 2, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(http_errors_total[5m])) / clamp_min(sum(rate(http_requests_total[5m])), 1)", "legendFormat": "Error Rate", "refId": "A" } + ], + "title": "HTTP Error Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 1, "fillOpacity": 8 }, "unit": "s" }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 }, + "id": 3, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C" } + ], + "title": "Request Latency (p50 / p95 / p99)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 10 }, { "color": "red", "value": 50 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 16 }, + "id": 4, + "options": { "colorMode": "background", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "auto" }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(http_active_requests)", "legendFormat": "Active Requests", "refId": "A" } + ], + "title": "Active Requests", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 1, "fillOpacity": 8 }, "unit": "bytes" }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, + "id": 5, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nodejs_heap_size_used_bytes{job=\"backend_metrics\"}", "legendFormat": "Heap Used", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nodejs_heap_size_total_bytes{job=\"backend_metrics\"}", "legendFormat": "Heap Total", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "process_resident_memory_bytes{job=\"backend_metrics\"}", "legendFormat": "RSS", "refId": "C" } + ], + "title": "Node.js Memory", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 1, "fillOpacity": 8 }, "unit": "percent" }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, + "id": 6, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "rate(process_cpu_seconds_total{job=\"backend_metrics\"}[1m]) * 100", "legendFormat": "CPU %", "refId": "A" } + ], + "title": "Node.js CPU Usage", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 1, "fillOpacity": 8 }, "unit": "ms" }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 28 }, + "id": 7, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nodejs_eventloop_lag_p50_seconds{job=\"backend_metrics\"} * 1000", "legendFormat": "p50", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nodejs_eventloop_lag_p99_seconds{job=\"backend_metrics\"} * 1000", "legendFormat": "p99", "refId": "B" } + ], + "title": "Event Loop Lag", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": ["rythmify", "backend", "api"], + "templating": { "list": [] }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Rythmify — Backend API", + "uid": "rythmify-backend-api", + "version": 1 +} diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..bced6a3 --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: Rythmify + orgId: 1 + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: false + options: + path: /etc/grafana/provisioning/dashboards + foldersFromFilesStructure: false diff --git a/monitoring/grafana/provisioning/dashboards/postgresql_health.json b/monitoring/grafana/provisioning/dashboards/postgresql_health.json new file mode 100644 index 0000000..077737b --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/postgresql_health.json @@ -0,0 +1,116 @@ +{ + "annotations": { "list": [] }, + "description": "Rythmify PostgreSQL health — connections, transactions, cache hit ratio, dead tuples, database size", + "editable": true, + "graphTooltip": 1, + "id": null, + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, "unit": "short", + "mappings": [{ "options": { "0": { "color": "red", "text": "DOWN" }, "1": { "color": "green", "text": "UP" } }, "type": "value" }], + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, + "id": 1, + "options": { "colorMode": "background", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "auto" }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "pg_up", "legendFormat": "Status", "refId": "A" } + ], + "title": "PostgreSQL Status", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 1, "fillOpacity": 8 }, "unit": "short" }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "id": 2, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (state) (pg_stat_activity_count)", "legendFormat": "{{ state }}", "refId": "A" } + ], + "title": "Connections by State", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 1, "fillOpacity": 8 }, "unit": "short" }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, + "id": 3, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(pg_stat_database_xact_commit[5m]) + rate(pg_stat_database_xact_rollback[5m]))", "legendFormat": "Transactions/s", "refId": "A" } + ], + "title": "Transaction Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 1, "fillOpacity": 8 }, "unit": "percentunit", + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.8 }, { "color": "green", "value": 0.95 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "id": 4, + "options": { "legend": { "calcs": ["mean"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(pg_stat_database_blks_hit) / clamp_min(sum(pg_stat_database_blks_hit) + sum(pg_stat_database_blks_read), 1)", "legendFormat": "Cache Hit Ratio", "refId": "A" } + ], + "title": "Cache Hit Ratio", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 1, "fillOpacity": 8 }, "unit": "short" }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "id": 5, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(pg_stat_user_tables_n_dead_tup)", "legendFormat": "Dead Tuples", "refId": "A" } + ], + "title": "Dead Tuples", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 1, "fillOpacity": 8 }, "unit": "bytes" }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 20 }, + "id": 6, + "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "pg_database_size_bytes", "legendFormat": "{{ datname }}", "refId": "A" } + ], + "title": "Database Size", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": ["rythmify", "postgresql"], + "templating": { "list": [] }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Rythmify — PostgreSQL Health", + "uid": "rythmify-postgresql", + "version": 1 +} diff --git a/monitoring/grafana/provisioning/dashboards/uptime.json b/monitoring/grafana/provisioning/dashboards/uptime.json new file mode 100644 index 0000000..bad8f0c --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/uptime.json @@ -0,0 +1,92 @@ +{ + "annotations": { "list": [] }, + "description": "Rythmify external uptime — endpoint availability, response time, SSL certificate expiry", + "editable": true, + "graphTooltip": 1, + "id": null, + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, "unit": "short", + "mappings": [{ "options": { "0": { "color": "red", "text": "DOWN" }, "1": { "color": "green", "text": "UP" } }, "type": "value" }], + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 12, "x": 0, "y": 0 }, + "id": 1, + "options": { "colorMode": "background", "orientation": "horizontal", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "auto" }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_success{job=\"blackbox\"}", "legendFormat": "{{ instance }}", "refId": "A" } + ], + "title": "Endpoint Status", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, "unit": "d", + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 14 }, { "color": "green", "value": 30 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 12, "x": 12, "y": 0 }, + "id": 2, + "options": { "colorMode": "background", "reduceOptions": { "calcs": ["lastNotNull"] }, "textMode": "auto" }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "(probe_ssl_earliest_cert_expiry{job=\"blackbox\"} - time()) / 86400", "legendFormat": "{{ instance }}", "refId": "A" } + ], + "title": "SSL Certificate Expiry (days remaining)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 1, "fillOpacity": 8 }, "unit": "s", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 3 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 4 }, + "id": 3, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_duration_seconds{job=\"blackbox\"}", "legendFormat": "{{ instance }}", "refId": "A" } + ], + "title": "Probe Response Time", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 1, "fillOpacity": 8 }, + "unit": "percentunit", "min": 0, "max": 1 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 12 }, + "id": 4, + "options": { "legend": { "calcs": ["mean"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "avg_over_time(probe_success{job=\"blackbox\"}[1h])", "legendFormat": "{{ instance }} (1h avg)", "refId": "A" } + ], + "title": "Uptime — 1-Hour Rolling Average", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": ["rythmify", "uptime", "blackbox"], + "templating": { "list": [] }, + "time": { "from": "now-24h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Rythmify — Uptime & Availability", + "uid": "rythmify-uptime", + "version": 1 +} diff --git a/monitoring/prometheus/alert_rules.yml b/monitoring/prometheus/alert_rules.yml new file mode 100644 index 0000000..7caa3d6 --- /dev/null +++ b/monitoring/prometheus/alert_rules.yml @@ -0,0 +1,75 @@ +groups: + - name: rythmify + interval: 30s + rules: + - alert: BackendDown + expr: up{job="backend_metrics"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Rythmify backend is down" + description: "Backend /metrics endpoint has been unreachable for more than 2 minutes." + + - alert: PostgresDown + expr: pg_up == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "PostgreSQL is down" + description: "postgres_exporter cannot connect to PostgreSQL at 10.0.1.4." + + - alert: PublicEndpointDown + expr: probe_success{job="blackbox"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Public endpoint {{ $labels.instance }} is unreachable" + description: "Blackbox HTTPS probe for {{ $labels.instance }} has been failing for more than 2 minutes." + + - alert: HighErrorRate + expr: | + sum(rate(http_errors_total[5m])) + / clamp_min(sum(rate(http_requests_total[5m])), 1) + > 0.05 + for: 5m + labels: + severity: warning + annotations: + summary: "High HTTP error rate ({{ $value | humanizePercentage }})" + description: "HTTP 4xx/5xx error rate has exceeded 5% for 5 consecutive minutes." + + - alert: HighMemoryUsage + expr: | + (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85 + for: 5m + labels: + severity: warning + annotations: + summary: "High memory usage on {{ $labels.instance }}" + description: "Memory usage is {{ $value | humanize }}% on {{ $labels.instance }}." + + - alert: HighCPUUsage + expr: | + 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85 + for: 5m + labels: + severity: warning + annotations: + summary: "High CPU usage on {{ $labels.instance }}" + description: "CPU usage is {{ $value | humanize }}% on {{ $labels.instance }}." + + - alert: DiskSpaceLow + expr: | + (1 - ( + node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} + / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"} + )) * 100 > 85 + for: 5m + labels: + severity: warning + annotations: + summary: "Low disk space on {{ $labels.instance }}" + description: "Disk {{ $labels.mountpoint }} is {{ $value | humanize }}% full on {{ $labels.instance }}." diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index 7858acc..bae16df 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -1,5 +1,15 @@ global: scrape_interval: 15s + evaluation_interval: 15s + +alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + +rule_files: + - /etc/prometheus/alert_rules.yml scrape_configs: - job_name: prometheus @@ -12,10 +22,10 @@ scrape_configs: - targets: - node_exporter:9100 - - job_name: nginx_exporter + - job_name: nginx static_configs: - targets: - - nginx_exporter:9113 + - nginx-exporter:9113 - job_name: backend_metrics metrics_path: /metrics @@ -23,16 +33,20 @@ scrape_configs: - targets: - host.docker.internal:8080 + - job_name: postgres_exporter + static_configs: + - targets: + - postgres_exporter:9187 + - job_name: blackbox metrics_path: /probe params: module: - - http_2xx + - https_2xx static_configs: - targets: - - http://rythmify.duckdns.org/ - - http://rythmify-back.duckdns.org/ - - http://rythmify-back.duckdns.org/metrics + - https://rythmify-back.duckdns.org/health + - https://rythmify.duckdns.org relabel_configs: - source_labels: [__address__] target_label: __param_target diff --git a/nginx/README.md b/nginx/README.md new file mode 100644 index 0000000..2b2554b --- /dev/null +++ b/nginx/README.md @@ -0,0 +1,192 @@ +# Nginx Configuration — Rythmify VM1 + +## Overview + +This folder contains the Nginx virtual host configuration files for **VM1** (`rythmify-app-vm`, IP `20.196.3.253`). These files are not loaded automatically — they must be deployed to the server manually (see [Deployment](#deployment)). + +| File | Domain | Purpose | +|---|---|---| +| `rythmify-back.duckdns.org.conf` | `rythmify-back.duckdns.org` | Backend API + Socket.IO proxy, GeoIP2 country injection, HTTPS | +| `rythmify.duckdns.org.conf` | `rythmify.duckdns.org` | Frontend Vite dev server proxy, HTTPS, Nginx stub_status for Prometheus | + +The backend config also defines a `geoip2` directive (above its `server {}` block, in the `http {}` context) that resolves every client IP to a country code and injects it as `X-Country-Code` on all proxied requests. + +--- + +## Prerequisites + +Install the following packages on VM1: + +```bash +# Nginx + GeoIP2 dynamic module +sudo apt update +sudo apt install -y nginx libnginx-mod-http-geoip2 + +# MaxMind GeoIP database updater +sudo apt install -y geoipupdate + +# Certbot for SSL certificate issuance and renewal +sudo apt install -y certbot python3-certbot-nginx +``` + +`libnginx-mod-http-geoip2` drops a module config file into `/etc/nginx/modules-enabled/` that loads the GeoIP2 module automatically — no manual `load_module` directive is needed in `nginx.conf`. + +--- + +## GeoIP Setup + +The backend config resolves client IPs using MaxMind's free **GeoLite2-Country** database. A free MaxMind account is required. + +**1. Create a MaxMind account** + +Sign up at https://www.maxmind.com/en/geolite2/signup and generate a License Key under **My Account → Manage License Keys**. + +**2. Configure `/etc/GeoIP.conf`** + +```ini +AccountID YOUR_ACCOUNT_ID +LicenseKey YOUR_LICENSE_KEY +EditionIDs GeoLite2-Country +DatabaseDirectory /var/lib/GeoIP +``` + +Replace `YOUR_ACCOUNT_ID` and `YOUR_LICENSE_KEY` with your actual values. + +**3. Download the database** + +```bash +sudo geoipupdate +``` + +**4. Verify the file exists** + +```bash +ls -lh /var/lib/GeoIP/GeoLite2-Country.mmdb +``` + +Expected: a file of approximately 6–7 MB. If the file is missing, Nginx will fail to start with an error such as: + +``` +geoip2: unable to open database "/var/lib/GeoIP/GeoLite2-Country.mmdb" +``` + +--- + +## Deployment + +**1. Copy configs to `sites-available`** + +```bash +sudo cp rythmify-back.duckdns.org.conf /etc/nginx/sites-available/ +sudo cp rythmify.duckdns.org.conf /etc/nginx/sites-available/ +``` + +**2. Enable the sites** + +```bash +sudo ln -s /etc/nginx/sites-available/rythmify-back.duckdns.org.conf \ + /etc/nginx/sites-enabled/ +sudo ln -s /etc/nginx/sites-available/rythmify.duckdns.org.conf \ + /etc/nginx/sites-enabled/ +``` + +If the default Nginx site is enabled and conflicts with the `default_server` block in `rythmify.duckdns.org.conf`, disable it: + +```bash +sudo rm -f /etc/nginx/sites-enabled/default +``` + +**3. Test the configuration** + +```bash +sudo nginx -t +``` + +Both lines must read `syntax is ok` and `test is successful`. Fix any reported errors before continuing. + +**4. Reload Nginx** + +```bash +sudo systemctl reload nginx +``` + +--- + +## X-Country-Code Header + +Nginx resolves each request's client IP against the **MaxMind GeoLite2-Country** database using the `ngx_http_geoip2` module. The `geoip2` block at the top of `rythmify-back.duckdns.org.conf` declares the variable: + +```nginx +geoip2 /var/lib/GeoIP/GeoLite2-Country.mmdb { + $geoip2_data_country_code country iso_code; +} +``` + +That variable is then injected as a request header before the request is forwarded to the backend: + +```nginx +proxy_set_header X-Country-Code $geoip2_data_country_code; +``` + +The backend reads it as: + +```js +const country = req.headers['x-country-code']; // e.g. "EG", "SA", "US" +``` + +The value is an **ISO 3166-1 alpha-2** two-letter code. Examples: `EG` (Egypt), `SA` (Saudi Arabia), `US` (United States), `GB` (United Kingdom). The variable resolves to an empty string for private or unroutable IP addresses (e.g. `127.0.0.1`, `10.x.x.x`, `172.16.x.x`). + +Because the header is set by Nginx at the proxy layer, it **cannot be spoofed** by clients — any `X-Country-Code` value sent in the original request is overwritten by Nginx before it reaches the backend. + +--- + +## SSL + +SSL certificates are managed by **Certbot** and must **not** be committed to this repository. + +The configs reference certificates at: + +``` +/etc/letsencrypt/live/rythmify.duckdns.org/fullchain.pem +/etc/letsencrypt/live/rythmify.duckdns.org/privkey.pem +``` + +The certificate covers both `rythmify.duckdns.org` and `rythmify-back.duckdns.org` as Subject Alternative Names. Issue it with: + +```bash +sudo certbot certonly --nginx \ + -d rythmify.duckdns.org \ + -d rythmify-back.duckdns.org +``` + +Certbot installs a systemd timer (`certbot.timer`) for automatic renewal. Verify it is active: + +```bash +sudo systemctl status certbot.timer +``` + +The files `options-ssl-nginx.conf` and `ssl-dhparams.pem` referenced in both configs are created automatically by Certbot when the first certificate is issued. + +--- + +## Updating the GeoIP Database + +MaxMind releases updated GeoLite2-Country databases approximately twice per month. To refresh the local copy manually: + +```bash +sudo geoipupdate +sudo systemctl reload nginx +``` + +To automate this, create a cron job at `/etc/cron.d/geoipupdate`: + +```cron +# Refresh MaxMind GeoLite2-Country on the 1st and 15th of each month at 03:00 +0 3 1,15 * * root /usr/bin/geoipupdate && systemctl reload nginx +``` + +After adding the file, verify cron picks it up: + +```bash +sudo crontab -l +``` diff --git a/nginx/rythmify-back.duckdns.org.conf b/nginx/rythmify-back.duckdns.org.conf new file mode 100644 index 0000000..635fb58 --- /dev/null +++ b/nginx/rythmify-back.duckdns.org.conf @@ -0,0 +1,64 @@ +# GeoIP2 — resolves client IP to ISO 3166-1 alpha-2 country code. +# Requires: libnginx-mod-http-geoip2 and /var/lib/GeoIP/GeoLite2-Country.mmdb +# $geoip2_data_country_code is injected as X-Country-Code on every proxied +# request. Backend reads it via req.headers['x-country-code'] (e.g. "EG", "SA"). +geoip2 /var/lib/GeoIP/GeoLite2-Country.mmdb { + $geoip2_data_country_code country iso_code; +} + +# ── HTTP → HTTPS redirect ────────────────────────────────────────────────────── +server { + listen 80; + listen [::]:80; + server_name rythmify-back.duckdns.org; + + return 301 https://$host$request_uri; +} + +# ── HTTPS — backend API + WebSocket (Socket.IO) ──────────────────────────────── +server { + listen 443 ssl; + listen [::]:443 ssl; + server_name rythmify-back.duckdns.org; + + # Certificate covers both rythmify.duckdns.org and rythmify-back.duckdns.org + # as Subject Alternative Names. Issued with: + # sudo certbot certonly --nginx \ + # -d rythmify.duckdns.org -d rythmify-back.duckdns.org + ssl_certificate /etc/letsencrypt/live/rythmify.duckdns.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/rythmify.duckdns.org/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + # ── /health — liveness probe used by Blackbox exporter ────────────────────── + location /health { + proxy_pass http://localhost:8080; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # ── / — all API and WebSocket traffic ─────────────────────────────────────── + location / { + proxy_pass http://localhost:8080; + proxy_http_version 1.1; + + # WebSocket upgrade — required for Socket.IO long-polling and ws transports + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Country resolved from client IP via MaxMind GeoLite2-Country. + # Empty string when IP is unresolvable (private ranges, localhost). + proxy_set_header X-Country-Code $geoip2_data_country_code; + + proxy_cache_bypass $http_upgrade; + } +} diff --git a/nginx/rythmify.duckdns.org.conf b/nginx/rythmify.duckdns.org.conf new file mode 100644 index 0000000..415f645 --- /dev/null +++ b/nginx/rythmify.duckdns.org.conf @@ -0,0 +1,76 @@ +# ── HTTP → HTTPS redirect ────────────────────────────────────────────────────── +server { + listen 80; + listen [::]:80; + server_name rythmify.duckdns.org; + + return 301 https://$host$request_uri; +} + +# ── HTTPS — frontend (Vite dev server on port 5173) ─────────────────────────── +# +# The frontend is a React + Vite application. The Dockerfile runs: +# npm run dev -- --host 0.0.0.0 +# which starts the Vite dev server inside Docker, mapped to host port 5173. +# WebSocket upgrade headers are required for Vite HMR connections. +server { + listen 443 ssl; + listen [::]:443 ssl; + server_name rythmify.duckdns.org; + + ssl_certificate /etc/letsencrypt/live/rythmify.duckdns.org/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/rythmify.duckdns.org/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + location /nginx_status { + stub_status; + allow 127.0.0.1; + deny all; + } + + location / { + proxy_pass http://localhost:5173; + proxy_http_version 1.1; + + # WebSocket upgrade — required for Vite HMR + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_bypass $http_upgrade; + } +} + +# ── Nginx stub_status — for prometheus nginx_exporter ───────────────────────── +# +# devops/docker-compose.monitoring.yml nginx_exporter scrapes: +# http://host.docker.internal/nginx_status +# host.docker.internal resolves to the Docker host gateway (172.x.x.x), not +# 127.0.0.1, so this block listens on 0.0.0.0:80 as default_server and +# restricts access by source IP. +# +# Named server blocks above handle their own server_name directives. This +# default_server block catches requests whose Host header matches no other +# server_name (including host.docker.internal from the nginx_exporter container). +server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + + location /nginx_status { + stub_status on; + allow 127.0.0.1; + allow ::1; + allow 172.16.0.0/12; + deny all; + } + + location / { + return 444; + } +}