feat(monitoring): enhance blackbox and alerting configurations

docker-compose.monitoring.yml

@@ -1,3 +1,9 @@
+# Run with:
+#   docker compose --env-file ./environments/.env.monitoring -f docker-compose.monitoring.yml up -d
+
+networks:
+  monitoring:
+
 services:
   prometheus:
     image: prom/prometheus:v2.55.1
@@ -6,18 +12,44 @@ services:
     command:
       - --config.file=/etc/prometheus/prometheus.yml
       - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=30d
       - --web.enable-lifecycle
     extra_hosts:
       - host.docker.internal:host-gateway
     ports:
       - "9090:9090"
     volumes:
       - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./monitoring/prometheus/alert_rules.yml:/etc/prometheus/alert_rules.yml:ro
       - prometheus_data:/prometheus
+    networks:
+      - monitoring
     depends_on:
+      - alertmanager
       - node_exporter
-      - nginx_exporter
+      - nginx-exporter
       - blackbox_exporter
+      - postgres_exporter
+
+  alertmanager:
+    image: prom/alertmanager:v0.27.0
+    container_name: rythmify_alertmanager
+    restart: unless-stopped
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        apk add --no-cache gettext &&
+        envsubst < /etc/alertmanager/alertmanager.yml.tmpl > /tmp/alertmanager.yml &&
+        /bin/alertmanager --config.file=/tmp/alertmanager.yml --storage.path=/alertmanager
+    environment:
+      ALERTMANAGER_SMTP_PASSWORD: ${ALERTMANAGER_SMTP_PASSWORD}
+    volumes:
+      - alertmanager_data:/alertmanager
+      - ./monitoring/alertmanager/alertmanager.yml.tmpl:/etc/alertmanager/alertmanager.yml.tmpl
+    ports:
+      - "9093:9093"
+    networks:
+      - monitoring
 
   grafana:
     image: grafana/grafana:11.4.0
@@ -26,18 +58,21 @@ services:
     ports:
       - "3000:3000"
     environment:
-      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER}
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
       GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
-      GF_SMTP_ENABLED: ${GF_SMTP_ENABLED}
-      GF_SMTP_HOST: ${SMTP_HOST}
-      GF_SMTP_USER: ${SMTP_USER}
-      GF_SMTP_PASSWORD: ${SMTP_PASSWORD}
-      GF_SMTP_FROM_ADDRESS: ${GF_SMTP_FROM_ADDRESS}
-      GF_SMTP_FROM_NAME: ${GF_SMTP_FROM_NAME}
-      ALERT_EMAIL_TO: ${ALERT_EMAIL_TO}
+      GF_SMTP_ENABLED: ${GF_SMTP_ENABLED:-false}
+      GF_SMTP_HOST: ${SMTP_HOST:-smtp.gmail.com:587}
+      GF_SMTP_USER: ${SMTP_USER:-}
+      GF_SMTP_PASSWORD: ${SMTP_PASSWORD:-}
+      GF_SMTP_FROM_ADDRESS: ${GF_SMTP_FROM_ADDRESS:-}
+      GF_SMTP_FROM_NAME: ${GF_SMTP_FROM_NAME:-Rythmify Monitoring}
+      GF_SMTP_STARTTLS_POLICY: MandatoryStartTLS
+      ALERT_EMAIL_TO: ${ALERT_EMAIL_TO:-}
     volumes:
       - grafana_data:/var/lib/grafana
       - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
+    networks:
+      - monitoring
     depends_on:
       - prometheus
 
@@ -52,17 +87,21 @@ services:
       - "9100:9100"
     volumes:
       - /:/host:ro,rslave
+    networks:
+      - monitoring
 
-  nginx_exporter:
-    image: nginx/nginx-prometheus-exporter:1.3.0
+  nginx-exporter:
+    image: nginx/nginx-prometheus-exporter:latest
     container_name: rythmify_nginx_exporter
     restart: unless-stopped
     command:
-      - --nginx.scrape-uri=http://host.docker.internal/nginx_status
+      - '--nginx.scrape-uri=http://host.docker.internal/nginx_status'
     extra_hosts:
       - host.docker.internal:host-gateway
     ports:
       - "9113:9113"
+    networks:
+      - monitoring
 
   blackbox_exporter:
     image: prom/blackbox-exporter:v0.25.0
@@ -74,7 +113,24 @@ services:
       - "9115:9115"
     volumes:
       - ./monitoring/blackbox/blackbox.yml:/etc/blackbox_exporter/blackbox.yml:ro
+    networks:
+      - monitoring
+
+  # Reaches PostgreSQL on VM2 private IP 10.0.1.4.
+  # Pre-requisite on VM2:
+  #   GRANT pg_monitor TO rythmify_admin;
+  postgres_exporter:
+    image: quay.io/prometheuscommunity/postgres-exporter:v0.15.0
+    container_name: rythmify_postgres_exporter
+    restart: unless-stopped
+    environment:
+      DATA_SOURCE_NAME: "postgresql://rythmify_admin:${POSTGRES_PASSWORD}@10.0.1.4:5432/rythmify_db?sslmode=disable"
+    ports:
+      - "9187:9187"
+    networks:
+      - monitoring
 
 volumes:
   prometheus_data:
   grafana_data:
+  alertmanager_data:

monitoring/.env.monitoring.example

@@ -0,0 +1,32 @@
+# Copy this file to devops/environments/.env.monitoring and fill in all values.
+# Start the stack with:
+#   docker compose --env-file ./environments/.env.monitoring -f docker-compose.monitoring.yml up -d
+
+# ── Grafana ────────────────────────────────────────────────────────────────────
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=your_grafana_password_here
+
+# ── SMTP (Gmail) ───────────────────────────────────────────────────────────────
+# Generate an App Password at https://myaccount.google.com/apppasswords
+# Use port 587 (STARTTLS). Port 465 (SSL) is not used here.
+GF_SMTP_ENABLED=true
+SMTP_HOST=smtp.gmail.com:587
+SMTP_USER=your-sender@gmail.com
+SMTP_PASSWORD=your-16-char-app-password
+GF_SMTP_FROM_ADDRESS=your-sender@gmail.com
+GF_SMTP_FROM_NAME=Rythmify Monitoring
+
+# ── Alert recipient ────────────────────────────────────────────────────────────
+ALERT_EMAIL_TO=oncall@example.com
+
+# ── Alertmanager SMTP ─────────────────────────────────────────────────────────
+# Generate a Gmail App Password at: https://myaccount.google.com/apppasswords
+# WARNING: do NOT use special characters in this value — envsubst will break
+# if the password contains /, &, @, or $. Gmail App Passwords (16 lowercase
+# letters, no punctuation) are safe by default.
+ALERTMANAGER_SMTP_PASSWORD=your_gmail_app_password_here
+
+# ── PostgreSQL exporter ────────────────────────────────────────────────────────
+# rythmify_admin must have the pg_monitor role on VM2:
+#   GRANT pg_monitor TO rythmify_admin;
+POSTGRES_PASSWORD=your_postgres_password_here

monitoring/alertmanager/alertmanager.yml.tmpl

@@ -0,0 +1,36 @@
+# Alertmanager config template — ${ALERTMANAGER_SMTP_PASSWORD} is substituted at
+# container start via envsubst in docker-compose.monitoring.yml. The raw template
+# is never executed by Alertmanager directly.
+
+global:
+  smtp_smarthost: 'smtp.gmail.com:587'
+  smtp_from: 'rythmify.noreply@gmail.com'
+  smtp_auth_username: 'rythmify.noreply@gmail.com'
+  smtp_auth_password: '${ALERTMANAGER_SMTP_PASSWORD}'
+  smtp_require_tls: true
+
+route:
+  receiver: email
+  group_by: [alertname, severity]
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
+  routes:
+    - matchers:
+        - severity = critical
+      receiver: email
+      group_wait: 10s
+      repeat_interval: 1h
+
+receivers:
+  - name: email
+    email_configs:
+      - to: 'alabasy@live.com'
+        send_resolved: true
+
+inhibit_rules:
+  - source_matchers:
+      - severity = critical
+    target_matchers:
+      - severity = warning
+    equal: [alertname]

monitoring/blackbox/blackbox.yml

@@ -1,4 +1,5 @@
 modules:
+  # Used internally or for plain HTTP endpoints
   http_2xx:
     prober: http
     timeout: 5s
@@ -9,3 +10,18 @@ modules:
         - HTTP/1.1
         - HTTP/2.0
       valid_status_codes: []
+
+  # Used for all public HTTPS endpoints — fails if TLS absent or cert invalid
+  https_2xx:
+    prober: http
+    timeout: 10s
+    http:
+      method: GET
+      preferred_ip_protocol: ip4
+      valid_http_versions:
+        - HTTP/1.1
+        - HTTP/2.0
+      valid_status_codes: []
+      tls_config:
+        insecure_skip_verify: false
+      fail_if_not_ssl: true

monitoring/grafana/provisioning/alerting/alerts.yml

@@ -16,13 +16,25 @@ policies:
     group_by:
       - grafana_folder
       - alertname
+    group_wait: 30s
+    group_interval: 5m
+    repeat_interval: 4h
+    routes:
+      - receiver: Rythmify Email
+        matchers:
+          - severity = critical
+        group_wait: 10s
+        repeat_interval: 1h
 
 groups:
   - orgId: 1
     name: Rythmify Monitoring Alerts
     folder: Rythmify Monitoring
     interval: 1m
     rules:
+      # FIX: was `up{job="backend_metrics"} == 0` with `type: gt, params: [0]`
+      # That logic was inverted — fired when UP (no-data), silent when DOWN (0 > 0 = false).
+      # Fixed: query returns raw `up` value; condition fires when up < 1.
       - uid: backend-down
         title: Backend Down
         condition: C
@@ -33,7 +45,7 @@ groups:
               from: 300
               to: 0
             model:
-              expr: up{job="backend_metrics"} == 0
+              expr: up{job="backend_metrics"}
               instant: true
               refId: A
           - refId: C
@@ -42,8 +54,8 @@ groups:
               conditions:
                 - evaluator:
                     params:
-                      - 0
-                    type: gt
+                      - 1
+                    type: lt
                   operator:
                     type: and
                   query:
@@ -257,6 +269,8 @@ groups:
         labels:
           severity: warning
 
+      # FIX: same inverted-logic bug as backend-down.
+      # Fixed: query returns raw `probe_success`; condition fires when probe_success < 1.
       - uid: public-endpoint-down
         title: Public Endpoint Down
         condition: C
@@ -267,7 +281,7 @@ groups:
               from: 300
               to: 0
             model:
-              expr: probe_success{job="blackbox"} == 0
+              expr: probe_success{job="blackbox"}
               instant: true
               refId: A
           - refId: C
@@ -276,8 +290,8 @@ groups:
               conditions:
                 - evaluator:
                     params:
-                      - 0
-                    type: gt
+                      - 1
+                    type: lt
                   operator:
                     type: and
                   query: