diff --git a/.github/workflows/devops.yml b/.github/workflows/devops.yml index 23514cb..1839428 100644 --- a/.github/workflows/devops.yml +++ b/.github/workflows/devops.yml @@ -223,11 +223,6 @@ jobs: - name: Run migrations run: docker exec rythmify_backend sh -c "npm run migrate:up" - - name: Run seeds - run: | - docker cp $GITHUB_WORKSPACE/backend/seeds/seed.sql rythmify_db:/seed.sql - docker exec rythmify_db psql -U dev -d rythmify_dev -f /seed.sql - - name: Run backend smoke check run: | echo "Running backend smoke check..." diff --git a/.gitignore b/.gitignore index 07d43cc..6675b61 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *.env *.env.* !.env.example +!monitoring/.env.monitoring.example environments/dev.env environments/staging.env environments/production.env @@ -80,4 +81,4 @@ coverage/ .terraform/ *.tfstate *.tfstate.* -crash.log \ No newline at end of file +crash.log diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml new file mode 100644 index 0000000..2aca50e --- /dev/null +++ b/docker-compose.monitoring.yml @@ -0,0 +1,80 @@ +services: + prometheus: + image: prom/prometheus:v2.55.1 + container_name: rythmify_prometheus + restart: unless-stopped + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --web.enable-lifecycle + extra_hosts: + - host.docker.internal:host-gateway + ports: + - "9090:9090" + volumes: + - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus_data:/prometheus + depends_on: + - node_exporter + - nginx_exporter + - blackbox_exporter + + grafana: + image: grafana/grafana:11.4.0 + container_name: rythmify_grafana + restart: unless-stopped + ports: + - "3000:3000" + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} + GF_SMTP_ENABLED: ${GF_SMTP_ENABLED} + GF_SMTP_HOST: ${SMTP_HOST} + GF_SMTP_USER: ${SMTP_USER} + GF_SMTP_PASSWORD: ${SMTP_PASSWORD} + GF_SMTP_FROM_ADDRESS: ${GF_SMTP_FROM_ADDRESS} + GF_SMTP_FROM_NAME: ${GF_SMTP_FROM_NAME} + ALERT_EMAIL_TO: ${ALERT_EMAIL_TO} + volumes: + - grafana_data:/var/lib/grafana + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + depends_on: + - prometheus + + node_exporter: + image: prom/node-exporter:v1.8.2 + container_name: rythmify_node_exporter + restart: unless-stopped + command: + - --path.rootfs=/host + pid: host + ports: + - "9100:9100" + volumes: + - /:/host:ro,rslave + + nginx_exporter: + image: nginx/nginx-prometheus-exporter:1.3.0 + container_name: rythmify_nginx_exporter + restart: unless-stopped + command: + - --nginx.scrape-uri=http://host.docker.internal/nginx_status + extra_hosts: + - host.docker.internal:host-gateway + ports: + - "9113:9113" + + blackbox_exporter: + image: prom/blackbox-exporter:v0.25.0 + container_name: rythmify_blackbox_exporter + restart: unless-stopped + command: + - --config.file=/etc/blackbox_exporter/blackbox.yml + ports: + - "9115:9115" + volumes: + - ./monitoring/blackbox/blackbox.yml:/etc/blackbox_exporter/blackbox.yml:ro + +volumes: + prometheus_data: + grafana_data: diff --git a/monitoring/README.md b/monitoring/README.md new file mode 100644 index 0000000..6d263e0 --- /dev/null +++ b/monitoring/README.md @@ -0,0 +1,122 @@ +# Rythmify Monitoring + +This stack runs Prometheus, Grafana, node_exporter, nginx_exporter, and blackbox_exporter on the Dir 2 VM. + +## Architecture + +- Prometheus scrapes every 15 seconds. +- Backend app metrics are scraped from `host.docker.internal:8080/metrics`. +- VM CPU, memory, and disk metrics come from `node_exporter`. +- Nginx metrics come from `nginx_exporter`, which reads `http://host.docker.internal/nginx_status`. +- Public uptime checks are run through `blackbox_exporter` for: + - `http://rythmify.duckdns.org/` + - `http://rythmify-back.duckdns.org/` + - `http://rythmify-back.duckdns.org/metrics` +- Grafana provisions Prometheus as the default data source and email alerts for core health signals. + +## VM Access + +```powershell +ssh -i "C:\Users\alaba\OneDrive\Desktop\rythmify-app-vm_key.pem" rythmify@20.196.3.253 +``` + +## Nginx stub_status + +Add this location to the Nginx server block that listens on localhost: + +```nginx +location /nginx_status { + stub_status; + allow 127.0.0.1; + allow 172.16.0.0/12; + allow 172.17.0.0/16; + deny all; +} +``` + +Validate and reload Nginx: + +```bash +sudo nginx -t +sudo systemctl reload nginx +curl http://localhost/nginx_status +``` + +## Environment + +Create a runtime environment file from the example: + +```bash +cp monitoring/.env.monitoring.example .env.monitoring +``` + +Edit `.env.monitoring` and replace `SMTP_PASSWORD` with a Gmail app password. Do not commit `.env.monitoring`. + +## Docker Commands + +Start the stack: + +```bash +docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml up -d +``` + +Check containers: + +```bash +docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml ps +``` + +View logs: + +```bash +docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml logs -f prometheus grafana +``` + +Restart after config changes: + +```bash +docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml restart prometheus grafana +``` + +Stop the stack: + +```bash +docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml down +``` + +## Verification + +```bash +curl http://localhost:9090/-/ready +curl http://localhost:9115/-/healthy +curl http://localhost/nginx_status +curl http://localhost:9090/api/v1/targets +``` + +Open Grafana: + +```text +http://20.196.3.253:3000 +``` + +Default credentials are configured through `.env.monitoring`. + +## Alerts + +Grafana provisions alerts for: + +- backend down +- high CPU +- high memory +- disk full +- high backend error rate +- high backend latency +- public endpoint down + +## Troubleshooting + +- If `backend_metrics` is down, verify the backend is listening on VM port `8080` and `curl http://localhost:8080/metrics` works on the VM. +- If `nginx_exporter` is down, verify `curl http://localhost/nginx_status` works on the VM and that the Nginx allow list includes Docker bridge ranges. +- If blackbox targets are down, run `curl -I` against each public URL from the VM. +- If Grafana email alerts do not send, verify `GF_SMTP_ENABLED=true`, `SMTP_HOST`, `SMTP_USER`, and the app password in `.env.monitoring`. +- If Prometheus cannot resolve `host.docker.internal`, confirm the compose services include `extra_hosts: host.docker.internal:host-gateway` and the host has a recent Docker Engine. diff --git a/monitoring/blackbox/blackbox.yml b/monitoring/blackbox/blackbox.yml new file mode 100644 index 0000000..383a16f --- /dev/null +++ b/monitoring/blackbox/blackbox.yml @@ -0,0 +1,11 @@ +modules: + http_2xx: + prober: http + timeout: 5s + http: + method: GET + preferred_ip_protocol: ip4 + valid_http_versions: + - HTTP/1.1 + - HTTP/2.0 + valid_status_codes: [] diff --git a/monitoring/grafana/provisioning/alerting/alerts.yml b/monitoring/grafana/provisioning/alerting/alerts.yml new file mode 100644 index 0000000..7135484 --- /dev/null +++ b/monitoring/grafana/provisioning/alerting/alerts.yml @@ -0,0 +1,297 @@ +apiVersion: 1 + +contactPoints: + - orgId: 1 + name: Rythmify Email + receivers: + - uid: rythmify-email + type: email + settings: + addresses: ${ALERT_EMAIL_TO} + singleEmail: true + +policies: + - orgId: 1 + receiver: Rythmify Email + group_by: + - grafana_folder + - alertname + +groups: + - orgId: 1 + name: Rythmify Monitoring Alerts + folder: Rythmify Monitoring + interval: 1m + rules: + - uid: backend-down + title: Backend Down + condition: C + data: + - refId: A + datasourceUid: prometheus + relativeTimeRange: + from: 300 + to: 0 + model: + expr: up{job="backend_metrics"} == 0 + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - A + reducer: + type: last + type: query + refId: C + type: classic_conditions + noDataState: Alerting + execErrState: Alerting + for: 2m + annotations: + summary: Backend metrics endpoint is down + labels: + severity: critical + + - uid: high-cpu + title: High CPU + condition: C + data: + - refId: A + datasourceUid: prometheus + relativeTimeRange: + from: 300 + to: 0 + model: + expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85 + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - A + reducer: + type: last + type: query + refId: C + type: classic_conditions + noDataState: NoData + execErrState: Error + for: 5m + annotations: + summary: VM CPU usage is above 85% + labels: + severity: warning + + - uid: high-memory + title: High Memory + condition: C + data: + - refId: A + datasourceUid: prometheus + relativeTimeRange: + from: 300 + to: 0 + model: + expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85 + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - A + reducer: + type: last + type: query + refId: C + type: classic_conditions + noDataState: NoData + execErrState: Error + for: 5m + annotations: + summary: VM memory usage is above 85% + labels: + severity: warning + + - uid: disk-full + title: Disk Full + condition: C + data: + - refId: A + datasourceUid: prometheus + relativeTimeRange: + from: 300 + to: 0 + model: + expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"})) * 100 > 85 + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - A + reducer: + type: last + type: query + refId: C + type: classic_conditions + noDataState: NoData + execErrState: Error + for: 5m + annotations: + summary: VM disk usage is above 85% + labels: + severity: warning + + - uid: high-error-rate + title: High Error Rate + condition: C + data: + - refId: A + datasourceUid: prometheus + relativeTimeRange: + from: 300 + to: 0 + model: + expr: sum(rate(http_errors_total[5m])) / clamp_min(sum(rate(http_requests_total[5m])), 1) > 0.05 + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - A + reducer: + type: last + type: query + refId: C + type: classic_conditions + noDataState: NoData + execErrState: Error + for: 5m + annotations: + summary: Backend HTTP error rate is above 5% + labels: + severity: warning + + - uid: high-latency + title: High Latency + condition: C + data: + - refId: A + datasourceUid: prometheus + relativeTimeRange: + from: 300 + to: 0 + model: + expr: histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m]))) > 1 + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - A + reducer: + type: last + type: query + refId: C + type: classic_conditions + noDataState: NoData + execErrState: Error + for: 5m + annotations: + summary: Backend p95 latency is above 1 second + labels: + severity: warning + + - uid: public-endpoint-down + title: Public Endpoint Down + condition: C + data: + - refId: A + datasourceUid: prometheus + relativeTimeRange: + from: 300 + to: 0 + model: + expr: probe_success{job="blackbox"} == 0 + instant: true + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + type: gt + operator: + type: and + query: + params: + - A + reducer: + type: last + type: query + refId: C + type: classic_conditions + noDataState: Alerting + execErrState: Alerting + for: 2m + annotations: + summary: One or more public Rythmify endpoints are down + labels: + severity: critical diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..00f9915 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..7858acc --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,42 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - prometheus:9090 + + - job_name: node_exporter + static_configs: + - targets: + - node_exporter:9100 + + - job_name: nginx_exporter + static_configs: + - targets: + - nginx_exporter:9113 + + - job_name: backend_metrics + metrics_path: /metrics + static_configs: + - targets: + - host.docker.internal:8080 + + - job_name: blackbox + metrics_path: /probe + params: + module: + - http_2xx + static_configs: + - targets: + - http://rythmify.duckdns.org/ + - http://rythmify-back.duckdns.org/ + - http://rythmify-back.duckdns.org/metrics + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox_exporter:9115