From 0951cc257974faaaa43043f3c0dcd3e10a9ab3e0 Mon Sep 17 00:00:00 2001 From: Mohamed jameel alabasy Date: Sat, 2 May 2026 09:07:00 +0300 Subject: [PATCH] feat(docs): add Rythmify VM Operations Guide for Azure VMs --- README.md | 578 ++++----------------------- docs/vm-guide.md | 198 +++++++++ monitoring/README.md | 253 ++++++++---- nginx/README.md | 81 +++- nginx/rythmify-back.duckdns.org.conf | 26 +- nginx/rythmify.duckdns.org.conf | 35 -- 6 files changed, 542 insertions(+), 629 deletions(-) create mode 100644 docs/vm-guide.md diff --git a/README.md b/README.md index 83f0578..02b20ae 100644 --- a/README.md +++ b/README.md @@ -1,548 +1,110 @@ -# 🎡 Rythmify β€” DevOps Repository +# Rythmify DevOps -> **Rythmify** is a university software engineering project β€” a SoundCloud-like music streaming platform. This repository contains all DevOps infrastructure, CI/CD pipelines, containerization configuration, environment management, monitoring plans, and deployment strategies that support the development lifecycle. +Infrastructure, CI/CD, Docker, Azure deployment, Nginx, and monitoring for the Rythmify platform. --- -## Table of Contents +## Overview -1. [Project Overview](#1-project-overview) -2. [DevOps Architecture Overview](#2-devops-architecture-overview) -3. [Repository Structure Explanation](#3-repository-structure-explanation) -4. [Local Development Workflow](#4-local-development-workflow) -5. [CI/CD Strategy β€” GitHub Actions](#5-cicd-strategy--github-actions) -6. [Containerization Strategy β€” Docker](#6-containerization-strategy--docker) -7. [Environment Configuration](#7-environment-configuration) -8. [Monitoring Plan β€” Grafana (Future Phase)](#8-monitoring-plan--grafana-future-phase) -9. [Future Deployment Architecture β€” Azure](#9-future-deployment-architecture--azure) -10. [Quick Start for Developers](#10-quick-start-for-developers) +This repository manages everything that keeps Rythmify running: containerization for local development, GitHub Actions pipelines for CI/CD, Nginx virtual host configuration for both public domains, and the full Prometheus/Grafana/Alertmanager monitoring stack deployed on Azure VMs. --- -## 1. Project Overview - -**Rythmify** is a music streaming web platform inspired by SoundCloud, enabling users to upload, stream, discover, and interact with music content. The system is built with a decoupled frontend/backend architecture and is designed for scalability through cloud deployment in later phases. - -| Attribute | Detail | -| ----------------- | ----------------------------------------- | -| **Project Type** | University Software Engineering Capstone | -| **Domain** | Music Streaming Platform | -| **Architecture** | Frontend + Backend + Database (decoupled) | -| **Current Phase** | Local Development | -| **Future Phases** | Cloud deployment on Microsoft Azure | - -### DevOps Stack at a Glance - -| Category | Technology | -| ----------------------- | ----------------------------- | -| CI/CD | GitHub Actions | -| Containerization | Docker & Docker Compose | -| Cloud Provider (future) | Microsoft Azure | -| Database (future) | Azure Database for PostgreSQL | -| File Storage (future) | Azure Blob Storage | -| Monitoring (future) | Grafana | - ---- - -## 2. DevOps Architecture Overview - -The DevOps pipeline for Rythmify is structured around three progressive phases: +## Repository Structure ``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ PHASE 1 β€” CURRENT β”‚ -β”‚ Local Development β”‚ -β”‚ β”‚ -β”‚ Developer ──► Docker Compose ──► Backend + Frontend (local) β”‚ -β”‚ β”‚ β”‚ -β”‚ GitHub Actions β”‚ -β”‚ (CI: Lint, Test, Build) β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ PHASE 2 β€” FUTURE β”‚ -β”‚ Azure Deployment β”‚ -β”‚ β”‚ -β”‚ GitHub Actions ──► Build & Push Images β”‚ -β”‚ β”‚ β”‚ -β”‚ └──► Azure App Service (Backend + Frontend) β”‚ -β”‚ Azure PostgreSQL (Database) β”‚ -β”‚ Azure Blob Storage (Media Files) β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ PHASE 3 β€” FUTURE β”‚ -β”‚ Monitoring & Observability β”‚ -β”‚ β”‚ -β”‚ Grafana Dashboards ◄── Metrics, Logs & Alerts β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - ---- - -## 3. Repository Structure Explanation - -``` -C:. -β”œβ”€β”€ .gitignore -β”œβ”€β”€ README.md -β”‚ +devops/ β”œβ”€β”€ .github/ -β”‚ β”œβ”€β”€ CODEOWNERS -β”‚ └── workflows/ -β”‚ β”œβ”€β”€ the ci pipelines will be here -β”‚ -β”œβ”€β”€ docker/ -β”‚ β”œβ”€β”€ docker-compose.yml -β”‚ β”œβ”€β”€ backend/ -β”‚ β”‚ └── Dockerfile -β”‚ └── frontend/ -β”‚ └── Dockerfile -β”‚ -β”œβ”€β”€ docs/ -β”‚ β”œβ”€β”€ deployment-guide.md -β”‚ β”œβ”€β”€ infrastructure-overview.md -β”‚ └── system-architecture.md -β”‚ -β”œβ”€β”€ environments/ -β”‚ β”œβ”€β”€ .env -β”‚ β”œβ”€β”€ .env.example -β”‚ -β”œβ”€β”€ monitoring/ -β”‚ β”œβ”€β”€ alerts.md -β”‚ β”œβ”€β”€ grafana.md -β”‚ └── logs.md -β”‚ -└── scripts/ - β”œβ”€β”€ here is where all the scripts will be -``` - -### Directory Breakdown - -#### `.github/` - -Contains all GitHub-specific configuration for the repository. - -| File / Folder | Purpose | -| --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `CODEOWNERS` | Defines which team members are responsible for reviewing changes to specific parts of the codebase. Ensures the right people are auto-assigned to pull requests. | -| `workflows/backend-ci.yml` | GitHub Actions pipeline triggered on backend code changes. Runs linting, unit tests, and build validation. | -| `workflows/frontend-ci.yml` | GitHub Actions pipeline triggered on frontend code changes. Runs linting, component tests, and build checks. | -| `workflows/mobile-ci.yml` | GitHub Actions pipeline for any mobile client code. Mirrors the frontend CI structure for the mobile platform. | -| `workflows/deploy.yml` | Deployment workflow (initially a stub). Will orchestrate building Docker images and deploying to Azure in future phases. | - ---- - -#### `docker/` - -Centralizes all containerization configuration for the Rythmify platform. - -| File / Folder | Purpose | -| --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `docker-compose.yml` | Orchestrates the full local development environment. Defines services (backend, frontend), networking between containers, volume mounts, and environment variable injection. | -| `backend/Dockerfile` | Defines the Docker image for the backend service β€” base image, dependency installation, build steps, and the runtime command. | -| `frontend/Dockerfile` | Defines the Docker image for the frontend service β€” Node.js build stage, static asset generation, and serving configuration. | - ---- - -#### `docs/` - -Houses human-readable documentation for infrastructure and system design decisions. - -| File | Purpose | -| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | -| `system-architecture.md` | High-level description of how the frontend, backend, database, and storage components interact. Useful for onboarding new developers. | -| `infrastructure-overview.md` | Documents the infrastructure stack β€” what tools are used, why they were chosen, and how they fit together. | -| `deployment-guide.md` | Step-by-step instructions for deploying the system, both locally and (in future phases) to Azure. | - ---- - -#### `environments/` - -Manages environment-specific configuration variables across all deployment stages. - -| File | Purpose | -| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `.env.example` | A template file showing all required environment variables with placeholder values. Committed to version control. New developers copy this to create their local `.env`. | -| `dev.env` | Environment variables for local development (e.g., local DB URLs, debug flags, mock storage paths). | -| `staging.env` | Variables for the staging environment, pointing to staging Azure resources. Used for pre-production validation. | -| `production.env` | Variables for the production environment. Contains references to live Azure services. **Secrets must never be stored in plain text β€” use Azure Key Vault or GitHub Secrets in later phases.** | - -> ⚠️ **Security Note:** `.env` files containing real credentials must be listed in `.gitignore` and should never be committed to version control. - ---- - -#### `monitoring/` - -Documents the monitoring and observability strategy for the platform (to be implemented in a future phase). - -| File | Purpose | -| ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| `grafana.md` | Describes the planned Grafana dashboard setup β€” which metrics to visualize, how to connect Grafana to the backend, and dashboard layout plans. | -| `alerts.md` | Defines alerting rules and thresholds β€” what conditions should trigger alerts, who gets notified, and via which channels (email, Slack, etc.). | -| `logs.md` | Describes the logging strategy β€” log formats, log levels, which services emit logs, and how logs will be aggregated and queried. | - ---- - -#### `scripts/` - -Cross-platform automation scripts for common DevOps tasks. Each action has both a Bash (`.sh`) version for Linux/macOS and a PowerShell (`.ps1`) version for Windows. - -| Script | Purpose | -| -------------------------- | -------------------------------------------------------------------------------------------------- | -| `build.sh` / `build.ps1` | Builds Docker images for the backend and frontend services. | -| `deploy.sh` / `deploy.ps1` | Triggers a deployment (currently a stub; will deploy to Azure in future phases). | -| `test.sh` / `test.ps1` | Runs the test suite across all services β€” useful for pre-commit validation or local CI simulation. | - ---- - -## 4. Local Development Workflow - -In the current phase, Rythmify runs entirely on a developer's local machine via Docker Compose. - -``` -Developer Workstation -β”‚ -β”œβ”€β”€ Edits source code in the main application repositories -β”‚ -β”œβ”€β”€ Runs: docker-compose up --build -β”‚ β”‚ -β”‚ β”œβ”€β”€ Starts Backend container (port 8000) -β”‚ └── Starts Frontend container (port 3000) -β”‚ -└── Accesses app at http://localhost:3000 -``` - -### Workflow Steps - -1. **Clone** this DevOps repository alongside the application repositories. -2. **Copy** `environments/.env.example` to `environments/dev.env` and fill in local values. -3. **Build & start** all services using Docker Compose: - ```bash - docker-compose -f docker/docker-compose.yml --env-file environments/dev.env up --build - ``` -4. **Develop** β€” changes to application code can be reflected via volume mounts (hot reload). -5. **Test** locally before pushing: - - ```bash - # Linux / macOS - bash scripts/test.sh - - # Windows - .\scripts\test.ps1 - ``` - -6. **Push** to a feature branch β€” CI pipelines trigger automatically. - ---- - -## 5. CI/CD Strategy β€” GitHub Actions - -GitHub Actions powers the automated CI/CD pipeline. Workflows are triggered on push and pull request events. - -### Pipeline Overview - -``` -Push / Pull Request - β”‚ - β–Ό -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ GitHub Actions β”‚ -β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ backend-ci β”‚ β”‚ frontend-ci β”‚ β”‚ -β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ -β”‚ β”‚ β€’ Lint β”‚ β”‚ β€’ Lint β”‚ β”‚ -β”‚ β”‚ β€’ Unit Test β”‚ β”‚ β€’ Unit Test β”‚ β”‚ -β”‚ β”‚ β€’ Build β”‚ β”‚ β€’ Build β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ mobile-ci β”‚ β”‚ deploy (stub) β”‚ β”‚ -β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ -β”‚ β”‚ β€’ Lint β”‚ β”‚ Future: Push β”‚ β”‚ -β”‚ β”‚ β€’ Test β”‚ β”‚ images to ACR β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` - -### Workflow Trigger Strategy - -| Workflow | Trigger | Target Branch(es) | -| ----------------- | ----------------------------------------- | ----------------------------------- | -| `backend-ci.yml` | Push / PR to paths matching `backend/**` | `main`, `develop`, feature branches | -| `frontend-ci.yml` | Push / PR to paths matching `frontend/**` | `main`, `develop`, feature branches | -| `mobile-ci.yml` | Push / PR to paths matching `mobile/**` | `main`, `develop`, feature branches | -| `deploy.yml` | Manual trigger / merge to `main` | `main` only | - -### Branch Protection - -The `CODEOWNERS` file enforces code review requirements β€” designated owners must approve pull requests before merging into protected branches. This ensures accountability and knowledge sharing across the team. - ---- - -## 6. Containerization Strategy β€” Docker - -Docker is used to create consistent, reproducible environments across all developer machines and deployment targets. - -### Service Architecture - -``` -docker-compose.yml -β”‚ -β”œβ”€β”€ backend (service) -β”‚ β”œβ”€β”€ Image: built from docker/backend/Dockerfile -β”‚ β”œβ”€β”€ Port: 8000:8000 -β”‚ └── Env: injected from dev.env -β”‚ -└── frontend (service) - β”œβ”€β”€ Image: built from docker/frontend/Dockerfile - β”œβ”€β”€ Port: 3000:3000 - └── Depends on: backend +β”‚ └── workflows/ # GitHub Actions CI/CD pipelines +β”œβ”€β”€ docker/ # Docker Compose files for backend and frontend stacks +β”œβ”€β”€ monitoring/ # Prometheus, Grafana, Alertmanager, Blackbox configs +β”œβ”€β”€ nginx/ # Nginx virtual host configs and GeoIP setup for VM1 +β”œβ”€β”€ scripts/ # Deployment and utility scripts (bash + PowerShell) +β”œβ”€β”€ docs/ # Operational guides +β”œβ”€β”€ environments/ # .env.example templates (never commit filled .env files) +└── docker-compose.monitoring.yml # Monitoring stack compose file ``` -### Dockerfile Philosophy - -| Service | Base Image Strategy | Build Approach | -| -------- | ----------------------------------------------------- | ---------------------------------------------- | -| Backend | Lightweight runtime image (e.g., `python:3.x-slim`) | Install dependencies, copy source, expose port | -| Frontend | Multi-stage build (`node:lx-alpine` β†’ `nginx:alpine`) | Build static assets, serve via Nginx | - -### Why Docker? - -- **Consistency** β€” eliminates "works on my machine" issues across the team. -- **Isolation** β€” each service runs in its own container with its own dependencies. -- **Cloud-readiness** β€” the same images used locally can be pushed to Azure Container Registry (ACR) in future phases. -- **Onboarding speed** β€” new developers can run the full stack with a single command. +| Folder | Description | +|---|---| +| `.github/workflows/` | GitHub Actions pipelines β€” validates repo structure on PRs to `main` | +| `docker/` | Docker Compose files (`docker-compose.yml`, `docker-compose.backend.yml`, `docker-compose.frontend.yml`) and Dockerfiles | +| `monitoring/` | Full observability stack config β€” Prometheus rules, Grafana dashboards, Alertmanager templates, Blackbox module config | +| `nginx/` | Nginx virtual host configs for `rythmify.duckdns.org` and `rythmify-back.duckdns.org`, GeoIP2 setup | +| `scripts/` | `build`, `start`, `stop`, `migrate`, `seed`, `clean` β€” available in both `.sh` (Linux/Mac) and `.ps1` (Windows) | +| `docs/` | Operational guides β€” local setup, VM operations | +| `environments/` | `.env.example` templates for local development | --- -## 7. Environment Configuration - -The `environments/` directory provides a structured approach to managing configuration across all deployment stages. - -### Environment Hierarchy - -| Environment | File | Purpose | Azure Resources | -| ----------- | ---------------- | ---------------------- | -------------------------- | -| Development | `dev.env` | Local machine | None (local services) | -| Staging | `staging.env` | Pre-production testing | Staging Azure instances | -| Production | `production.env` | Live user traffic | Production Azure instances | +## Quick Start -### Required Variables (see `.env.example`) +**Prerequisites:** Docker Desktop, Git. -```env -# Application -APP_ENV=development -SECRET_KEY=your-secret-key-here -DEBUG=true +```bash +# Clone the repo +git clone +cd devops -# Database -DATABASE_URL=postgresql://user:password@localhost:5432/rythmify_db +# Copy and fill the environment file +cp environments/.env.example environments/.env +# Edit environments/.env with your values -# Storage -STORAGE_BUCKET=your-blob-storage-bucket -STORAGE_CONNECTION_STRING=your-connection-string +# Build images +./scripts/build.sh # Linux/Mac +.\scripts\build.ps1 # Windows -# API -API_BASE_URL=http://localhost:8000 +# Start everything +./scripts/start.sh # Linux/Mac +.\scripts\start.ps1 # Windows ``` -### Security Guidelines - -- Real credentials are **never** committed to version control. -- The `.env.example` file serves as the contract for required variables. -- In production, secrets will be managed through **Azure Key Vault** and injected via **GitHub Actions Secrets**. - ---- - -## 8. Monitoring Plan β€” Grafana (Future Phase) - -> πŸ“Œ **Status:** Planned for a future implementation phase. The `monitoring/` directory contains design documentation only. - -Grafana will provide real-time observability into the Rythmify platform once deployed to Azure. - -### Planned Dashboards - -| Dashboard | Metrics Tracked | -| --------------------- | -------------------------------------------------------- | -| **System Health** | CPU usage, memory, disk I/O per container/service | -| **API Performance** | Request rate, response times (p50/p95/p99), error rates | -| **Streaming Metrics** | Active streams, bandwidth usage, buffering events | -| **Database** | Query latency, connection pool utilization, slow queries | -| **User Activity** | Active sessions, uploads per hour, plays per track | +Services after `start`: -### Alerting Strategy (defined in `monitoring/alerts.md`) +| Service | URL | +|---|---| +| Backend API | http://localhost:8080/api/v1 | +| Frontend | http://localhost:5173 | +| pgAdmin | http://localhost:5050 | +| Azurite (blob storage) | http://localhost:10000 | -| Alert | Condition | Severity | Notification Channel | -| --------------------- | ------------------------- | -------- | -------------------- | -| High Error Rate | HTTP 5xx > 5% for 5 min | Critical | Email + Slack | -| Slow API Response | p95 latency > 2s | Warning | Slack | -| Database Overload | Connections > 80% of pool | Critical | Email | -| Storage Near Capacity | Azure Blob > 85% used | Warning | Email | - -### Logging Strategy (defined in `monitoring/logs.md`) - -- **Format:** Structured JSON logs for machine parseability. -- **Levels:** `DEBUG` (dev only), `INFO`, `WARNING`, `ERROR`, `CRITICAL`. -- **Aggregation:** Azure Monitor Logs / Log Analytics Workspace (future). -- **Retention:** 30 days for standard logs; 90 days for audit logs. +See [docs/setup-guide.md](docs/setup-guide.md) for the full first-time setup walkthrough including blob container creation and database migrations. --- -## 9. Future Deployment Architecture β€” Azure - -> πŸ“Œ **Status:** Planned for a future implementation phase. - -Once local development is stable, the platform will be deployed to Microsoft Azure. - -### Target Architecture - -``` - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ Microsoft Azure β”‚ - β”‚ β”‚ - GitHub Actions ────► β”‚ Azure Container Registry (ACR) β”‚ - (deploy.yml) β”‚ (Docker Images) β”‚ - β”‚ β”‚ - β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ - β”‚ β”‚ Azure App Service β”‚ β”‚ - β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ - β”‚ β”‚ β”‚ Backend β”‚ β”‚Frontend β”‚ β”‚ β”‚ - β”‚ β”‚ β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ - β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ - β”‚ β”‚ β”‚ - β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ - β”‚ β”‚ Azure PostgreSQL DB β”‚ β”‚ - β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ - β”‚ β”‚ - β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ - β”‚ β”‚ Azure Blob Storage β”‚ β”‚ - β”‚ β”‚ (Audio & Media Files) β”‚ β”‚ - β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ - β”‚ β”‚ - β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ - β”‚ β”‚ Grafana (Monitoring) β”‚ β”‚ - β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` +## Environments -### Azure Service Mapping +| Environment | Branch | Approval | +|---|---|---| +| Development | any feature branch | none β€” auto on push | +| Production | `main` | manual approval gate via GitHub Environment protection rule | -| Azure Service | Role in Rythmify | -| ---------------------------------- | ----------------------------------------------------------------------------------- | -| **Azure App Service** | Hosts the backend API and frontend web application as containerized services. | -| **Azure Container Registry (ACR)** | Private registry for storing built Docker images pushed by GitHub Actions. | -| **Azure Database for PostgreSQL** | Managed relational database for user data, track metadata, playlists, and more. | -| **Azure Blob Storage** | Scalable object storage for audio files, album artwork, and other media assets. | -| **Azure Key Vault** | Secure storage for secrets, API keys, and connection strings β€” injected at runtime. | -| **Azure Monitor** | Platform-level metrics and log aggregation feeding into Grafana dashboards. | - -### Deployment Flow (Future) - -```bash -# GitHub Actions deploy.yml will: - -1. Build Docker images for backend and frontend -2. Push images to Azure Container Registry (ACR) -3. Pull new images into Azure App Service (rolling update) -4. Run database migrations (if applicable) -5. Health check the deployment endpoint -6. Notify team of success or failure -``` +Deployments are triggered by `.github/workflows/devops.yml`. Production deploys require a reviewer to approve the job in the GitHub Actions UI before it runs. --- -## 10. Quick Start for Developers +## Secrets Required -### Prerequisites +The following secrets must be configured in the GitHub repository or org settings. Production secrets go under the **production** GitHub Environment; org-level secrets are shared across environments. -Ensure the following tools are installed on your machine: - -| Tool | Version | Purpose | -| -------------- | ------- | --------------------------------------- | -| Git | Latest | Version control | -| Docker | 24.x+ | Container runtime | -| Docker Compose | 2.x+ | Multi-container orchestration | -| PowerShell | 7.x+ | Windows script execution (Windows only) | - -### Setup Instructions - -```bash -# 1. Clone this DevOps repository -git clone https://github.com/your-org/rythmify-devops.git -cd rythmify-devops - -# 2. Set up your local environment file -cp environments/.env.example environments/dev.env -# Edit dev.env with your local configuration values - -# 3. Build and start all services -docker-compose -f docker/docker-compose.yml --env-file environments/dev.env up --build - -# 4. Verify services are running -docker ps - -# 5. Access the application -# Frontend: http://localhost:3000 -# Backend: http://localhost:8000 -# API Docs: http://localhost:8000/docs -``` - -### Common Commands - -```bash -# Start services (detached mode) -docker-compose -f docker/docker-compose.yml up -d +| Secret | Where | Description | +|---|---|---| +| `ACR_USERNAME` | Org or repo | Azure Container Registry username | +| `ACR_PASSWORD` | Org or repo | Azure Container Registry password | +| `AZURE_CREDENTIALS` | Org or repo | Service principal JSON for `az login` | +| `AZURE_RESOURCE_GROUP` | Org or repo | Resource group name (`rythmify-rg`) | +| `AZURE_SUBSCRIPTION_ID` | Org or repo | Azure subscription ID | +| `POSTGRES_PASSWORD` | Production env | PostgreSQL password for `rythmify_admin` | +| `ALERTMANAGER_SMTP_PASSWORD` | Production env | Gmail App Password for Alertmanager email alerts | +| `GRAFANA_ADMIN_PASSWORD` | Production env | Grafana admin login password | -# Stop all services -docker-compose -f docker/docker-compose.yml down - -# View logs for a specific service -docker-compose -f docker/docker-compose.yml logs -f backend - -# Rebuild a single service after code changes -docker-compose -f docker/docker-compose.yml up --build backend - -# Run tests locally (Linux/macOS) -bash scripts/test.sh - -# Run tests locally (Windows) -.\scripts\test.ps1 - -# Run build scripts (Linux/macOS) -bash scripts/build.sh - -# Run build scripts (Windows) -.\scripts\build.ps1 -``` - -### Troubleshooting - -| Issue | Solution | -| --------------------------- | --------------------------------------------------------------------------------- | -| Port already in use | Stop conflicting processes: `lsof -i :3000` or `lsof -i :8000`, then `kill ` | -| Docker daemon not running | Start Docker Desktop or run `sudo systemctl start docker` | -| `.env` file missing | Copy `.env.example`: `cp environments/.env.example environments/dev.env` | -| Container fails to start | Check logs: `docker-compose logs ` | -| Database connection refused | Ensure DB container is healthy: `docker ps` and check the `STATUS` column | - ---- - -## Contributing - -1. Create a feature branch from `develop`: `git checkout -b feature/your-feature` -2. Make your changes and test locally using the scripts in `scripts/`. -3. Push your branch and open a Pull Request β€” CI pipelines will run automatically. -4. Request a review from the relevant `CODEOWNERS`. -5. Merge only after CI passes and reviews are approved. +To add a secret: **GitHub repo β†’ Settings β†’ Secrets and variables β†’ Actions β†’ New repository secret** (or org-level for shared secrets). --- -## License - -This project is developed for academic purposes as part of a university software engineering course. - ---- +## Links -_Maintained by the Rythmify DevOps Team._ +- [Nginx config guide](nginx/README.md) β€” GeoIP setup, stub_status on 172.17.0.1:8081, Grafana proxy, SSL, deployment +- [Monitoring stack guide](monitoring/README.md) β€” Prometheus, Grafana, Alertmanager, alert rules, env vars +- [Local development setup](docs/setup-guide.md) β€” first-time setup, scripts reference, blob containers +- [VM operations guide](docs/vm-guide.md) β€” SSH access, common commands, security notes diff --git a/docs/vm-guide.md b/docs/vm-guide.md new file mode 100644 index 0000000..ad0297f --- /dev/null +++ b/docs/vm-guide.md @@ -0,0 +1,198 @@ +# Rythmify VM Operations Guide + +Practical reference for navigating and operating the two Azure VMs that run the Rythmify platform. + +--- + +## Overview + +Two VMs in **Azure UAE North**, connected by a shared VNet (private subnet `10.0.1.0/24`). + +| | VM1 β€” App | VM2 β€” Database | +|---|---|---| +| **Name** | `rythmify-app-vm` | `rythmify-db-vm` | +| **Public IP** | `20.196.3.253` | `20.233.118.212` | +| **Private IP** | `10.0.1.5` (VNet) | `10.0.1.4` | +| **Size** | B2s_v2 | B2als_v2 | +| **Runs** | Backend container, Redis, full monitoring stack, Nginx | PostgreSQL in Docker | + +VM1 handles all public traffic. VM2 is database-only β€” PostgreSQL is not exposed publicly, only reachable from VM1 via the private VNet address `10.0.1.4:5432`. + +--- + +## SSH Access + +**VM1:** +```bash +ssh rythmify@20.196.3.253 -i rythmify-app-vm_key.pem +``` + +**VM2:** +```bash +ssh rythmify@20.233.118.212 -i rythmify-db-vm_key.pem +``` + +Never commit `.pem` key files to the repository. Store them locally and keep permissions restricted: + +```bash +chmod 400 rythmify-app-vm_key.pem +chmod 400 rythmify-db-vm_key.pem +``` + +--- + +## VM1 Folder Structure + +| Path | Description | +|---|---| +| `~/devops/` | Cloned DevOps repo β€” all container configs, compose files, and nginx configs live here | +| `~/devops/monitoring/.env.monitoring` | Monitoring stack secrets β€” never committed to git | +| `/etc/nginx/sites-available/rythmify-back.duckdns.org.conf` | Active backend Nginx config | +| `/etc/nginx/sites-available/rythmify.duckdns.org.conf` | Active frontend Nginx config | +| `/var/lib/GeoIP/` | MaxMind GeoLite2-Country database (`GeoLite2-Country.mmdb`) | +| `/var/www/rythmify/` | Frontend static files served by Nginx (if applicable) | + +--- + +## Common VM1 Commands + +### Container management + +```bash +# Show all running containers with status and ports +docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" + +# View backend logs (last 50 lines) +docker logs rythmify-backend --tail 50 + +# Follow backend logs live +docker logs rythmify-backend -f + +# Restart only the backend container +docker compose -f docker-compose.yml up -d --force-recreate backend + +# Restart the full monitoring stack +docker compose -f docker-compose.monitoring.yml --env-file monitoring/.env.monitoring up -d + +# Pull latest repo changes +cd ~/devops && git pull +``` + +### Nginx + +```bash +# Test config syntax before applying +sudo nginx -t + +# Reload Nginx (applies config changes with zero downtime) +sudo systemctl reload nginx + +# Check Nginx status +sudo systemctl status nginx +``` + +### GeoIP + +```bash +# Update the MaxMind GeoLite2-Country database +sudo geoipupdate + +# Reload Nginx after database update +sudo systemctl reload nginx +``` + +--- + +## Common VM2 Commands + +```bash +# Check PostgreSQL container is running +docker ps + +# View PostgreSQL logs (last 50 lines) +docker logs --tail 50 + +# Connect to the database +docker exec -it psql -U rythmify_admin -d rythmify_db + +# Grant pg_monitor role (required for postgres_exporter on VM1) +docker exec -it psql -U rythmify_admin -d rythmify_db \ + -c "GRANT pg_monitor TO rythmify_admin;" +``` + +Replace `` with the actual container name shown in `docker ps`. + +--- + +## Monitoring Quick Checks + +Run these from VM1 to verify the monitoring stack is healthy. + +```bash +# All Prometheus targets β€” look for "health":"up" for each +curl -s http://localhost:9090/api/v1/targets | python3 -m json.tool | grep health + +# Postgres exporter connected (should return pg_up 1) +curl -s http://localhost:9187/metrics | grep pg_up + +# Blackbox probe success for backend health endpoint (should return probe_success 1) +curl -s "http://localhost:9115/probe?target=https://rythmify-back.duckdns.org/health&module=https_2xx" \ + | grep probe_success + +# Nginx stub_status β€” confirms nginx_exporter can reach the endpoint +curl -s http://172.17.0.1:8081/nginx_status + +# Grafana β€” open in browser (no SSH tunnel needed) +# https://rythmify-back.duckdns.org/grafana/ +``` + +--- + +## Security Notes + +**Azure NSG (Network Security Group) β€” VM1 inbound rules:** + +| Port | Protocol | Access | +|---|---|---| +| 80 | TCP | Public β€” HTTP (redirected to HTTPS by Nginx) | +| 443 | TCP | Public β€” HTTPS | +| 22 | TCP | Restricted β€” SSH (limit to known IPs if possible) | +| All others | β€” | Denied | + +Grafana (3000), Prometheus (9090), Alertmanager (9093), Redis, and the backend (8080) are internal only β€” they are **not** in the NSG allow list and are only accessible via Nginx proxy or SSH tunnel. + +Never add monitoring ports (3000, 9090, 9093) to NSG public inbound rules. + +--- + +## Updating the Nginx Config + +The active Nginx config lives on the VM at `/etc/nginx/sites-available/`. The repo in `~/devops/nginx/` is the source of truth β€” always edit the repo file, then copy to the system path. + +**Workflow:** + +```bash +# 1. Pull latest repo changes +cd ~/devops && git pull + +# 2. Copy updated configs to sites-available +sudo cp nginx/rythmify-back.duckdns.org.conf /etc/nginx/sites-available/ +sudo cp nginx/rythmify.duckdns.org.conf /etc/nginx/sites-available/ + +# 3. Test syntax +sudo nginx -t + +# 4. Reload +sudo systemctl reload nginx +``` + +If you make a hotfix directly on the VM, copy it back to the repo and commit: + +```bash +cp /etc/nginx/sites-available/rythmify-back.duckdns.org.conf ~/devops/nginx/ +cp /etc/nginx/sites-available/rythmify.duckdns.org.conf ~/devops/nginx/ +cd ~/devops +git add nginx/ +git commit -m "fix(nginx): " +git push +``` diff --git a/monitoring/README.md b/monitoring/README.md index 6d263e0..81efb3e 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -1,122 +1,225 @@ -# Rythmify Monitoring +# Rythmify Monitoring Stack -This stack runs Prometheus, Grafana, node_exporter, nginx_exporter, and blackbox_exporter on the Dir 2 VM. +## Overview -## Architecture +Full observability stack for the Rythmify platform running on VM1 (`rythmify-app-vm`). Covers backend API health, system resources, database connectivity, public uptime, and Nginx traffic β€” with email alerting via Gmail when anything goes wrong. + +Components: **Prometheus** Β· **Grafana** Β· **Alertmanager** Β· **Blackbox Exporter** Β· **Nginx Exporter** Β· **Postgres Exporter** Β· **Node Exporter** -- Prometheus scrapes every 15 seconds. -- Backend app metrics are scraped from `host.docker.internal:8080/metrics`. -- VM CPU, memory, and disk metrics come from `node_exporter`. -- Nginx metrics come from `nginx_exporter`, which reads `http://host.docker.internal/nginx_status`. -- Public uptime checks are run through `blackbox_exporter` for: - - `http://rythmify.duckdns.org/` - - `http://rythmify-back.duckdns.org/` - - `http://rythmify-back.duckdns.org/metrics` -- Grafana provisions Prometheus as the default data source and email alerts for core health signals. +--- -## VM Access +## Architecture -```powershell -ssh -i "C:\Users\alaba\OneDrive\Desktop\rythmify-app-vm_key.pem" rythmify@20.196.3.253 +``` + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ VM1 β€” Docker monitoring stack β”‚ + β”‚ β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ + β”‚ Backend │◄─────┼──│ Prometheus│───►│ Grafana β”‚ β”‚ + β”‚ :8080/metricsβ”‚ β”‚ β”‚ :9090 β”‚ β”‚ :3000 β”‚ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β””β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β”‚ /grafana/ proxy β”‚ β”‚ + β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ + β”‚ Node Exporter│◄─────┼───────── β”‚ Alertmanager β”‚ β”‚ + β”‚ :9100 β”‚ β”‚ β”‚ β”‚ :9093 β”‚ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β”‚ Gmail SMTP β”‚ β”‚ + β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ β”‚ β”‚ + β”‚ Nginx │◄─────┼───────── β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ + β”‚ stub_status β”‚ β”‚ β”‚ β”‚ Blackbox Exporter :9115 β”‚ β”‚ + β”‚ 172.17.0.1 β”‚ β”‚ β”œβ”€β”€β”€ probes public HTTPS URLs β”‚ β”‚ + β”‚ :8081 β”‚ β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ β”‚ + β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ └─── Postgres Exporter :9187 β”‚ β”‚ + β”‚ PostgreSQL │◄─────┼──────────── connects to VM2 :5432 β”‚ β”‚ + β”‚ VM2:5432 β”‚ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ ``` -## Nginx stub_status +--- -Add this location to the Nginx server block that listens on localhost: +## Stack -```nginx -location /nginx_status { - stub_status; - allow 127.0.0.1; - allow 172.16.0.0/12; - allow 172.17.0.0/16; - deny all; -} -``` +### Prometheus -Validate and reload Nginx: +Scrapes all targets every 15 seconds. Evaluates alert rules from `prometheus/alert_rules.yml`. -```bash -sudo nginx -t -sudo systemctl reload nginx -curl http://localhost/nginx_status -``` +Targets: -## Environment +| Job | Target | What it scrapes | +|---|---|---| +| `backend_metrics` | `host.docker.internal:8080/metrics` | Backend API metrics | +| `node_exporter` | `node_exporter:9100` | VM1 CPU, memory, disk | +| `nginx` | `nginx-exporter:9113` | Nginx connection stats | +| `postgres_exporter` | `postgres_exporter:9187` | PostgreSQL health and query stats | +| `blackbox` | `blackbox_exporter:9115` | Public HTTPS probes | +| `prometheus` | `prometheus:9090` | Prometheus self-metrics | -Create a runtime environment file from the example: +### Grafana -```bash -cp monitoring/.env.monitoring.example .env.monitoring -``` +Dashboards at: **https://rythmify-back.duckdns.org/grafana/** -Edit `.env.monitoring` and replace `SMTP_PASSWORD` with a Gmail app password. Do not commit `.env.monitoring`. +Login with `GRAFANA_ADMIN_PASSWORD` from `monitoring/.env.monitoring`. No SSH tunnel needed β€” Nginx proxies `/grafana/` to port 3000. -## Docker Commands +Provisioned dashboards: -Start the stack: +| Dashboard | What it shows | +|---|---| +| Backend API | Request rate, error rate, latency, active connections | +| PostgreSQL Health | DB connections, query duration, rows fetched/inserted | +| Uptime & Availability | Blackbox probe success, TLS certificate expiry | +| Node Exporter Full | VM CPU, memory, disk I/O, network | +| NGINX Exporter | Active connections, requests per second, handled vs accepted | -```bash -docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml up -d -``` +### Alertmanager -Check containers: +Sends email alerts via Gmail SMTP (port 587, STARTTLS). Recipient is configured in `ALERT_EMAIL_TO` in `.env.monitoring`. -```bash -docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml ps +See [Alerting](#alerting) for the full list of alert rules. + +### Blackbox Exporter + +Probes the following public HTTPS endpoints every 15 seconds (via Prometheus relabeling): + +- `https://rythmify-back.duckdns.org/health` β€” backend liveness +- `https://rythmify.duckdns.org` β€” frontend availability + +A probe is considered failed when the HTTP response is not 2xx or when TLS handshake fails. + +### Postgres Exporter + +Connects to VM2 at `10.0.1.4:5432` as `rythmify_admin`. The `pg_monitor` role must be granted: + +```sql +GRANT pg_monitor TO rythmify_admin; ``` -View logs: +Exposes PostgreSQL metrics at `:9187/metrics`, scraped by Prometheus. + +### Nginx Exporter + +Scrapes Nginx stub_status at `http://172.17.0.1:8081/nginx_status`. This endpoint is defined in `nginx/rythmify-back.duckdns.org.conf` as a dedicated server block bound to the Docker bridge host IP β€” reachable from Docker containers but not from the public internet. + +Verify the endpoint is reachable from the VM: ```bash -docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml logs -f prometheus grafana +curl -s http://172.17.0.1:8081/nginx_status ``` -Restart after config changes: +### Node Exporter + +Collects VM1 system metrics: CPU usage, memory, disk space, disk I/O, and network traffic. Runs as a Docker container and exports on port 9100. + +--- + +## Setup + +**1. Copy the example env file** ```bash -docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml restart prometheus grafana +cp monitoring/.env.monitoring.example monitoring/.env.monitoring ``` -Stop the stack: +**2. Fill in all values** + +Edit `monitoring/.env.monitoring`. Required values: + +- `GRAFANA_ADMIN_PASSWORD` β€” Grafana admin login password +- `SMTP_USER` / `SMTP_PASSWORD` β€” Gmail account and App Password for Grafana SMTP +- `GF_SMTP_FROM_ADDRESS` β€” sender address shown in alert emails +- `ALERT_EMAIL_TO` β€” recipient for all alert emails +- `ALERTMANAGER_SMTP_PASSWORD` β€” Gmail App Password used by Alertmanager directly +- `POSTGRES_PASSWORD` β€” `rythmify_admin` database password + +Do not commit `monitoring/.env.monitoring` to git. + +**3. Start the stack** ```bash -docker compose --env-file .env.monitoring -f docker-compose.monitoring.yml down +docker compose -f docker-compose.monitoring.yml --env-file monitoring/.env.monitoring up -d ``` -## Verification +**4. Verify** ```bash -curl http://localhost:9090/-/ready -curl http://localhost:9115/-/healthy -curl http://localhost/nginx_status +# All containers running +docker ps + +# All Prometheus targets healthy curl http://localhost:9090/api/v1/targets ``` -Open Grafana: +--- -```text -http://20.196.3.253:3000 -``` +## Environment Variables + +All variables are defined in `monitoring/.env.monitoring.example`: + +| Variable | Description | +|---|---| +| `GRAFANA_ADMIN_USER` | Grafana admin username (default: `admin`) | +| `GRAFANA_ADMIN_PASSWORD` | Grafana admin password | +| `GF_SMTP_ENABLED` | Enable Grafana SMTP (set to `true`) | +| `SMTP_HOST` | SMTP server and port (e.g. `smtp.gmail.com:587`) | +| `SMTP_USER` | Gmail address used as sender | +| `SMTP_PASSWORD` | Gmail App Password for Grafana SMTP | +| `GF_SMTP_FROM_ADDRESS` | From address in Grafana alert emails | +| `GF_SMTP_FROM_NAME` | From name in Grafana alert emails | +| `ALERT_EMAIL_TO` | Email address that receives alerts | +| `ALERTMANAGER_SMTP_PASSWORD` | Gmail App Password for Alertmanager | +| `POSTGRES_PASSWORD` | PostgreSQL password for `rythmify_admin` | + +Generate a Gmail App Password at https://myaccount.google.com/apppasswords. Do not use special characters (`/`, `&`, `@`, `$`) in the Alertmanager password β€” `envsubst` will break on them. Gmail App Passwords (16 lowercase letters) are safe by default. + +--- -Default credentials are configured through `.env.monitoring`. +## Alerting -## Alerts +Seven alert rules are defined in `monitoring/prometheus/alert_rules.yml`: -Grafana provisions alerts for: +| Alert | Condition | Severity | Fires after | +|---|---|---|---| +| `BackendDown` | `up{job="backend_metrics"} == 0` | critical | 2 min | +| `PostgresDown` | `pg_up == 0` | critical | 2 min | +| `PublicEndpointDown` | `probe_success{job="blackbox"} == 0` | critical | 2 min | +| `HighErrorRate` | HTTP 4xx/5xx rate > 5% of total requests | warning | 5 min | +| `HighMemoryUsage` | Available memory < 15% of total | warning | 5 min | +| `HighCPUUsage` | CPU idle < 15% (i.e. usage > 85%) | warning | 5 min | +| `DiskSpaceLow` | Disk usage > 85% on any non-tmpfs filesystem | warning | 5 min | -- backend down -- high CPU -- high memory -- disk full -- high backend error rate -- high backend latency -- public endpoint down +All alerts are routed through Alertmanager to the email address in `ALERT_EMAIL_TO`. + +--- + +## Accessing Grafana + +**URL:** https://rythmify-back.duckdns.org/grafana/ + +Nginx on VM1 proxies `/grafana/` to Grafana on port 3000. No SSH tunnel or port forwarding is needed β€” the proxy is already configured in `nginx/rythmify-back.duckdns.org.conf`. + +Login credentials: +- **Username:** value of `GRAFANA_ADMIN_USER` in `.env.monitoring` (default: `admin`) +- **Password:** value of `GRAFANA_ADMIN_PASSWORD` in `.env.monitoring` + +--- ## Troubleshooting -- If `backend_metrics` is down, verify the backend is listening on VM port `8080` and `curl http://localhost:8080/metrics` works on the VM. -- If `nginx_exporter` is down, verify `curl http://localhost/nginx_status` works on the VM and that the Nginx allow list includes Docker bridge ranges. -- If blackbox targets are down, run `curl -I` against each public URL from the VM. -- If Grafana email alerts do not send, verify `GF_SMTP_ENABLED=true`, `SMTP_HOST`, `SMTP_USER`, and the app password in `.env.monitoring`. -- If Prometheus cannot resolve `host.docker.internal`, confirm the compose services include `extra_hosts: host.docker.internal:host-gateway` and the host has a recent Docker Engine. +**`backend_metrics` target is down** +Verify the backend container is running and listening: `curl http://localhost:8080/metrics` on VM1. + +**`nginx` target is down** +Check that the stub_status endpoint is reachable: `curl -s http://172.17.0.1:8081/nginx_status`. If it returns nothing, confirm Nginx is running (`sudo systemctl status nginx`) and the `listen 172.17.0.1:8081` block is active in the Nginx config. + +**`postgres_exporter` target is down** +Verify VM2 is reachable from VM1: `nc -zv 10.0.1.4 5432`. Confirm `rythmify_admin` has the `pg_monitor` role. + +**Blackbox targets are down** +Run `curl -I https://rythmify-back.duckdns.org/health` from VM1. Check that the SSL certificate is valid and not expired (`sudo certbot certificates`). + +**Grafana email alerts not sending** +Verify `GF_SMTP_ENABLED=true`, `SMTP_HOST=smtp.gmail.com:587`, `SMTP_USER`, and `SMTP_PASSWORD` in `.env.monitoring`. Use an App Password, not your Google account password. + +**`host.docker.internal` not resolving in Prometheus** +Confirm `docker-compose.monitoring.yml` includes `extra_hosts: - "host.docker.internal:host-gateway"` on the Prometheus service. diff --git a/nginx/README.md b/nginx/README.md index 2b2554b..2a25290 100644 --- a/nginx/README.md +++ b/nginx/README.md @@ -6,10 +6,15 @@ This folder contains the Nginx virtual host configuration files for **VM1** (`ry | File | Domain | Purpose | |---|---|---| -| `rythmify-back.duckdns.org.conf` | `rythmify-back.duckdns.org` | Backend API + Socket.IO proxy, GeoIP2 country injection, HTTPS | -| `rythmify.duckdns.org.conf` | `rythmify.duckdns.org` | Frontend Vite dev server proxy, HTTPS, Nginx stub_status for Prometheus | +| `rythmify-back.duckdns.org.conf` | `rythmify-back.duckdns.org` | Backend API + Socket.IO proxy, GeoIP2 country injection, HTTPS, Grafana proxy, Nginx stub_status | +| `rythmify.duckdns.org.conf` | `rythmify.duckdns.org` | Frontend Vite dev server proxy, HTTPS | -The backend config also defines a `geoip2` directive (above its `server {}` block, in the `http {}` context) that resolves every client IP to a country code and injects it as `X-Country-Code` on all proxied requests. +The backend config contains three server blocks: +- **HTTP β†’ HTTPS redirect** on port 80 +- **HTTPS backend** on port 443 β€” proxies `/health`, `/grafana/`, and all API + WebSocket traffic to `localhost:8080` / `localhost:3000` +- **stub_status** on `172.17.0.1:8081` β€” scraped by `nginx_exporter` inside Docker + +The `geoip2` directive (above all `server {}` blocks, in the `http {}` context) resolves every client IP to a country code and injects it as `X-Country-Code` on all proxied requests to the backend. --- @@ -90,7 +95,7 @@ sudo ln -s /etc/nginx/sites-available/rythmify.duckdns.org.conf \ /etc/nginx/sites-enabled/ ``` -If the default Nginx site is enabled and conflicts with the `default_server` block in `rythmify.duckdns.org.conf`, disable it: +Disable the default Nginx site if it is enabled (it conflicts with the new configs): ```bash sudo rm -f /etc/nginx/sites-enabled/default @@ -140,6 +145,68 @@ Because the header is set by Nginx at the proxy layer, it **cannot be spoofed** --- +## Grafana Proxy + +Grafana runs inside Docker on port `3000`. It is not exposed publicly. Instead, the backend server block proxies `/grafana/` to `http://localhost:3000/`: + +```nginx +location /grafana/ { + proxy_pass http://localhost:3000/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; +} +``` + +Access Grafana at: **https://rythmify-back.duckdns.org/grafana/** + +No SSH tunnel or port forwarding is needed. + +--- + +## Nginx stub_status + +`nginx_exporter` (in `docker-compose.monitoring.yml`) scrapes Nginx connection metrics from a dedicated `stub_status` endpoint. The endpoint is defined in `rythmify-back.duckdns.org.conf` as a server block bound to the Docker bridge host IP: + +```nginx +server { + listen 172.17.0.1:8081; + server_name _; + + location /nginx_status { + stub_status; + allow 127.0.0.1; + allow 172.16.0.0/12; + deny all; + } + + location / { + return 444; + } +} +``` + +`172.17.0.1` is the Docker bridge host IP β€” it is reachable from Docker containers but not from the public internet or from the VM loopback. `nginx_exporter` in the monitoring compose stack is configured to scrape `http://172.17.0.1:8081/nginx_status`. + +Verify from the VM: + +```bash +curl -s http://172.17.0.1:8081/nginx_status +``` + +Expected output: + +``` +Active connections: 3 +server accepts handled requests + 12 12 24 +Reading: 0 Writing: 1 Waiting: 2 +``` + +--- + ## SSL SSL certificates are managed by **Certbot** and must **not** be committed to this repository. @@ -184,9 +251,3 @@ To automate this, create a cron job at `/etc/cron.d/geoipupdate`: # Refresh MaxMind GeoLite2-Country on the 1st and 15th of each month at 03:00 0 3 1,15 * * root /usr/bin/geoipupdate && systemctl reload nginx ``` - -After adding the file, verify cron picks it up: - -```bash -sudo crontab -l -``` diff --git a/nginx/rythmify-back.duckdns.org.conf b/nginx/rythmify-back.duckdns.org.conf index 172682b..b79eb2c 100644 --- a/nginx/rythmify-back.duckdns.org.conf +++ b/nginx/rythmify-back.duckdns.org.conf @@ -41,10 +41,11 @@ server { proxy_set_header X-Forwarded-Proto $scheme; } - # ── /grafana/ β€” Grafana dashboard (proxied from port 3000) ──────────────────── + # ── /grafana/ β€” Grafana dashboard (proxied from port 3000) ────────────────── location /grafana/ { proxy_pass http://localhost:3000/; proxy_http_version 1.1; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -72,3 +73,26 @@ server { proxy_cache_bypass $http_upgrade; } } + +# ── Nginx stub_status β€” for prometheus nginx_exporter ───────────────────────── +# +# Binds to the Docker bridge host IP (172.17.0.1) on a dedicated port 8081. +# nginx_exporter in docker-compose.monitoring.yml scrapes: +# http://172.17.0.1:8081/nginx_status +# Binding to 172.17.0.1 limits access to Docker bridge traffic β€” this address +# is not reachable from public interfaces or from the VM loopback (127.0.0.1). +server { + listen 172.17.0.1:8081; + server_name _; + + location /nginx_status { + stub_status; + allow 127.0.0.1; + allow 172.16.0.0/12; + deny all; + } + + location / { + return 444; + } +} diff --git a/nginx/rythmify.duckdns.org.conf b/nginx/rythmify.duckdns.org.conf index 415f645..a29da77 100644 --- a/nginx/rythmify.duckdns.org.conf +++ b/nginx/rythmify.duckdns.org.conf @@ -23,12 +23,6 @@ server { include /etc/letsencrypt/options-ssl-nginx.conf; ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; - location /nginx_status { - stub_status; - allow 127.0.0.1; - deny all; - } - location / { proxy_pass http://localhost:5173; proxy_http_version 1.1; @@ -45,32 +39,3 @@ server { proxy_cache_bypass $http_upgrade; } } - -# ── Nginx stub_status β€” for prometheus nginx_exporter ───────────────────────── -# -# devops/docker-compose.monitoring.yml nginx_exporter scrapes: -# http://host.docker.internal/nginx_status -# host.docker.internal resolves to the Docker host gateway (172.x.x.x), not -# 127.0.0.1, so this block listens on 0.0.0.0:80 as default_server and -# restricts access by source IP. -# -# Named server blocks above handle their own server_name directives. This -# default_server block catches requests whose Host header matches no other -# server_name (including host.docker.internal from the nginx_exporter container). -server { - listen 80 default_server; - listen [::]:80 default_server; - server_name _; - - location /nginx_status { - stub_status on; - allow 127.0.0.1; - allow ::1; - allow 172.16.0.0/12; - deny all; - } - - location / { - return 444; - } -}