Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 46 additions & 18 deletions .github/workflows/docker_push.yml
Original file line number Diff line number Diff line change
@@ -1,29 +1,57 @@
name: Build and Push Docker Images to GitHub Registry
name: Build and push images

# on:
# push:
# branches:
# - main
# Build the backend and worker images and push them to ghcr, tagged by commit
# SHA, on every green main build (and on demand). Deploys then PULL a tagged
# image — production never builds on the host. Building an image for a commit
# that is never deployed is cheap and harmless.

on:
workflow_run:
workflows: ["CI"]
types: [completed]
branches: [main]
workflow_dispatch:

permissions:
contents: read
packages: write

jobs:
build-and-push:
build-push:
# Only build images for commits that passed CI.
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
runs-on: ubuntu-latest

env:
SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
steps:
- name: Check out the repo
uses: actions/checkout@v2
- uses: actions/checkout@v4
with:
ref: ${{ github.event.workflow_run.head_sha || github.sha }}

- uses: docker/setup-buildx-action@v3

- name: Login to GitHub Docker Registry
run: echo "${{secrets.DOCKER_LOGIN_TOKEN}}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Build Docker images using docker-compose.build.yml
run: docker-compose -f docker-compose.build.yml build
- name: backend
uses: docker/build-push-action@v6
with:
context: ./back-end
file: ./back-end/Dockerfile.backend
push: true
tags: |
ghcr.io/spin-vt/backend:${{ env.SHA }}
ghcr.io/spin-vt/backend:latest

- name: Push Docker images to registry
run: |
docker push ghcr.io/spin-vt/worker:latest
docker push ghcr.io/spin-vt/backend:latest
docker push ghcr.io/spin-vt/frontend:latest
- name: worker
uses: docker/build-push-action@v6
with:
context: ./back-end
file: ./back-end/Dockerfile.worker
push: true
tags: |
ghcr.io/spin-vt/worker:${{ env.SHA }}
ghcr.io/spin-vt/worker:latest
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,14 @@ front-end/Map_marker.svg:Zone.Identifier

pushgateway-*/
profiling_data/
data/
/data/
deploy/*/pgdata/
.env.local
helpful.txt
.env.*
!.env.example
README.md
!deploy/README.md

# Real ISP data for local dev/test — never commit (sensitive FCC fabric/location data)
dev-data/
Expand Down
7 changes: 7 additions & 0 deletions back-end/Dockerfile.backend
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@ WORKDIR /app
COPY pyproject.toml uv.lock ./
RUN uv sync --frozen --no-dev

# Bake the application code into the image (as Dockerfile.worker does) so a
# backend:<sha> image is a self-contained artifact: production pulls it from
# the registry; code is never built or bind-mounted on the host. The dev
# compose still bind-mounts ./back-end over /app for live reload (the venv at
# /opt/venv is outside /app, so neither path hides it).
COPY . .

COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh

Expand Down
55 changes: 55 additions & 0 deletions back-end/scripts/legacy_data_policy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
"""One-off data policy for upgrading a legacy (pre-filing-model) database,
run once after `alembic upgrade head` (see deploy/restore-and-migrate.sh).
Idempotent — safe to re-run; prints one line per decision.

1. Upload filings whose filing window is already past are marked filed:
under the old model "filed" didn't exist, and anything from a closed
window was necessarily submitted (or abandoned) long ago. Filings in the
open or a future window stay open.
2. Service plans are synthesized for every upload filing from its files'
legacy speed columns (plan_service.synthesize_legacy_plans): one plan per
distinct tech/down/up/latency/category combination, the first per
technology marked the tech default, files attached. Export snapshots stay
plan-less — they are frozen history and resolution falls back to file
columns.
"""

from datetime import date, datetime

from database.models import folder
from database.sessions import Session
from services import filing_windows, plan_service


def main():
session = Session()
now = date.today()
uploads = session.query(folder).filter(folder.type == "upload").order_by(folder.id).all()
for f in uploads:
w = filing_windows.window_from_deadline(f.deadline)
if f.status == "filed":
print(f"folder {f.id} ({w.label}): already filed")
elif w.due < now:
f.status = "filed"
if f.filed_at is None:
f.filed_at = datetime.combine(w.due, datetime.min.time())
print(f"folder {f.id} ({w.label}, was due {w.due}): marked filed")
else:
print(f"folder {f.id} ({w.label}, due {w.due}): left open")

plans = plan_service.synthesize_legacy_plans(f.id, session)
if plans:
for p in plans:
print(
f" plan: {p.name} (tech {p.tech_code}, "
f"{p.max_download}/{p.max_upload}, "
f"{'default' if p.is_default else 'extra'})"
)
else:
print(" plans: already present")
session.commit()
session.close()


if __name__ == "__main__":
main()
37 changes: 37 additions & 0 deletions back-end/tests/test_legacy_data_policy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
"""The legacy upgrade data policy (scripts/legacy_data_policy.py): filings
from past windows get marked filed, open-window filings stay open, and every
upload filing gets plans synthesized from its files' legacy columns.
Idempotent. (Plan synthesis itself is pinned in test_services_plans.)"""

from datetime import date

from database.models import service_plan
from scripts.legacy_data_policy import main as run_policy
from tests import conftest_helpers as H


def test_policy_files_past_windows_and_synthesizes_plans(db_session, monkeypatch):
s = db_session
org = H.make_org(s)
past = H.make_folder(s, org.id, name="old", deadline=date(2024, 9, 3))
open_now = H.make_folder(s, org.id, name="current", deadline=date(2026, 9, 1))
H.seed_coverage(s, past.id, b"x", "cov.kml", "wired", 50, maxDownloadSpeed=100)
H.seed_coverage(s, open_now.id, b"x", "cov2.kml", "wired", 50, maxDownloadSpeed=200)
s.commit()

# The script opens its own Session; run against the test DB as-is.
run_policy()
s.expire_all()

assert past.status == "filed" and past.filed_at is not None
assert open_now.status == "open"
plans = {p.folder_id: p for p in s.query(service_plan).all()}
assert plans[past.id].max_download == 100 and plans[past.id].is_default
assert plans[open_now.id].max_download == 200

# Idempotent: a second run changes nothing.
filed_at = past.filed_at
run_policy()
s.expire_all()
assert past.filed_at == filed_at
assert s.query(service_plan).count() == 2
52 changes: 52 additions & 0 deletions deploy/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
# Deploying BDK

Production runs on two hosts, each with its own compose file in this
directory:

- **web/** — the public-facing box: nginx (TLS termination) and the Flask
backend under gunicorn. Stateless; everything it serves comes from the
data host.
- **data/** — the data box: PostgreSQL 16 (PostGIS-capable image, extension
off until needed), Redis (Celery broker + coordination flags), and the
Celery worker + beat, colocated with the database because the heavy
byte-flows are worker↔database.

## The model

CI builds the backend and worker images on every green `main` build and
pushes them to ghcr tagged by commit SHA
(`.github/workflows/docker_push.yml`). **Nothing is built on the hosts** —
they pull the tagged image.

Each host carries a git checkout of this repository at a **pinned commit**
(for the versioned compose files / nginx.conf), owned by the deploy user and
never hand-edited (`deploy.sh` refuses a dirty tree). Deploying a version =
`deploy.sh <role> <git-sha>` on each host, which checks out the SHA, then
`docker compose pull && up -d` with `TAG` = that SHA. Rollback = the same
command with a previous SHA. The hosts need registry read access (a
`docker login ghcr.io` with a read-only token, or public packages).

## Secrets

Real configuration lives ONLY in a `.env` file beside each compose file
(`web/.env`, `data/.env`), created by hand from `env.example` and chmod 600.
It is never committed, never copied off the host, and rotated by editing the
file and re-running `docker compose up -d`. The database and Redis
passwords appear in both hosts' files; everything else is single-host.

## First-time bring-up / restoring from a dump

`restore-and-migrate.sh` builds a production database from a `pg_dump -Fc`
dump: fresh Postgres container, restore, `alembic upgrade head`, then
`legacy_data_policy.py` (marks past-window filings filed and synthesizes
service plans from legacy per-file values — idempotent, safe on every run).
A from-scratch install skips the restore and just lets the entrypoint run
migrations against the empty database.

## Backups

A nightly cron on the data host (see DEPLOYMENT notes; not part of compose):

pg_dump -Fc -h localhost -U bdk bdk > /backups/bdk-$(date +%F).dump

Keep at least 14 days and copy them off-host.
64 changes: 64 additions & 0 deletions deploy/data/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
# BDK production — DATA host: PostgreSQL 16 + Redis + Celery worker/beat,
# colocated (the heavy byte-flows are worker<->database). Code bind-mounted
# from this checkout (pinned by deploy.sh). Real config in ./\.env.
#
# PGDATA lives on local fast disk via ./pgdata — keep this directory on the
# NVMe volume. Resource caps come from .env (DB_CPUS etc.).
#
# docker compose --env-file .env up -d

services:
db:
# PostGIS-capable image, extension NOT created — enabling it later is a
# one-line migration instead of an image swap.
image: postgis/postgis:16-3.4
restart: unless-stopped
cpus: ${DB_CPUS}
mem_limit: ${DB_MEM_LIMIT}
environment:
- POSTGRES_USER=${POSTGRES_USER}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- POSTGRES_DB=${POSTGRES_DB}
volumes:
- ./pgdata:/var/lib/postgresql/data
ports:
- "5432:5432"

redis:
image: redis:7
restart: unless-stopped
command: redis-server --requirepass ${REDIS_PASSWORD}
ports:
- "6379:6379"

worker:
image: ghcr.io/spin-vt/worker:${TAG}
command: celery -A controllers.celery_controller.celery_config.celery worker -B --loglevel=INFO
restart: unless-stopped
cpus: ${WORKER_CPUS}
mem_limit: ${WORKER_MEM_LIMIT}
depends_on:
- db
- redis
environment:
- POSTGRES_USER=${POSTGRES_USER}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
- POSTGRES_DB=${POSTGRES_DB}
- DB_HOST=db
- DB_PORT=5432
- PYTHONPATH=/app
- SECRET_KEY=${SECRET_KEY}
- JWT_SECRET=${JWT_SECRET}
- JWT_TOKEN_LOCATION=${JWT_TOKEN_LOCATION}
- JWT_ACCESS_COOKIE_NAME=${JWT_ACCESS_COOKIE_NAME}
- IN_PRODUCTION=${IN_PRODUCTION}
- PUBLIC_BASE_URL=${PUBLIC_BASE_URL}
- MAIL_SERVER=${MAIL_SERVER}
- MAIL_PORT=${MAIL_PORT}
- MAIL_USE_TLS=${MAIL_USE_TLS}
- MAIL_USE_SSL=${MAIL_USE_SSL}
- MAIL_USERNAME=${MAIL_USERNAME}
- MAIL_PASSWORD=${MAIL_PASSWORD}
- MAIL_DEFAULT_SENDER=${MAIL_DEFAULT_SENDER}
- CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis:6379/0
- CELERY_RESULT_BACKEND=redis://:${REDIS_PASSWORD}@redis:6379/0
37 changes: 37 additions & 0 deletions deploy/deploy.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#!/bin/sh
# Deploy BDK on this host by pulling the CI-built images for a commit and
# restarting — nothing is built on the host. Images are built and pushed to
# ghcr by .github/workflows/docker_push.yml on every green main build, tagged
# by commit SHA.
#
# ./deploy.sh web <git-sha>
# ./deploy.sh data <git-sha>
#
# Rollback = the same command with a previous sha. The checkout is pinned to
# that sha too (so the compose files / nginx.conf match the images) and must
# be clean — production is never hand-edited.
set -eu

ROLE=${1:?usage: deploy.sh <web|data> <git-sha>}
SHA=${2:?usage: deploy.sh <web|data> <git-sha>}
DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
REPO=$(dirname "$DIR")

[ -f "$DIR/$ROLE/.env" ] || { echo "missing $DIR/$ROLE/.env (copy from env.example)"; exit 1; }

cd "$REPO"
if [ -n "$(git status --porcelain)" ]; then
echo "checkout is dirty — production is never hand-edited. Aborting."
git status --short
exit 1
fi
git fetch --all --quiet
git checkout --quiet "$SHA"
TAG=$(git rev-parse HEAD)
export TAG
echo "deploying $ROLE at $(git rev-parse --short HEAD) (image tag $TAG)"

cd "$DIR/$ROLE"
docker compose --env-file .env pull
docker compose --env-file .env up -d
docker compose --env-file .env ps
Loading
Loading