From b307fcc58c4b9e6e7970e924993df58e7f0e3e38 Mon Sep 17 00:00:00 2001 From: Shaddi Hasan Date: Mon, 6 Jul 2026 20:04:47 -0400 Subject: [PATCH] deploy: maintenance-mode flag and a worker shutdown grace period MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Maintenance mode: touching /srv/bdk/maintenance/on on the web host makes nginx answer every new request with a plain 503 + Retry-After (checked per request — no reload; rm restores instantly). In-flight requests are unaffected. The flag lives outside the checkout on purpose: deploy.sh refuses a dirty tree and an untracked flag file would count as dirty. Verified end-to-end against a stub backend (200 -> touch -> 503 with body -> rm -> 200). Worker stop_grace_period 15m: Docker's 10s default SIGKILLed celery's warm shutdown mid-task, which would kill a minutes-long tile build or fabric import on every deploy that caught one in flight. With the queue drained first, shutdown remains instant. --- deploy/data/docker-compose.yml | 5 +++++ deploy/nginx.conf | 14 ++++++++++++++ deploy/web/docker-compose.yml | 4 ++++ 3 files changed, 23 insertions(+) diff --git a/deploy/data/docker-compose.yml b/deploy/data/docker-compose.yml index a1fe122..54cf6f8 100644 --- a/deploy/data/docker-compose.yml +++ b/deploy/data/docker-compose.yml @@ -37,6 +37,11 @@ services: restart: unless-stopped cpus: ${WORKER_CPUS} mem_limit: ${WORKER_MEM_LIMIT} + # On stop, celery's warm shutdown finishes the active task before exiting. + # Docker's default grace is 10s and then SIGKILLs — which would kill a + # minutes-long tile build or fabric import mid-flight. Give it real time; + # when the queue is drained first (deploy scripts), shutdown is instant. + stop_grace_period: 15m # tippecanoe opens temp files scaled to the host's core count plus # per-tile geometry files; on a many-core host a large tile build blows # through Docker's default soft limit of 1024 and dies with EMFILE diff --git a/deploy/nginx.conf b/deploy/nginx.conf index b48e1ae..c305b89 100644 --- a/deploy/nginx.conf +++ b/deploy/nginx.conf @@ -20,7 +20,14 @@ server { gzip on; gzip_types text/css application/javascript application/json application/vnd.mapbox-vector-tile; + # Maintenance mode: `touch /srv/bdk/maintenance/on` on the host 503s every + # new request instantly (the -f is checked per request — no reload); + # `rm -f` restores. In-flight requests are unaffected. Used to quiesce the + # site before a deploy so no new work reaches the backend or the worker. location / { + if (-f /maintenance/on) { + return 503; + } proxy_pass http://backend:8000; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -32,4 +39,11 @@ server { proxy_send_timeout 300s; proxy_connect_timeout 60s; } + + error_page 503 @maintenance; + location @maintenance { + default_type text/plain; + add_header Retry-After 300 always; + return 503 "BDK is down for maintenance and will be back shortly."; + } } diff --git a/deploy/web/docker-compose.yml b/deploy/web/docker-compose.yml index aa59e99..de580f7 100644 --- a/deploy/web/docker-compose.yml +++ b/deploy/web/docker-compose.yml @@ -49,5 +49,9 @@ services: - ../nginx.conf:/etc/nginx/templates/default.conf.template:ro - ${SSL_CERT_PATH}:/etc/nginx/ssl/server.crt:ro - ${SSL_KEY_PATH}:/etc/nginx/ssl/server.key:ro + # Maintenance flag dir — OUTSIDE the checkout on purpose: deploy.sh + # refuses a dirty tree, and an untracked flag file would count as dirty. + # `touch /srv/bdk/maintenance/on` = down for maintenance; `rm -f` = up. + - /srv/bdk/maintenance:/maintenance:ro environment: - SERVER_NAME=${SERVER_NAME}