From f83df31745e1635481c4eeb40d4e41445fbc8b19 Mon Sep 17 00:00:00 2001 From: Matteo Date: Fri, 18 Sep 2026 16:03:33 +0200 Subject: [PATCH 1/2] fix(cloud): cap container logs, which filled the droplet's disk today MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker's json-file driver is unbounded by default and not one of the five cloud services overrode it. On 2026-09-18 MOTIS's debug log — it writes a line per unresolvable GTFS-RT trip, continuously — reached 32 GB and filled the 77 GB disk. The result was a silent production incident that had been running for most of a day: every container reported unhealthy, `docker exec` into Postgres failed with "no space left on device", and the connector MCP endpoints answered 503. It surfaced only because a deploy failed at the scp step, before touching anything, because nothing could be written to /tmp. Truncating the log freed the 32 GB and everything returned to healthy, but that is a fix with a ~30-hour shelf life at MOTIS's current rate. Every service now caps its logs at 50 MB × 5 files, so the worst case is 250 MB per container rather than the whole disk. MOTIS's log level is worth a separate look — a line per unresolved trip at debug level is a lot of noise to keep at all — but the cap is what stops one chatty container taking the droplet down. Co-Authored-By: Claude Opus 5 (1M context) --- docker-compose.cloud.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docker-compose.cloud.yml b/docker-compose.cloud.yml index 20b1d5c1..b00a966e 100644 --- a/docker-compose.cloud.yml +++ b/docker-compose.cloud.yml @@ -26,6 +26,16 @@ services: app: condition: service_healthy restart: unless-stopped + # Docker's json-file driver is unbounded by default. On 2026-09-18 MOTIS's + # debug log reached 32 GB and filled the droplet's 77 GB disk: every + # container went unhealthy, Postgres could not write, and a deploy failed + # at the scp step because nothing could be written to /tmp. Caps here, on + # every service, so one chatty container cannot do that again. + logging: + driver: json-file + options: + max-size: "50m" + max-file: "5" # NestJS Backend + Next.js Frontend (single container) app: @@ -131,6 +141,11 @@ services: # start_period elapses it still goes unhealthy and the deploy fails for real. start_period: 180s restart: unless-stopped + logging: + driver: json-file + options: + max-size: "50m" + max-file: "5" # PostgreSQL Database postgres: @@ -148,6 +163,11 @@ services: timeout: 3s retries: 5 restart: unless-stopped + logging: + driver: json-file + options: + max-size: "50m" + max-file: "5" # Redis Cache — caching & rate limiting redis: @@ -161,6 +181,11 @@ services: timeout: 3s retries: 5 restart: unless-stopped + logging: + driver: json-file + options: + max-size: "50m" + max-file: "5" # MOTIS — routing engine behind the Deutsche Bahn connector, fed by the open # gtfs.de timetable (CC BY 4.0) plus its GTFS-RT feed for live delays. @@ -197,6 +222,11 @@ services: # minutes); the margin is for a slow gtfs.de. start_period: 180s restart: unless-stopped + logging: + driver: json-file + options: + max-size: "50m" + max-file: "5" volumes: postgres_data: From 26e0d92918322a6a253c7f004bf2562b2982a192 Mon Sep 17 00:00:00 2001 From: Matteo Date: Sat, 19 Sep 2026 00:13:15 +0200 Subject: [PATCH 2/2] fix(motis): log at info, which is where 99.98% of the volume came from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capping the logs stops one container taking the disk down; it does not stop the writing. Measured on the droplet after the truncation: 19,995 of any 20,000 lines were the same debug message, and the log regrew 11 GB in seven and a half hours — 1.5 GB an hour, which refills the free space in about thirteen. The message is `rt.gtfs.resolve: could not resolve trip_id`, and it is unavoidable by construction rather than a symptom: the GTFS-RT feed carries every bus in Germany while the static datasets are deliberately rail-only (the full feed needs 7 GB of RAM to import, see config.yml), so nearly every trip in the realtime feed has nothing to resolve against. MOTIS logs one line per trip, per update_interval, forever. `--log-level info` drops those and keeps the [info] rt-update timings, which are the ones worth having. Overridable with MOTIS_LOG_LEVEL for when a routing problem genuinely needs debug — turn it back afterwards. Co-Authored-By: Claude Opus 5 (1M context) --- deploy/motis/entrypoint.sh | 8 +++++++- docker-compose.cloud.yml | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/deploy/motis/entrypoint.sh b/deploy/motis/entrypoint.sh index 9c619815..cea75d85 100755 --- a/deploy/motis/entrypoint.sh +++ b/deploy/motis/entrypoint.sh @@ -83,7 +83,13 @@ fi while :; do log "starting server on $CURRENT" - /motis server -d "$CURRENT" & + # --log-level info, not MOTIS's default debug. The GTFS-RT feed carries every + # bus in the country while the static datasets are rail only, so nearly every + # trip in it is unresolvable by construction and MOTIS logged a debug line + # per trip, every update_interval. Measured on the droplet: 19,995 of any + # 20,000 lines were that one message, 1.5 GB an hour, and on 2026-09-18 it + # filled the 77 GB disk. The [info] lines (rt update timings) are kept. + /motis server -d "$CURRENT" --log-level "${MOTIS_LOG_LEVEL:-info}" & pid=$! swap=0 while kill -0 "$pid" 2>/dev/null; do diff --git a/docker-compose.cloud.yml b/docker-compose.cloud.yml index b00a966e..3d8938e4 100644 --- a/docker-compose.cloud.yml +++ b/docker-compose.cloud.yml @@ -208,6 +208,10 @@ services: - motis_data:/data environment: - MOTIS_REFRESH_DAYS=${MOTIS_REFRESH_DAYS:-7} + # info, not MOTIS's default debug — see deploy/motis/entrypoint.sh. + # Set to debug temporarily when diagnosing a routing problem, then + # put it back: debug writes ~1.5 GB an hour. + - MOTIS_LOG_LEVEL=${MOTIS_LOG_LEVEL:-info} healthcheck: # 127.0.0.1, not localhost: MOTIS binds IPv4 only and localhost resolves # to ::1 first inside the container, so the probe was refused every time.