From 1f1c0db96c7f388ee51a5f83585a3c7a70ad8cba Mon Sep 17 00:00:00 2001 From: Marten Klitzke Date: Sun, 18 Oct 2026 03:38:24 +0200 Subject: [PATCH 1/5] feat(servers): set server types per web server and keep the disk on a rescale --- cloud.tf | 8 +++-- tests/multiple_servers.tftest.hcl | 48 +++++++++++++++++++++---- variables.tf | 60 ++++++++++++++++++------------- 3 files changed, 84 insertions(+), 32 deletions(-) diff --git a/cloud.tf b/cloud.tf index 15393fb..6b41a20 100644 --- a/cloud.tf +++ b/cloud.tf @@ -18,8 +18,11 @@ resource "hcloud_server" "web_server" { count = local.env.web_servers_count name = local.env.web_servers_count > 1 ? "fltyrd-${terraform.workspace}-web-${count.index + 1}" : "fltyrd-${terraform.workspace}-web" image = var.operating_system - server_type = local.env.server_type + server_type = local.env.web_server_types[count.index] location = var.region + # Rescale without growing the disk, which would rule out ever moving back + # to a smaller type. + keep_disk = true labels = { "ssh" = "yes" "http" = "yes" @@ -51,8 +54,9 @@ resource "hcloud_server" "accessory_server" { count = local.env.accessories_count name = local.env.accessories_count > 1 ? "fltyrd-${terraform.workspace}-accessories-${count.index + 1}" : "fltyrd-${terraform.workspace}-accessories" image = var.operating_system - server_type = local.env.server_type + server_type = local.env.accessories_server_type location = var.region + keep_disk = true labels = { "http" = "no" "ssh" = "yes" diff --git a/tests/multiple_servers.tftest.hcl b/tests/multiple_servers.tftest.hcl index fa5d3b2..0ff5958 100644 --- a/tests/multiple_servers.tftest.hcl +++ b/tests/multiple_servers.tftest.hcl @@ -6,12 +6,13 @@ mock_provider "onepassword" {} variables { env_config = { default = { - server_type = "cx23" - web_servers_count = 2 - accessories_count = 1 - domains = ["example.com"] - short_domains = ["ex.com"] - cors_origins = ["https://example.com"] + web_server_types = ["cx33", "cx23"] + accessories_server_type = "cx23" + web_servers_count = 2 + accessories_count = 1 + domains = ["example.com"] + short_domains = ["ex.com"] + cors_origins = ["https://example.com"] } } } @@ -24,6 +25,21 @@ run "create_servers" { error_message = "Server name is not correct" } + assert { + condition = hcloud_server.web_server.*.server_type == ["cx33", "cx23"] + error_message = "Each web server should take its own entry in web_server_types" + } + + assert { + condition = hcloud_server.accessory_server[0].server_type == "cx23" + error_message = "The accessories server should take accessories_server_type" + } + + assert { + condition = alltrue(concat(hcloud_server.web_server.*.keep_disk, hcloud_server.accessory_server.*.keep_disk)) + error_message = "A rescale must keep the disk so the type can be lowered again" + } + assert { condition = hcloud_load_balancer.web_load_balancer[0].name != null error_message = "Load balancer was not created" @@ -39,3 +55,23 @@ run "create_servers" { error_message = "Cloud-init config for fltyrd-web-2 is not correct" } } + +run "rejects_a_type_list_shorter_than_the_server_count" { + command = plan + + variables { + env_config = { + default = { + web_server_types = ["cx23"] + accessories_server_type = "cx23" + web_servers_count = 2 + accessories_count = 1 + domains = ["example.com"] + short_domains = ["ex.com"] + cors_origins = ["https://example.com"] + } + } + } + + expect_failures = [var.env_config] +} diff --git a/variables.tf b/variables.tf index 459f2e9..791387e 100644 --- a/variables.tf +++ b/variables.tf @@ -18,39 +18,51 @@ variable "operating_system" { variable "env_config" { description = "Per-environment server configuration, keyed by workspace name." type = map(object({ - server_type = string - web_servers_count = number - accessories_count = number - domains = list(string) - short_domains = list(string) - cors_origins = list(string) + web_server_types = list(string) + accessories_server_type = string + web_servers_count = number + accessories_count = number + domains = list(string) + short_domains = list(string) + cors_origins = list(string) })) default = { default = { - server_type = "cx23" - web_servers_count = 1 - accessories_count = 1 - domains = [] - short_domains = [] - cors_origins = ["http://fleetyards.test", "http://*.fleetyards.test"] + web_server_types = ["cx23"] + accessories_server_type = "cx23" + web_servers_count = 1 + accessories_count = 1 + domains = [] + short_domains = [] + cors_origins = ["http://fleetyards.test", "http://*.fleetyards.test"] } stage = { - server_type = "cx23" - web_servers_count = 0 - accessories_count = 0 - domains = ["fleetyards.dev"] - short_domains = ["fltyrd.dev"] - cors_origins = ["https://fleetyards.dev", "https://*.fleetyards.dev"] + web_server_types = [] + accessories_server_type = "cx23" + web_servers_count = 0 + accessories_count = 0 + domains = ["fleetyards.dev"] + short_domains = ["fltyrd.dev"] + cors_origins = ["https://fleetyards.dev", "https://*.fleetyards.dev"] } live = { - server_type = "cx33" - web_servers_count = 2 - accessories_count = 1 - domains = ["fleetyards.net"] - short_domains = ["fltyrd.net"] - cors_origins = ["https://fleetyards.net", "https://*.fleetyards.net"] + # One entry per web server, so a type change can roll out one server per + # apply: an apply rescales every server whose entry changed at once, and + # rescaling powers the server off. + web_server_types = ["cx43", "cx33"] + accessories_server_type = "cx33" + web_servers_count = 2 + accessories_count = 1 + domains = ["fleetyards.net"] + short_domains = ["fltyrd.net"] + cors_origins = ["https://fleetyards.net", "https://*.fleetyards.net"] } } + + validation { + condition = alltrue([for env in values(var.env_config) : length(env.web_server_types) == env.web_servers_count]) + error_message = "Each environment needs exactly one entry in web_server_types per web server." + } } variable "manage_dns" { From 60895060af24a088b1ede7468b0b55a1fed40829 Mon Sep 17 00:00:00 2001 From: Marten Klitzke Date: Sun, 18 Oct 2026 03:40:22 +0200 Subject: [PATCH 2/5] docs(maintenance): add the rolling web server rescale runbook --- MAINTENANCE.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/MAINTENANCE.md b/MAINTENANCE.md index bae15c6..77e50dc 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -170,6 +170,34 @@ ssh -J kamal@ kamal@ "sudo reboot" ssh -J kamal@ kamal@ "docker ps" ``` +## Rescaling Web Servers (Live) + +A server type change is an in-place update, but Hetzner powers the server off to apply it, and one apply rescales every server whose type changed at the same time. `web_server_types` in `env_config` has one entry per web server so each apply can change one server. `keep_disk = true` keeps the disk at its current size, so a later move back to the smaller type stays possible. + +For each web server, one at a time: + +```bash +export HCLOUD_TOKEN=$(op read "op://Fleetyards/HCLOUD_LIVE/credential") + +# 1. Take it out of the load balancer (targets are selected by label). +# The apply in step 3 puts the label back. +hcloud server remove-label fltyrd-live-web-1 http + +# 2. Stop its app containers, so Sidekiq requeues in-flight jobs instead of +# losing them to the power-off (run from the app repo) +bundle exec kamal app stop -d live --hosts + +# 3. Change only this server's entry in web_server_types, merge, then run the +# Deploy workflow with target "live". The plan must show this server's +# server_type changing and nothing else. + +# 4. Start the app again and wait for the target to report healthy +bundle exec kamal app start -d live --hosts +hcloud load-balancer describe fltyrd-live-web-load-balancer +``` + +Then repeat for the next server. The Discord bot runs on one web server only, so it is offline while that server is rescaled. `kamal app stop` and `start` take the deploy lock; if one dies on an SSH reset, check `kamal lock status -d live` before going on. + ## Immutable Server Replacement (OS Upgrades) For major OS upgrades (e.g., Ubuntu 24.04 to 26.04) or significant base configuration changes, replace servers entirely rather than upgrading in-place. From 07ada573ab9e77494c07cb25f7505b6a97d43627 Mon Sep 17 00:00:00 2001 From: Marten Klitzke Date: Sun, 18 Oct 2026 03:44:19 +0200 Subject: [PATCH 3/5] docs(maintenance): expect the restored http label in the rescale plan --- MAINTENANCE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MAINTENANCE.md b/MAINTENANCE.md index 77e50dc..548b58f 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -189,7 +189,8 @@ bundle exec kamal app stop -d live --hosts # 3. Change only this server's entry in web_server_types, merge, then run the # Deploy workflow with target "live". The plan must show this server's -# server_type changing and nothing else. +# server_type changing and its http label coming back (removed in step 1), +# and nothing else. # 4. Start the app again and wait for the target to report healthy bundle exec kamal app start -d live --hosts From e6326ec24f7900bee236fb8784f03d04d0e1e791 Mon Sep 17 00:00:00 2001 From: Marten Klitzke Date: Wed, 4 Nov 2026 12:41:34 +0100 Subject: [PATCH 4/5] docs(maintenance): power the server back on in the rescale runbook, and read health per server The provider powers a server off to change its type and does not start it again. hcloud load-balancer list shows mixed for a label selector whatever the health. --- MAINTENANCE.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/MAINTENANCE.md b/MAINTENANCE.md index 548b58f..a44d239 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -174,6 +174,8 @@ ssh -J kamal@ kamal@ "docker ps" A server type change is an in-place update, but Hetzner powers the server off to apply it, and one apply rescales every server whose type changed at the same time. `web_server_types` in `env_config` has one entry per web server so each apply can change one server. `keep_disk = true` keeps the disk at its current size, so a later move back to the smaller type stays possible. +Hold merges in the app repo for the whole rollout: a deploy would try to reach a server that is off. Avoid the 20:00 and 22:00 Berlin loaders, which do not recover from a stopped worker. + For each web server, one at a time: ```bash @@ -192,9 +194,17 @@ bundle exec kamal app stop -d live --hosts # server_type changing and its http label coming back (removed in step 1), # and nothing else. -# 4. Start the app again and wait for the target to report healthy +# 4. Power it back on and wait for SSH. The provider powers the server off to +# change its type and does not start it again. The label is already back, but +# the target stays unhealthy, so it gets no traffic until step 5. +hcloud server poweron fltyrd-live-web-1 + +# 5. Start the app again and wait for the target to report healthy. `hcloud +# load-balancer list` shows "mixed" for a label selector however healthy the +# targets are, so read the status per server. bundle exec kamal app start -d live --hosts -hcloud load-balancer describe fltyrd-live-web-load-balancer +hcloud load-balancer describe fltyrd-live-web-load-balancer -o json \ + | jq '.targets[].targets[] | {server: .server.id, health: .health_status}' ``` Then repeat for the next server. The Discord bot runs on one web server only, so it is offline while that server is rescaled. `kamal app stop` and `start` take the deploy lock; if one dies on an SSH reset, check `kamal lock status -d live` before going on. From e6b9992fd969680fa7d0069fbde1748b4f440e7b Mon Sep 17 00:00:00 2001 From: Marten Klitzke Date: Wed, 4 Nov 2026 12:49:14 +0100 Subject: [PATCH 5/5] docs(maintenance): make the rescale runbook safe to follow as written Pin the version kamal app start uses, shut down before the apply, merge right before the rollout, cover a failed apply, and check capacity first. --- MAINTENANCE.md | 64 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/MAINTENANCE.md b/MAINTENANCE.md index a44d239..e69e48e 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -174,40 +174,66 @@ ssh -J kamal@ kamal@ "docker ps" A server type change is an in-place update, but Hetzner powers the server off to apply it, and one apply rescales every server whose type changed at the same time. `web_server_types` in `env_config` has one entry per web server so each apply can change one server. `keep_disk = true` keeps the disk at its current size, so a later move back to the smaller type stays possible. -Hold merges in the app repo for the whole rollout: a deploy would try to reach a server that is off. Avoid the 20:00 and 22:00 Berlin loaders, which do not recover from a stopped worker. +**Once a type change is on `main`, the next live apply of any kind performs it**, including `scripts/maintenance`: the provider cuts the power on a running server and does not start it again. So merge it right before the rollout, and run no other live apply until the rollout is done. Never have two servers' changes on `main` before the first is applied, as one apply would power off both. + +Before starting: + +- Hold merges in the app repo for the whole rollout: a deploy would try to reach a server that is off. +- Pick a quiet window. `config/sidekiq_schedule.yml` runs heavy jobs around 20:00-20:30, 22:00-22:45, 01:00-03:15 and 04:15-06:30, and 18:00-22:00 on Sundays. Stopping a worker requeues the jobs it is running, but a loader caught by the stop leaves its import marked as running, and the remaining server carries everything meanwhile. For each web server, one at a time: ```bash export HCLOUD_TOKEN=$(op read "op://Fleetyards/HCLOUD_LIVE/credential") -# 1. Take it out of the load balancer (targets are selected by label). -# The apply in step 3 puts the label back. +# 1. Check cx43 shows as available in the server's location. This is an +# indicator, not a guarantee. +hcloud server-type describe cx43 + +# 2. Merge the change to this server's entry in web_server_types. Live only +# applies on a manual run, so this changes nothing yet. + +# 3. Note the running version (run from the app repo). `kamal app start` uses +# the local git HEAD unless it is given a version, and the container for that +# version may not exist. +bundle exec kamal app version -d live --hosts + +# 4. Take it out of the load balancer (targets are selected by label). This also +# detaches the allow-http-https firewall, so in-flight requests and WebSockets +# drop rather than drain, and clients reconnect. The apply puts the label back. hcloud server remove-label fltyrd-live-web-1 http -# 2. Stop its app containers, so Sidekiq requeues in-flight jobs instead of -# losing them to the power-off (run from the app repo) +# 5. Stop its app containers (run from the app repo) bundle exec kamal app stop -d live --hosts -# 3. Change only this server's entry in web_server_types, merge, then run the -# Deploy workflow with target "live". The plan must show this server's -# server_type changing and its http label coming back (removed in step 1), -# and nothing else. +# 6. Shut it down cleanly. The provider only cuts the power on a running server. +hcloud server shutdown fltyrd-live-web-1 +hcloud server list -o columns=name,status -# 4. Power it back on and wait for SSH. The provider powers the server off to -# change its type and does not start it again. The label is already back, but -# the target stays unhealthy, so it gets no traffic until step 5. +# 7. Once it is off, run the Deploy workflow with target "live". The plan must +# show this server's server_type changing and its http label coming back, and +# nothing else. + +# 8. Power it back on and wait for SSH. The label is already back, but the target +# stays unhealthy, so it gets no traffic until the app is up. hcloud server poweron fltyrd-live-web-1 -# 5. Start the app again and wait for the target to report healthy. `hcloud -# load-balancer list` shows "mixed" for a label selector however healthy the -# targets are, so read the status per server. -bundle exec kamal app start -d live --hosts -hcloud load-balancer describe fltyrd-live-web-load-balancer -o json \ - | jq '.targets[].targets[] | {server: .server.id, health: .health_status}' +# 9. Start the running version from step 3, and wait for the target to report +# healthy. `hcloud load-balancer list` shows "mixed" for a label selector +# however healthy the targets are, so ask about this server. +bundle exec kamal app start -d live --hosts --version= +SERVER_ID=$(hcloud server describe fltyrd-live-web-1 -o json | jq .id) +until hcloud load-balancer describe fltyrd-live-web-load-balancer -o json | jq -e --argjson id "$SERVER_ID" \ + '[.targets[].targets[] | select(.server.id == $id) | .health_status[].status] | length > 0 and all(. == "healthy")' >/dev/null; do + sleep 10 +done ``` -Then repeat for the next server. The Discord bot runs on one web server only, so it is offline while that server is rescaled. `kamal app stop` and `start` take the deploy lock; if one dies on an SSH reset, check `kamal lock status -d live` before going on. +### If the apply fails + +The provider applies the label first, then the power change, then the type change, so a failed apply can leave the server off, on either type, and back in the load balancer. Check it with `hcloud server describe fltyrd-live-web-1`, then carry on from step 8 to bring it back, on whichever type it has. Retry from step 4 once the cause is fixed, for example when `resource_unavailable` means the location has no capacity for the new type. + +Then repeat for the next server. The Discord bot runs on one web server only, so it is offline while that server is rescaled; check which one first with `bundle exec kamal app details -d live -r discord_bot`, as it follows the first IP the server list returns. `kamal app stop` and `start` take the deploy lock; if one dies on an SSH reset, check `kamal lock status -d live` before going on. ## Immutable Server Replacement (OS Upgrades)