From a14639d62b775c9c51e890a94a2b17033f480d71 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 06:16:11 +0000 Subject: [PATCH] cockpit-server: bucket-read multi-region OSM catalogue + /api/osm/regions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the "adding/selecting a region needs an env-var change + redeploy" friction: OSM_BAKE_REGION previously selected the single servable region, and switching regions meant a Railway env-var edit (which triggers a redeploy) even though the bake itself already lives in S3 with no code change needed. - New osm_region_catalogue.rs: reads .config/q2/config.yaml from the bucket via lance_graph::soa_config's shared schema/parser (the same boot-config pattern every other AdaWorldAPI deployment uses for its own bake catalogue) and projects it onto (name, hydrate). table/classid are validated but unused here — this deployment's OSM bakes are still raw .soa/.books/.chains file triplets, not Lance tables; reusing the shared, already-hardened parser (unknown-key rejection, duplicate-name rejection) beats hand-rolling a near-identical one. - osm_slab_hydrate::ensure_slab_local now: loads the catalogue (falling back to a single synthetic entry from OSM_BAKE_REGION/DEFAULT_REGION when no config.yaml exists — zero behavior change for deploys that never opt in), hydrates every hydrate:true entry (hydrate:false stays declared-but-not-pulled, never fetched), and publishes the resolved catalogue for the new endpoint. OSM_BAKE_REGION, when set, still wins outright as the active region even if absent from the catalogue or declared hydrate:false there — the pre-existing escape hatch is unchanged, including bake_region()'s validate-or-fall-back-to-default behavior on an invalid value. - Region-name validation ([a-z0-9-], <=64 bytes — guards the S3 key/path join against traversal) now applies to every catalogue-sourced name too, not only the env-var path. - New GET /api/osm/regions: the map-menu data source (name, hydrated, active per region). Read-only discovery — this deployment still serves exactly one active region at a time (open_slab()'s mmap is process-global); switching which one is active still needs the operator to flip hydrate:true (or set OSM_BAKE_REGION) and restart. That is a narrower claim than "hot-swap without restart," which this change does not attempt. Test plan: - 12 new unit tests in osm_region_catalogue.rs (TDD): parse_catalogue_yaml projects name/hydrate correctly, keeps hydrate:false entries (does not drop or promote them), rejects malformed YAML, propagates the shared parser's own duplicate-name validation; pick_active prefers the env override even when absent from the catalogue or over a hydrate:true entry, falls back to the first hydrate:true entry, returns None on an all-false catalogue with no override (paired negative case); the publish/snapshot OnceLock round-trips and a second publish is a silent no-op rather than a panic. - cargo check -p cockpit-server --bin q2-cockpit --tests: clean, 0 errors, 0 new warnings (34 pre-existing baseline warnings unchanged). - cargo test/nextest for this crate could not be run to completion in this session: the q2-cockpit test binary transitively links nearly the entire q2 workspace (lance/arrow/datafusion/geo/aws-sdk/opendal/ tree-sitter/wasmtime/mlua/...), and the final link step exceeded this container's disk allowance on repeated attempts (ENOSPC, then a linker SIGBUS), even immediately after a full target/ clean. This is the same environment limitation documented on q2#141 for the same binary, unrelated to this change. Reported honestly: type/borrow-check-verified, not executed-test-verified. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01NMeiLmtDKhomJNSo2ecbJw --- crates/cockpit-server/src/main.rs | 5 + .../src/osm_region_catalogue.rs | 342 ++++++++++++++++++ crates/cockpit-server/src/osm_slab_hydrate.rs | 191 ++++++++-- 3 files changed, 506 insertions(+), 32 deletions(-) create mode 100644 crates/cockpit-server/src/osm_region_catalogue.rs diff --git a/crates/cockpit-server/src/main.rs b/crates/cockpit-server/src/main.rs index 0e72b9e36..755ea775b 100644 --- a/crates/cockpit-server/src/main.rs +++ b/crates/cockpit-server/src/main.rs @@ -47,6 +47,7 @@ mod osm_chains_books_lance; mod osm_features; mod osm_lance; mod osm_lifecycle; +mod osm_region_catalogue; mod osm_slab_hydrate; // ── Embed the Vite build at compile time ───────────────────────────────────── @@ -363,6 +364,10 @@ async fn main() { // working map on every other signal (200s, full tiles, correct // geometry) while drawing grey and untagged. .route("/api/osm/health", get(osm_features::osm_health_handler)) + .route( + "/api/osm/regions", + get(osm_region_catalogue::osm_regions_handler), + ) // Phase B of the OSM/Lance lifecycle plan: the OsmArtifactManager's // lifecycle/artifact snapshot + cgroup memory, observational only — // does not read map data and does not touch the OnceLock-backed diff --git a/crates/cockpit-server/src/osm_region_catalogue.rs b/crates/cockpit-server/src/osm_region_catalogue.rs new file mode 100644 index 000000000..58f9a9135 --- /dev/null +++ b/crates/cockpit-server/src/osm_region_catalogue.rs @@ -0,0 +1,342 @@ +//! The multi-region OSM bake catalogue — the consumer side of +//! `lance_graph::soa_config`'s boot-config pattern +//! (`docs/SOA_BAKE_DEPLOYMENT.md` §6 in the `lance-graph` repo). +//! +//! # The problem this closes +//! +//! Before this module, exactly one region was servable per deploy, chosen by +//! the `OSM_BAKE_REGION` env var (`osm_slab_hydrate::bake_region`). Adding or +//! switching a region meant editing that env var on Railway, which triggers a +//! redeploy — even though the bake itself already lives in S3 +//! (`q2/bakes/-v1`) and needs no code change at all. +//! +//! This module reads a declarative catalogue instead: `.config/q2/config.yaml` +//! in the same bucket, via [`lance_graph::soa_config`]'s shared schema and +//! parser — the SAME module/file every other AdaWorldAPI deployment reads its +//! own bake catalogue from (`SoaConfig`/`BakeEntry`). Adding a region to the +//! catalogue is an S3 object edit, not a redeploy. +//! +//! # `table`/`classid` are validated but not used here +//! +//! [`lance_graph::soa_config::BakeEntry`] was designed for bakes written as +//! Lance tables (`soa_to_lance`). This deployment's OSM bakes are still raw +//! `.soa`/`.books`/`.chains` file triplets fetched by checksum +//! (`osm_slab_hydrate::artifacts`) — a different, older hydration shape that +//! predates `soa_config` and is not being migrated by this change. Reusing +//! `soa_config`'s YAML schema and parser here is deliberate: `name` and +//! `hydrate` are exactly what a region catalogue needs, and reusing the +//! shared, already-hardened parser (unknown-key rejection, duplicate-name +//! rejection, `deny_unknown_fields`) is strictly better than hand-rolling a +//! near-identical one. `table` and `classid` still have to be present and +//! valid for a bake entry to parse (the schema requires them), but nothing in +//! this module reads either — a `table` value can be any non-empty string, by +//! convention the region's own bake identifier (e.g. `brandenburg-v1`). +//! +//! # `hydrate: false` is a real state, not a soft error +//! +//! A region declared with `hydrate: false` shows up in the +//! [`RegionEntry`] catalogue — and therefore in `/api/osm/regions`'s response, +//! for a UI region menu — but [`crate::osm_slab_hydrate::ensure_slab_local`] +//! never downloads its multi-GB artifact triplet. That is the entire point: +//! declaring a region costs nothing until an operator flips it to +//! `hydrate: true` (or names it via `OSM_BAKE_REGION`, see below). +//! +//! # The `OSM_BAKE_REGION` escape hatch still works, unchanged +//! +//! `OSM_BAKE_REGION`, when set, always selects the ACTIVE region — even if it +//! is absent from the catalogue entirely, or declared `hydrate: false` there. +//! This is the pre-existing single-region behavior and this module does not +//! narrow it: a deploy that has never heard of `.config/q2/config.yaml` +//! behaves identically to before, because [`pick_active`] with no catalogue +//! (or an empty one) and an env override just returns that override. + +use std::sync::OnceLock; + +use axum::Json; +use serde::Serialize; + +/// One declared region. `hydrate` is whether THIS deployment pulls its +/// artifact triplet to local disk at boot — see the module doc's "declared +/// but not pulled" section. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct RegionEntry { + pub name: String, + pub hydrate: bool, +} + +/// Parse `.config//config.yaml`'s body into a region catalogue. +/// +/// Delegates entirely to [`lance_graph::soa_config::parse`] for validation +/// (schema version, duplicate names/tables, malformed classid, unknown +/// keys) — this function's only job is projecting the validated +/// [`lance_graph::soa_config::BakeEntry`] list onto the two fields this +/// deployment actually reads. Returns `None` on any parse/validation +/// failure; the caller falls back to the single-region default rather than +/// serving a half-understood catalogue (same "absent config is not an +/// error, but never silent" posture as `osm_slab_hydrate`). +pub(crate) fn parse_catalogue_yaml(yaml: &str) -> Option> { + let config = match lance_graph::soa_config::parse(yaml) { + Ok(c) => c, + Err(e) => { + tracing::error!(error = %e, "osm regions: config.yaml did not parse; ignoring it"); + return None; + } + }; + Some( + config + .bakes + .iter() + .map(|b| RegionEntry { + name: b.name.clone(), + hydrate: b.hydrate, + }) + .collect(), + ) +} + +/// Which region name is ACTIVE — the one whose slab gets hydrated, +/// converted to Lance, mmap'd, and served. +/// +/// `override_region` (from `OSM_BAKE_REGION`) always wins when set, whether +/// or not it names a catalogue entry — see the module doc. Otherwise the +/// first `hydrate: true` catalogue entry wins (declaration order is +/// priority order); `None` if the catalogue is empty and there is no +/// override, meaning nothing is configured to serve. +pub(crate) fn pick_active<'a>( + catalogue: &'a [RegionEntry], + override_region: Option<&'a str>, +) -> Option<&'a str> { + if let Some(name) = override_region { + return Some(name); + } + catalogue + .iter() + .find(|e| e.hydrate) + .map(|e| e.name.as_str()) +} + +/// The catalogue snapshot [`crate::osm_slab_hydrate::ensure_slab_local`] +/// publishes at boot, for `/api/osm/regions` to read without redoing any +/// S3 I/O or re-parsing YAML per request. +struct CatalogueSnapshot { + regions: Vec, + active: String, +} + +static CATALOGUE: OnceLock> = OnceLock::new(); + +/// Publish the boot-resolved catalogue. Called exactly once, from +/// `ensure_slab_local`, before the listener binds — see that function's own +/// single-threaded-startup safety note. A second call is a silent no-op +/// (`OnceLock::set` returns `Err` and this function drops it) rather than a +/// panic, so a future refactor that accidentally calls this twice fails +/// quietly instead of crashing the boot sequence over a diagnostics +/// side-channel. +pub(crate) fn publish(regions: Vec, active: String) { + let _ = CATALOGUE.set(Some(CatalogueSnapshot { regions, active })); +} + +/// `(regions, active)` as published at boot, or `None` if OSM hydration was +/// never configured on this deploy (mirrors `ensure_slab_local` returning +/// `None`) — `/api/osm/regions` reads this directly. +pub(crate) fn snapshot() -> Option<(&'static [RegionEntry], &'static str)> { + CATALOGUE + .get() + .and_then(|opt| opt.as_ref()) + .map(|s| (s.regions.as_slice(), s.active.as_str())) +} + +/// `GET /api/osm/regions` — the map-menu data source: every declared +/// region, whether it is hydrated (immediately servable), and which one is +/// currently active. +/// +/// Read-only discovery. This deployment still serves exactly one ACTIVE +/// region at a time (the `open_slab()` mmap is process-global) — this +/// endpoint does not let a client switch it, only see the catalogue an +/// operator has declared in `.config/q2/config.yaml`. A region shown with +/// `"hydrated": false` needs the operator to flip it to `hydrate: true` (or +/// set `OSM_BAKE_REGION`) and restart before it can become active. +pub async fn osm_regions_handler() -> Json { + match snapshot() { + Some((regions, active)) => Json(serde_json::json!({ + "active": active, + "regions": regions.iter().map(|r| serde_json::json!({ + "name": r.name, + "hydrated": r.hydrate, + "active": r.name == active, + })).collect::>(), + })), + None => Json(serde_json::json!({ + "active": serde_json::Value::Null, + "regions": [], + "note": "OSM hydration is not configured on this deploy", + })), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn yaml_with_two_regions() -> &'static str { + r#" +version: 1 +ledger_prefix: "q2/ledger" +bakes: + - name: berlin + table: berlin-v1 + classid: "0x0F020000" + hydrate: true + - name: brandenburg + table: brandenburg-v1 + classid: "0x0F020000" + hydrate: false +"# + } + + #[test] + fn parse_catalogue_yaml_projects_name_and_hydrate() { + let entries = parse_catalogue_yaml(yaml_with_two_regions()).expect("must parse"); + assert_eq!( + entries, + vec![ + RegionEntry { + name: "berlin".to_string(), + hydrate: true + }, + RegionEntry { + name: "brandenburg".to_string(), + hydrate: false + }, + ] + ); + } + + /// The whole point of a hydrate:false entry: it survives the parse and + /// is neither dropped nor silently promoted to hydrate:true. + #[test] + fn hydrate_false_entries_are_kept_not_dropped() { + let entries = parse_catalogue_yaml(yaml_with_two_regions()).unwrap(); + assert_eq!(entries.len(), 2, "a hydrate:false entry must not vanish"); + let brandenburg = entries.iter().find(|e| e.name == "brandenburg").unwrap(); + assert!(!brandenburg.hydrate); + } + + #[test] + fn parse_catalogue_yaml_rejects_malformed_yaml() { + assert!(parse_catalogue_yaml("not: [valid").is_none()); + } + + /// The shared parser's own duplicate-name rejection must be visible + /// through this projection, not swallowed — a config with two regions + /// named "berlin" is a real authoring mistake and must fail loudly. + #[test] + fn parse_catalogue_yaml_propagates_shared_parser_validation() { + let yaml = r#" +version: 1 +ledger_prefix: "q2/ledger" +bakes: + - name: berlin + table: a + classid: "0x0F020000" + - name: berlin + table: b + classid: "0x0F020000" +"#; + assert!( + parse_catalogue_yaml(yaml).is_none(), + "duplicate region name must be rejected, not silently deduped" + ); + } + + #[test] + fn pick_active_prefers_the_env_override_even_when_absent_from_the_catalogue() { + let catalogue = parse_catalogue_yaml(yaml_with_two_regions()).unwrap(); + // "munich" names neither catalogue entry — the override must still win, + // preserving the pre-catalogue OSM_BAKE_REGION escape hatch exactly. + assert_eq!(pick_active(&catalogue, Some("munich")), Some("munich")); + } + + #[test] + fn pick_active_prefers_the_env_override_over_a_hydrate_true_entry() { + let catalogue = parse_catalogue_yaml(yaml_with_two_regions()).unwrap(); + // berlin is hydrate:true and would win by default — the override + // (naming the hydrate:false brandenburg) must still take priority. + assert_eq!( + pick_active(&catalogue, Some("brandenburg")), + Some("brandenburg") + ); + } + + #[test] + fn pick_active_falls_back_to_the_first_hydrate_true_entry() { + let catalogue = parse_catalogue_yaml(yaml_with_two_regions()).unwrap(); + assert_eq!(pick_active(&catalogue, None), Some("berlin")); + } + + /// Paired negative case: an ALL-false catalogue has no active region + /// without an override — this must not silently pick a declared-but- + /// unhydrated region and then fail to find its files on disk. + #[test] + fn pick_active_returns_none_when_nothing_is_hydrate_true_and_no_override() { + let catalogue = vec![RegionEntry { + name: "brandenburg".to_string(), + hydrate: false, + }]; + assert_eq!(pick_active(&catalogue, None), None); + } + + #[test] + fn pick_active_on_an_empty_catalogue_with_no_override_is_none() { + assert_eq!(pick_active(&[], None), None); + } + + #[test] + fn pick_active_on_an_empty_catalogue_still_honors_an_override() { + // The synthetic single-entry fallback path: no bucket config exists + // at all, so the catalogue is empty, but OSM_BAKE_REGION is set. + assert_eq!(pick_active(&[], Some("berlin")), Some("berlin")); + } + + #[test] + fn snapshot_is_none_before_publish() { + // NOTE: this test only holds if it runs before any other test in + // this process calls `publish` — OnceLock is process-global. Given + // nextest's one-process-per-test model (already relied on elsewhere + // in this crate, e.g. osm_features.rs's slab_digest tests), that is + // safe here. + assert!(snapshot().is_none()); + } + + #[test] + fn publish_then_snapshot_round_trips() { + let regions = vec![RegionEntry { + name: "berlin".to_string(), + hydrate: true, + }]; + publish(regions.clone(), "berlin".to_string()); + let (got_regions, got_active) = snapshot().expect("must be published now"); + assert_eq!(got_regions, regions.as_slice()); + assert_eq!(got_active, "berlin"); + } + + #[test] + fn a_second_publish_is_a_silent_no_op_not_a_panic() { + publish( + vec![RegionEntry { + name: "x".to_string(), + hydrate: true, + }], + "x".to_string(), + ); + // Must not panic — OnceLock::set failing on the second call is + // expected and handled, not propagated. + publish( + vec![RegionEntry { + name: "y".to_string(), + hydrate: true, + }], + "y".to_string(), + ); + } +} diff --git a/crates/cockpit-server/src/osm_slab_hydrate.rs b/crates/cockpit-server/src/osm_slab_hydrate.rs index bb3b0545e..77a732b28 100644 --- a/crates/cockpit-server/src/osm_slab_hydrate.rs +++ b/crates/cockpit-server/src/osm_slab_hydrate.rs @@ -169,9 +169,18 @@ fn missing_inputs(bucket: Option<&str>, vol: Option<&str>) -> Vec<&'static str> /// Resolve a local, verified slab path, hydrating from S3 if needed. /// -/// Returns the path to set as `OSM_SLAB_PATH`, or `None` when the feature is -/// not configured — never panics, and never returns a path that failed its -/// checksum. +/// Returns the path to set as `OSM_SLAB_PATH` for the ACTIVE region, or +/// `None` when the feature is not configured — never panics, and never +/// returns a path that failed its checksum. +/// +/// Since [`crate::osm_region_catalogue`] landed, this also: reads a +/// multi-region catalogue from `.config/q2/config.yaml` (falling back to a +/// single synthetic entry when absent/invalid — see that module's doc for +/// the full `OSM_BAKE_REGION` interaction), hydrates every catalogue entry +/// declared `hydrate: true` (not only the active one, so a deploy can +/// pre-warm several regions), and publishes the resolved catalogue via +/// [`crate::osm_region_catalogue::publish`] for `/api/osm/regions` to read. +/// `hydrate: false` entries are deliberately never fetched here. pub async fn ensure_slab_local() -> Option { // 1. An explicit local path always wins. This is the local-dev and // already-hydrated case, and it must not require S3 credentials. @@ -206,10 +215,6 @@ pub async fn ensure_slab_local() -> Option { } let bucket = bucket_env.expect("checked non-empty above"); let vol = vol_env.expect("checked non-empty above"); - let region = bake_region(); - let artifacts = artifacts(®ion); - let prefix = - env_var_nonempty("OSM_SLAB_S3_PREFIX").unwrap_or_else(|| default_prefix(®ion)); let dir = cache_dir(&vol); if let Err(e) = std::fs::create_dir_all(&dir) { @@ -217,18 +222,6 @@ pub async fn ensure_slab_local() -> Option { return None; } - // Announce BEFORE the transfer, not after. This call blocks the listener - // bind, and a cold boot moves ~1.42 GB — so without a line here the boot - // log is silent for 60-90s, which is indistinguishable from a hang for - // whoever is watching a deploy. Naming the bucket and destination also - // makes a misconfigured prefix obvious from the first line rather than - // from a later "not readable" error. - tracing::info!( - %region, %bucket, %prefix, dir = %dir.display(), - "osm slab: resolving from S3 (a cold boot transfers the whole bake and delays the \ - listener — Berlin is ~1.42 GB; a warm volume re-verifies in ~1s)" - ); - // `from_env()` reads AWS_ENDPOINT_URL / AWS_ACCESS_KEY_ID / // AWS_SECRET_ACCESS_KEY / AWS_DEFAULT_REGION with no glue — `aws_endpoint_url` // is an accepted alias for the endpoint key, so a non-AWS S3 endpoint needs @@ -244,16 +237,152 @@ pub async fn ensure_slab_local() -> Option { } }; - let sums = match fetch_sums(&store, &prefix).await { - Some(s) => s, - None => return None, + // The env-var override, when set, wins outright as the active region — + // see `crate::osm_region_catalogue`'s module doc for why this is + // preserved unchanged rather than superseded by the catalogue. + // + // `None` only when `OSM_BAKE_REGION` itself is unset/empty (the + // catalogue decides in that case). When it IS set, `bake_region()`'s + // exact validate-or-fall-back-to-`DEFAULT_REGION`-with-a-warning + // behavior is preserved unchanged — an invalid value (uppercase, `/`, + // `..`) must still gracefully degrade to the default region, not become + // a hard hydration failure. + let override_region = env_var_nonempty("OSM_BAKE_REGION").map(|_| bake_region()); + + let catalogue = fetch_catalogue(&store, &bucket).await; + let catalogue = if catalogue.is_empty() { + vec![crate::osm_region_catalogue::RegionEntry { + name: override_region.clone().unwrap_or_else(|| DEFAULT_REGION.to_string()), + hydrate: true, + }] + } else { + catalogue + }; + + let Some(active) = + crate::osm_region_catalogue::pick_active(&catalogue, override_region.as_deref()) + .map(str::to_string) + else { + tracing::warn!( + "osm slab: catalogue declares no hydrate:true region and OSM_BAKE_REGION is \ + unset; nothing to serve" + ); + return None; + }; + + // `OSM_SLAB_S3_PREFIX` is the pre-catalogue single-region escape hatch + // (module doc, "Absent configuration" section) — it applies ONLY to the + // active region, exactly as it always has. Pre-warmed, non-active + // catalogue entries always use `default_prefix` (the `{region}-v1` + // convention); a global prefix override cannot mean something different + // per region. + let active_prefix_override = env_var_nonempty("OSM_SLAB_S3_PREFIX"); + let active_path = + hydrate_one_region(&store, &dir, &active, active_prefix_override.as_deref()).await; + + // Pre-warm every OTHER declared hydrate:true region — best-effort. A + // failure here is logged by `hydrate_one_region` itself and does not + // affect the active region's own hydration result. + for entry in catalogue.iter().filter(|e| e.hydrate && e.name != active) { + hydrate_one_region(&store, &dir, &entry.name, None).await; + } + + crate::osm_region_catalogue::publish(catalogue, active); + + let Some(slab) = active_path else { + return None; + }; + tracing::info!(path = %slab.display(), "osm slab: hydrated and verified"); + Some(slab) +} + +/// Fetch and parse `.config//config.yaml` into a region catalogue. +/// Returns an EMPTY `Vec` — never an error — when the object is absent or +/// fails to parse: [`ensure_slab_local`] treats that identically to "no +/// catalogue configured" and falls back to the single-region default. A +/// parse failure is still logged loudly by +/// [`crate::osm_region_catalogue::parse_catalogue_yaml`] itself. +async fn fetch_catalogue( + store: &impl ObjectStore, + bucket: &str, +) -> Vec { + let key = lance_graph::soa_config::config_key("q2"); + let path = object_store::path::Path::from(key.clone()); + let bytes = match store.get(&path).await { + Ok(r) => match r.bytes().await { + Ok(b) => b, + Err(e) => { + tracing::warn!( + %bucket, %key, error = %e, + "osm regions: config.yaml body read failed; falling back to a single region" + ); + return Vec::new(); + } + }, + // Absent is the expected, common case for a deploy that has not + // opted into the multi-region catalogue — INFO, not WARN/ERROR. + Err(e) => { + tracing::info!( + %bucket, %key, error = %e, + "osm regions: no config.yaml (single-region mode via OSM_BAKE_REGION)" + ); + return Vec::new(); + } }; + crate::osm_region_catalogue::parse_catalogue_yaml(&String::from_utf8_lossy(&bytes)) + .unwrap_or_default() +} - for name in artifacts.iter().map(String::as_str) { +/// Hydrate ONE region's artifact triplet to local disk, verified against its +/// `SHA256SUMS`. Returns the path to `.soa`, or `None` on any +/// failure — logged at the call site's usual level, never panics. +/// +/// `region` is re-validated HERE, not only at `bake_region()`'s env-var +/// boundary: since `crate::osm_region_catalogue` landed, this is also +/// called with names read from a bucket-hosted YAML catalogue an operator +/// controls, and this string is interpolated into both an S3 key and a +/// filesystem path (see [`is_valid_region`]'s own doc for why that makes +/// validation load-bearing, not decoration). A rejected name hydrates +/// nothing and returns `None` rather than trying anyway. +async fn hydrate_one_region( + store: &impl ObjectStore, + dir: &Path, + region: &str, + prefix_override: Option<&str>, +) -> Option { + if !is_valid_region(region) { + tracing::error!( + region, + "osm slab: region name rejected (must be lowercase [a-z0-9-], \u{2264}64 bytes); \ + refusing to hydrate it" + ); + return None; + } + let region_artifacts = artifacts(region); + let prefix = prefix_override + .map(str::to_string) + .unwrap_or_else(|| default_prefix(region)); + + // Announce BEFORE the transfer, not after. This call blocks the listener + // bind for the ACTIVE region, and a cold boot moves multiple GB — so + // without a line here the boot log is silent for 60-90s, which is + // indistinguishable from a hang for whoever is watching a deploy. Naming + // the region and prefix also makes a misconfigured catalogue entry + // obvious from the first line rather than from a later "not readable" + // error. + tracing::info!( + region, %prefix, dir = %dir.display(), + "osm slab: resolving region from S3 (a cold boot transfers the whole bake; a warm \ + volume re-verifies in ~1s)" + ); + + let sums = fetch_sums(store, &prefix).await?; + + for name in region_artifacts.iter().map(String::as_str) { let want = match sums.iter().find(|(k, _)| k == name).map(|(_, h)| h.clone()) { Some(h) => h, None => { - tracing::error!(artifact = name, "osm slab: no checksum pinned; refusing"); + tracing::error!(region, artifact = name, "osm slab: no checksum pinned; refusing"); return None; } }; @@ -264,34 +393,32 @@ pub async fn ensure_slab_local() -> Option { match resolve_cache_hit(&dest, &want) { CacheDecision::TrustedViaMarker => { tracing::info!( - artifact = name, + region, artifact = name, "osm slab: cache hit, trusted via unchanged marker (mtime+len \ match the last real verification; skipped re-hash)" ); continue; } CacheDecision::Verified => { - tracing::info!(artifact = name, "osm slab: cache hit, checksum verified"); + tracing::info!(region, artifact = name, "osm slab: cache hit, checksum verified"); continue; } CacheDecision::Mismatch(got) => tracing::warn!( - artifact = name, %got, %want, + region, artifact = name, %got, %want, "osm slab: cached copy failed its checksum; re-fetching" ), CacheDecision::Unreadable(e) => { - tracing::warn!(artifact = name, error = %e, "osm slab: cannot hash cached copy; re-fetching") + tracing::warn!(region, artifact = name, error = %e, "osm slab: cannot hash cached copy; re-fetching") } } } - if !download_verified(&store, &prefix, name, &dest, &want).await { + if !download_verified(store, &prefix, name, &dest, &want).await { return None; } } - let slab = dir.join(&artifacts[0]); - tracing::info!(path = %slab.display(), "osm slab: hydrated and verified"); - Some(slab) + Some(dir.join(®ion_artifacts[0])) } /// Fetch and parse `SHA256SUMS` — ` ` per line, the `sha256sum`