From d6d6f5fe7bc1c434bf040c1fa149052aac2c26fe Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 30 Oct 2026 00:23:16 +0800 Subject: [PATCH 01/24] Fix CI startup races and component build scope --- .github/workflows/ci.yml | 4 ---- .../src/node/runtime/recovery/discovery.rs | 8 +++++++- .../crowdb-monitor/tests/member_trust_test.rs | 19 +++++++++++++++---- .../oci-image/tests/task_entrypoints_test.py | 2 +- .../access_iceberg/file_http/concurrency.rs | 12 ++++++++++-- e2e-test/tests/access_iceberg/signed_file.rs | 1 + pixi.toml | 6 ++++-- tools/README.md | 4 ++-- 8 files changed, 40 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 180c1fcf4..508b696da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,8 +63,6 @@ jobs: uses: ./.github/actions/compilation-cache - name: Clean env run: pixi run clean --runtime-only - - name: Build C++ libs - run: pixi run build - name: Run C++ and FFI tests run: pixi run test-cpp - name: Capture runner state on failure @@ -171,8 +169,6 @@ jobs: uses: ./.github/actions/compilation-cache - name: Clean env run: pixi run clean --runtime-only - - name: Build C++ libs (crowdb-diskio subprocess) - run: pixi run build - name: Run storage component tests run: pixi run test-storage - name: Capture runner state on failure diff --git a/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs b/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs index 4311b1498..bf050d51d 100644 --- a/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs +++ b/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs @@ -157,7 +157,13 @@ fn unique(endpoints: impl Iterator) -> (Vec String { .join("&") } +#[derive(Clone)] pub struct TestFileClient { pub client: Client, pub credentials: crowdb_access_iceberg::file::FileCredentials, diff --git a/pixi.toml b/pixi.toml index aa871e432..7d3088b2d 100644 --- a/pixi.toml +++ b/pixi.toml @@ -130,6 +130,8 @@ auditwheel = ">=6, <7" install-deps = { cmd = "bash tools/pixi-tasks/install-deps.sh" } # Stop services and remove build products/runtime state; --runtime-only preserves build products and persistent clusters. clean = { cmd = "bash tools/pixi-tasks/clean-build.sh" } +# Build the native libraries used by the C++ and storage component gates. +build-native = "bash tools/pixi-tasks/build-cpp.sh" # Build C++ libraries, Rust release binaries, frontend and workspace test targets. build = { cmd = "bash tools/pixi-tasks/build-release.sh", depends-on = ["install-deps"] } default = { depends-on = ["build"] } @@ -139,9 +141,9 @@ default = { depends-on = ["build"] } ################################################################################ test = "python tools/pixi-tasks/test.py" test-suite = { cmd = "python tools/pixi-tasks/test.py all", depends-on = ["build"] } -test-cpp = { cmd = "python tools/pixi-tasks/test.py unit cpp", depends-on = ["build"] } +test-cpp = { cmd = "python tools/pixi-tasks/test.py unit cpp", depends-on = ["build-native"] } test-core = { cmd = "python tools/pixi-tasks/test.py unit core" } -test-storage = { cmd = "python tools/pixi-tasks/test.py unit storage" } +test-storage = { cmd = "python tools/pixi-tasks/test.py unit storage", depends-on = ["build-native"] } test-access = { cmd = "python tools/pixi-tasks/test.py unit access" } test-console = { cmd = "python tools/pixi-tasks/test.py unit console" } test-console-ui = { cmd = "python tools/pixi-tasks/test.py unit ui", depends-on = ["install-deps"] } diff --git a/tools/README.md b/tools/README.md index dbe6614b0..4621f1926 100644 --- a/tools/README.md +++ b/tools/README.md @@ -127,5 +127,5 @@ pixi run test profile containerd-cluster pixi run test-e2e-iceberg ``` -On a fresh checkout, `test-cpp` depends on the unified build and `test-console-ui` -depends on dependency installation before dispatch. +On a fresh checkout, `test-cpp` and `test-storage` depend on the native library +build, while `test-console-ui` depends on dependency installation before dispatch. From 377d496f1fa9bbff92e87026a9cd8b0c4202d0f3 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 30 Oct 2026 10:51:21 +0800 Subject: [PATCH 02/24] Improve CI readiness and physical node test setup --- .../service/chunkdb_rpc_service/mutations.rs | 5 + .../src/service/chunkdb_rpc_service/wire.rs | 55 ++++++- app/crowdb-chunkdb/src/storage.rs | 4 +- container/crowdb-monitor/src/node/runtime.rs | 8 +- .../crowdb-monitor/src/node/runtime/health.rs | 60 ++++++- .../src/node/runtime/services/test_support.rs | 7 +- container/crowdb-monitor/src/node_args.rs | 32 +++- .../tests/node_service_readiness_test.rs | 135 ++++++++++++++++ container/single-node-container/entrypoint.sh | 21 ++- doc/working/plan-two-physical-nodes.md | 151 ++++++++++++++++++ doc/working/track-two-physical-nodes.md | 43 +++++ lib/crowdb-chunkdb-client/src/retry.rs | 7 +- 12 files changed, 515 insertions(+), 13 deletions(-) create mode 100644 doc/working/plan-two-physical-nodes.md create mode 100644 doc/working/track-two-physical-nodes.md diff --git a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs index d52c5fea9..f3fee7ee9 100644 --- a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs +++ b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs @@ -617,6 +617,7 @@ impl ChunkdbRpcService { create_nano, msg_type, result, + false, ); }); } @@ -714,6 +715,7 @@ impl ChunkdbRpcService { create_nano, msg_type, result, + true, ); }); } @@ -782,6 +784,7 @@ impl ChunkdbRpcService { create_nano, msg_type, result, + false, ); }); } @@ -924,6 +927,7 @@ impl ChunkdbRpcService { create_nano, msg_type, result, + false, ); }); } @@ -984,6 +988,7 @@ impl ChunkdbRpcService { create_nano, msg_type, result, + false, ); }); } diff --git a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs index d42cd699e..8ef19d8c6 100644 --- a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs +++ b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs @@ -24,6 +24,13 @@ use crowdb_protocol::chunkdb_fb::{FBPlacementAssessment, FBPlacementAssessmentAr /// Map a `LifecycleError` to `(ret_code, message, range_start, range_end)`. pub(super) fn map_error(e: &LifecycleError) -> (FBChunkdbRetCode, String, u32, u32) { + map_error_with_policy(e, false) +} + +fn map_error_with_policy( + e: &LifecycleError, + allocation_retryable: bool, +) -> (FBChunkdbRetCode, String, u32, u32) { match e { LifecycleError::Storage(crate::storage::StoreError::Authority) => { (FBChunkdbRetCode::NotMyRange, e.to_string(), 0, 0) @@ -45,6 +52,19 @@ pub(super) fn map_error(e: &LifecycleError) -> (FBChunkdbRetCode, String, u32, u LifecycleError::Allocation(_) | LifecycleError::Commit(_) | LifecycleError::Cleanup(_) => { (FBChunkdbRetCode::Internal, e.to_string(), 0, 0) } + // Chunk creation rolls back its physical strip allocation when the + // metadata batch cannot be committed. A routed KV quorum/election + // outage is therefore safe for the client to retry as a fresh + // allocation; keep deterministic metadata failures internal. + LifecycleError::Storage(crate::storage::StoreError::Kv(message)) + if allocation_retryable + && (message.contains("leadership quorum unavailable") + || message.contains("retries exhausted") + || message.contains("no known leader") + || message.contains("transport error")) => + { + (FBChunkdbRetCode::Unavailable, e.to_string(), 0, 0) + } LifecycleError::Storage(_) => (FBChunkdbRetCode::Internal, e.to_string(), 0, 0), LifecycleError::InvalidRequest(_) => (FBChunkdbRetCode::InvalidArgument, e.to_string(), 0, 0), LifecycleError::LockBusy @@ -89,6 +109,7 @@ pub(super) fn submit_chunk_result( create_nano: u64, msg_type: u16, result: Result, + allocation_retryable: bool, ) { match result { Ok(chunk) => { @@ -97,7 +118,7 @@ pub(super) fn submit_chunk_result( submit_fb_response(server, conn_handle, ctrl, msg_type, req_id); } Err(e) => { - let (code, msg, rs, re) = map_error(&e); + let (code, msg, rs, re) = map_error_with_policy(&e, allocation_retryable); let ctrl = build_chunk_response(req_id, create_nano, code, Some(&msg), rs, re, &Chunk::default()); submit_fb_response(server, conn_handle, ctrl, msg_type, req_id); } @@ -1020,3 +1041,35 @@ pub(super) fn fb_ec_state(s: ProtoEcState) -> FBEcState { ProtoEcState::Parity => FBEcState::Parity, } } + +#[cfg(test)] +mod tests { + use super::{map_error, map_error_with_policy, FBChunkdbRetCode, LifecycleError}; + use crate::storage::StoreError; + + #[test] + fn transient_kv_quorum_errors_are_retryable() { + for message in [ + "linearizable read: leadership quorum unavailable", + "retries exhausted after 11 attempts", + "no known leader for group", + "transport error to endpoint", + ] { + let error = LifecycleError::Storage(StoreError::Kv(message.into())); + assert_eq!( + map_error_with_policy(&error, true).0, + FBChunkdbRetCode::Unavailable, + "{message}" + ); + assert_eq!(map_error(&error).0, FBChunkdbRetCode::Internal); + } + } + + #[test] + fn deterministic_kv_errors_remain_internal() { + let error = LifecycleError::Storage(StoreError::Kv("invalid request encoding".into())); + assert_eq!(map_error(&error).0, FBChunkdbRetCode::Internal); + let unknown = LifecycleError::Storage(StoreError::OutcomeUnknown("quorum unavailable".into())); + assert_eq!(map_error(&unknown).0, FBChunkdbRetCode::Internal); + } +} diff --git a/app/crowdb-chunkdb/src/storage.rs b/app/crowdb-chunkdb/src/storage.rs index 7fa124866..80f1f17a7 100644 --- a/app/crowdb-chunkdb/src/storage.rs +++ b/app/crowdb-chunkdb/src/storage.rs @@ -42,6 +42,8 @@ pub enum StoreError { Route(#[from] crate::routing::RouteError), #[error("kv client error: {0}")] Kv(String), + #[error("kv write outcome is unknown: {0}")] + OutcomeUnknown(String), #[error("serialization error: {0}")] Serde(String), #[error("chunk is outside this runtime domain or service slot authority")] @@ -117,7 +119,7 @@ impl ChunkStore { Err(crowdb_kv_client::Error::OutcomeUnknown) => match self.get_chunk(id).await { Ok(stored) if stored == *chunk => Ok(()), Ok(_) | Err(StoreError::ChunkNotFound) => Err(StoreError::Conflict), - Err(error) => Err(error), + Err(error) => Err(StoreError::OutcomeUnknown(error.to_string())), }, Err(crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::CasBusy) => { Err(StoreError::ChunkAlreadyExists) diff --git a/container/crowdb-monitor/src/node/runtime.rs b/container/crowdb-monitor/src/node/runtime.rs index 11148c244..5aeef16fc 100644 --- a/container/crowdb-monitor/src/node/runtime.rs +++ b/container/crowdb-monitor/src/node/runtime.rs @@ -297,11 +297,17 @@ fn service(profile: &DeploymentProfile, id: &str) -> Result { fn web_service(profile: &DeploymentProfile, monitor_port: u16) -> Result { let root = &profile.paths.data_root; let mut web = service(profile, "web")?; + let web_port = profile + .public_endpoints + .iter() + .find(|endpoint| endpoint.id == "web") + .map(|endpoint| endpoint.port) + .ok_or("web public endpoint is absent from node profile")?; web.args = vec![ "--bind".into(), "0.0.0.0".into(), "--port".into(), - "9090".into(), + web_port.to_string(), "--runtime-dir".into(), root.join("console").to_string_lossy().into_owned(), "--node-monitor".into(), diff --git a/container/crowdb-monitor/src/node/runtime/health.rs b/container/crowdb-monitor/src/node/runtime/health.rs index f5377daf9..811fb00e2 100644 --- a/container/crowdb-monitor/src/node/runtime/health.rs +++ b/container/crowdb-monitor/src/node/runtime/health.rs @@ -14,6 +14,7 @@ pub(super) struct NodeHealth { store: StatusStore, status: MonitorStatus, attempts: BTreeMap, + probe_failures: BTreeMap, probes: crate::ProbeExecutor, } @@ -24,6 +25,7 @@ impl NodeHealth { store: StatusStore::new(&profile.paths.run_root)?, status: MonitorStatus::new(identity.uuid(), MonitorPhase::Initializing), attempts: BTreeMap::new(), + probe_failures: BTreeMap::new(), probes: crate::ProbeExecutor::new(true)?, }) } @@ -127,16 +129,36 @@ impl NodeHealth { } }; for intent in &intents { - if !super::services::pending(profile, intent, processes)? { + let pending = super::services::pending(profile, intent, processes)?; + let restart_unhealthy = if !pending + && matches!( + intent.action, + crowdb_protocol::mgmt::node::NodeServiceAction::Start + | crowdb_protocol::mgmt::node::NodeServiceAction::Restart + ) { + self.restart_if_unhealthy(profile, intent, processes).await? + } else { + false + }; + if !pending && !restart_unhealthy { continue; } if let Err(error) = super::services::dependencies_ready(profile, intent, &intents).await { tracing::debug!(service = %intent.service_id, %error, "service recovery waits for dependencies"); continue; } + if restart_unhealthy { + let process_id = if intent.kind == "kv" { + "kv" + } else { + &intent.service_id + }; + processes.stop(process_id, Duration::from_secs(10)).await?; + } match super::services::execute(profile, processes, intent.clone()).await { Ok(_) => { *self.attempts.entry(intent.service_id.clone()).or_default() += 1; + self.probe_failures.remove(&intent.service_id); } Err(error) => eprintln!("service recovery deferred for {}: {error}", intent.service_id), } @@ -144,6 +166,42 @@ impl NodeHealth { Ok(()) } + async fn restart_if_unhealthy( + &mut self, + profile: &DeploymentProfile, + intent: &crowdb_protocol::mgmt::node::NodeServiceIntent, + processes: &mut ProcessManager, + ) -> Result { + let process_id = if intent.kind == "kv" { + "kv" + } else { + &intent.service_id + }; + if !processes.owns(process_id) || !processes.alive(process_id)? { + self.probe_failures.remove(&intent.service_id); + return Ok(false); + } + let service = match super::probes::for_intent(profile, intent) { + Ok(service) => service, + Err(error) => { + tracing::debug!(service = %intent.service_id, %error, "service health probe unavailable"); + return Ok(false); + } + }; + if self + .probes + .probe_service(&service, &BTreeMap::new()) + .await + .is_ok() + { + self.probe_failures.remove(&intent.service_id); + return Ok(false); + } + let failures = self.probe_failures.entry(intent.service_id.clone()).or_default(); + *failures = failures.saturating_add(1); + Ok(*failures >= service.probe.failure_threshold) + } + fn restart_allowed(&mut self, id: &str) -> Result<()> { let attempts = self.attempts.entry(id.into()).or_default(); if *attempts >= 5 { diff --git a/container/crowdb-monitor/src/node/runtime/services/test_support.rs b/container/crowdb-monitor/src/node/runtime/services/test_support.rs index cf0d68968..da898a879 100644 --- a/container/crowdb-monitor/src/node/runtime/services/test_support.rs +++ b/container/crowdb-monitor/src/node/runtime/services/test_support.rs @@ -92,7 +92,12 @@ impl NodeServiceForTests { /// # Errors /// Propagates authority, journal and process observation errors. pub async fn reconcile_for_tests(&mut self) -> Result<()> { - super::super::health::NodeHealth::new(&self.profile)? + if self.health.is_none() { + self.health = Some(super::super::health::NodeHealth::new(&self.profile)?); + } + self.health + .as_mut() + .ok_or("missing test health worker")? .reconcile_services(&self.profile, &mut self.processes) .await } diff --git a/container/crowdb-monitor/src/node_args.rs b/container/crowdb-monitor/src/node_args.rs index ef401d7e9..7839aec30 100644 --- a/container/crowdb-monitor/src/node_args.rs +++ b/container/crowdb-monitor/src/node_args.rs @@ -45,6 +45,8 @@ pub(crate) struct NodeRunArgs { pub(crate) async fn run(args: NodeRunArgs) -> Result<(), Box> { let inputs = resolve(&args)?; + let offset = port_offset()?; + let monitor_port = shifted_port(9095, offset)?; let addresses = if_addrs::get_if_addrs()? .into_iter() .filter(|address| args.interface.contains(&address.name)) @@ -52,11 +54,11 @@ pub(crate) async fn run(args: NodeRunArgs) -> Result<(), Box Result<(), Box Result> { + let value = std::env::var("CROWDB_PORT_OFFSET").unwrap_or_default(); + if value.is_empty() { + return Ok(0); + } + value + .parse::() + .map_err(|_| "CROWDB_PORT_OFFSET must be a non-negative integer fitting a port".into()) +} + +fn shifted_port(base: u16, offset: u16) -> Result> { + base.checked_add(offset) + .ok_or_else(|| "CROWDB_PORT_OFFSET shifts a listener beyond port 65535".into()) +} + pub(crate) fn accept(args: &NodeRunArgs) -> Result<&'static str, Box> { let inputs = resolve(args)?; let policy = crowdb_monitor::accept_node_policy(&inputs.data_root, &inputs.file, &inputs.explicit)?; @@ -147,3 +164,14 @@ fn read_inputs( } Ok(toml::from_str(&content)?) } + +#[cfg(test)] +mod tests { + use super::shifted_port; + + #[test] + fn shifted_ports_are_bounded() { + assert_eq!(shifted_port(9095, 100).unwrap(), 9195); + assert!(shifted_port(u16::MAX, 1).is_err()); + } +} diff --git a/container/crowdb-monitor/tests/node_service_readiness_test.rs b/container/crowdb-monitor/tests/node_service_readiness_test.rs index 2a84e6cdc..dc3501c11 100644 --- a/container/crowdb-monitor/tests/node_service_readiness_test.rs +++ b/container/crowdb-monitor/tests/node_service_readiness_test.rs @@ -220,3 +220,138 @@ async fn recovery_waits_for_live_dependencies_without_consuming_launches() { server.abort(); } } + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn health_restarts_a_completed_service_after_consecutive_probe_failures() { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("service-health-restart"); + let root = directory.path().join("data"); + std::fs::create_dir_all(&root).unwrap(); + let request = NodeSetupRequest { + version: 1, + discovery_id: NodeIdentity::load_or_create(&root).unwrap().uuid().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + choices: crowdb_protocol::mgmt::node::setup::NodeSetupChoices::default(), + }; + let authority = node_setup::TestAuthority::start(&root, &request).await; + let mut profile = DeploymentProfile::load( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml"), + ) + .unwrap(); + profile.paths.data_root = root.clone(); + profile.paths.install_root = directory.path().to_path_buf(); + profile.paths.bin_root = directory.path().join("bin"); + profile.paths.template_root = directory.path().join("templates"); + profile.paths.log_root = root.join("log"); + profile.paths.run_root = directory.path().join("run"); + std::fs::create_dir_all(&profile.paths.bin_root).unwrap(); + std::fs::create_dir_all(&profile.paths.template_root).unwrap(); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + std::fs::create_dir_all(&profile.paths.run_root).unwrap(); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + let unhealthy = root.join("unhealthy"); + let invocations = root.join("invocations"); + let program = profile.paths.bin_root.join("managed-service"); + std::fs::write( + &program, + format!( + r#"#!/bin/sh +count=0 +if [ -f '{invocations}' ]; then count=$(cat '{invocations}'); fi +count=$((count + 1)) +echo "$count" > '{invocations}' +health_file='' +if [ "$count" -eq 1 ]; then health_file='{unhealthy}'; fi +exec python3 - "{port}" "$health_file" <<'PY' +from http.server import BaseHTTPRequestHandler, HTTPServer +import os +import sys + +port = int(sys.argv[1]) +health_file = sys.argv[2] + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + status = 503 if health_file and os.path.exists(health_file) else 200 + self.send_response(status) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *_args): + pass + +HTTPServer(("127.0.0.1", port), Handler).serve_forever() +PY +"#, + invocations = invocations.display(), + unhealthy = unhealthy.display(), + port = port, + ), + ) + .unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap(); + for service in &mut profile.services { + service.program = profile.paths.bin_root.join("unused-service"); + } + let child = profile + .services + .iter_mut() + .find(|service| service.id == "chunkdb") + .unwrap(); + child.program = program; + child.dependencies = vec!["kv".into()]; + child.probe.failure_threshold = 2; + profile.validate().unwrap(); + + let intent = NodeServiceIntent { + cluster_id: authority.binding.bootstrap.cluster_id.clone(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: authority.binding.node_id, + service_id: "chunkdb-2".into(), + kind: "chunkdb".into(), + action: NodeServiceAction::Start, + configuration: format!("[server]\nhttp_listen_addr = '127.0.0.1:{port}'\n"), + environment: std::collections::BTreeMap::default(), + }; + let key = b"/deployment/services/2/chunkdb-2"; + authority + .kv + .put(0, 0, key, &serde_json::to_vec(&intent).unwrap(), None) + .await + .unwrap(); + + let mut worker = NodeServiceForTests::new_for_tests(profile).await.unwrap(); + let first = worker.execute_for_tests(intent.clone()).await.unwrap(); + let first_pid = first["pid"].as_u64().unwrap(); + std::fs::write(&unhealthy, b"probe failure").unwrap(); + worker.reconcile_for_tests().await.unwrap(); + assert_eq!( + std::fs::read_to_string(&invocations).unwrap().trim(), + "1", + "one failed probe must not restart a live service" + ); + worker.reconcile_for_tests().await.unwrap(); + let state: serde_json::Value = + serde_json::from_slice(&std::fs::read(root.join("services/chunkdb-2/execution.json")).unwrap()) + .unwrap(); + assert_eq!(state["complete"], true); + assert_ne!(state["pid"].as_u64().unwrap(), first_pid); + assert_eq!( + std::fs::read_to_string(&invocations).unwrap().trim(), + "2", + "the threshold must restart the listener even while its child remains alive" + ); + + let mut stop = intent; + stop.operation_id = uuid::Uuid::new_v4().to_string(); + stop.action = NodeServiceAction::Stop; + authority + .kv + .put(0, 0, key, &serde_json::to_vec(&stop).unwrap(), None) + .await + .unwrap(); + worker.execute_for_tests(stop).await.unwrap(); +} diff --git a/container/single-node-container/entrypoint.sh b/container/single-node-container/entrypoint.sh index 46f84cb22..8fa74c3bd 100644 --- a/container/single-node-container/entrypoint.sh +++ b/container/single-node-container/entrypoint.sh @@ -60,18 +60,29 @@ if test "$port_offset" -gt 0; then offset_templates=$offset_root/templates mkdir -p "$offset_templates" cp "$profile" "$offset_root/profile.toml" - cp /opt/crowdb/etc/templates/*.toml "$offset_templates/" + if test -d /opt/crowdb/etc/templates; then + for template in /opt/crowdb/etc/templates/*.toml; do + test -f "$template" || continue + cp "$template" "$offset_templates/" + done + fi # The preview profile is intentionally self-contained. Shift every # declared listener and endpoint together so host-network containers can # coexist while retaining the same relative service layout. - for base in 9090 9091 9092 10000 10100 10101 11000 11100 11200 12100 12200 13000 15100 15200; do + for base in 9090 9091 9092 9093 9094 9095 10000 10100 10101 11000 11100 11200 12100 12200 13000 15100 15200; do next=$((base + port_offset)) - sed -i -E "s/(^|[^0-9])${base}([^0-9]|$)/\\1${next}\\2/g" \ - "$offset_root/profile.toml" "$offset_templates"/*.toml + sed -i -E "s/(^|[^0-9])${base}([^0-9]|$)/\\1${next}\\2/g" "$offset_root/profile.toml" + if test -d "$offset_templates"; then + for template in "$offset_templates"/*.toml; do + test -f "$template" || continue + sed -i -E "s/(^|[^0-9])${base}([^0-9]|$)/\\1${next}\\2/g" "$template" + done + fi done sed -i "s#/opt/crowdb/etc/templates#$offset_templates#g" "$offset_root/profile.toml" profile="$offset_root/profile.toml" fi +ssh_port=$((2222 + port_offset)) /opt/crowdb/bin/crowdb-monitor validate "$profile" if test -n "${CROWDB_STARTUP_MODE:-}"; then @@ -149,7 +160,7 @@ chmod 600 /opt/crowdb/data/ssh/authorized_keys chown -R crowdb:crowdb /opt/crowdb/data/ssh /opt/crowdb/run if ! test -e /opt/crowdb/.ssh; then ln -s /opt/crowdb/data/ssh /opt/crowdb/.ssh; fi cat > /opt/crowdb/run/sshd_config < --> + + +# Two Physical Node Validation Plan + +Upstream: [deployment design](../design/deploy/design-crowdb-deploy.md), +[OCI image guide](../design/deploy/design-crowdb-oci-image.md), +[KV quorum design](../design/kv/design-crowdb-kv-group0.md). + +Goal: validate the current CI reliability fixes on two real Linux hosts using +one reproducible CROWDB cluster, with cross-host discovery, quorum writes, +service readiness recovery, transient KV quorum loss during allocation, and +owned cleanup evidence. + +## Topology and limits + +The test uses three logical CROWDB voters because production Group 0 accepts +one or three initial voters. Two logical voters run on `192.168.31.129` and one +on `192.168.31.184`; the physical-host identity is recorded separately from +the logical node identity. This proves cross-host quorum and preserves quorum +when the single remote logical voter is stopped. It does not claim that the +two-voter host can fail as a whole; losing that host removes two of three +voters and is an expected quorum-loss observation. + +- Physical host A: `u24-amd`, `192.168.31.129`, Linux 6.8, Docker 29.8.1. +- Physical host B: `cj7960`, `192.168.31.184`, Linux 6.11, Docker 28.3.3. +- Inventory evidence: `.crowdb-runtime/artifacts/two-physical-nodes-20261030T010205Z/run-manifest.json`. +- Bidirectional ICMP and SSH/22 reachability passed; CROWDB service ports are + currently closed on both hosts. +- Remote account: `cj`; credentials are used only through SSH/`sshpass` and + are never written to the plan, logs, environment artifacts, or reports. +- Required source/image identity: the current checkout plus a newly built + immutable OCI image; the existing images differ in source revisions and are + not acceptable for the final run. +- Network: direct host addresses on `192.168.31.0/24`; all advertised + management, KV, DiskDB, DiskIO, ChunkDB, Chunk-KV, Access, Web and SSH + endpoints must be reachable from both hosts. Every listener gets a unique + port offset because host A owns two logical nodes. + +## Safety gates + +- Only run containers, data directories, bridges and logs labeled with this + run's UUID. Never reuse an existing CROWDB cluster or data root. +- Capture `docker inspect`, image receipt, host inventory, endpoint map and + service logs before cleanup. Cleanup stops and removes only this run's + containers, volumes, networks and remote directories. +- Do not change system firewall, routes, Docker daemon settings, kernel + parameters, physical NIC settings or shared services. A missing port or + privilege is a blocked prerequisite, not a reason to weaken the test. +- Keep the first workload small and bounded. Do not add retries or lengthen + readiness deadlines to hide a failure; record the first failing operation, + logs and persisted status, then stop the affected phase. + +## Phase 1: inventory and owned fixture + +- [x] **Record host inventory**: capture hostnames, IP/interface selection, + kernel, Docker, CPU/memory, disk space, image presence, and bidirectional + SSH/TCP reachability. Files: this plan, run manifest and artifact directory. +- [x] **Add host-network port offsets**: extend the existing + `CROWDB_PORT_OFFSET` path to shift monitor, Web, Access and SSH listeners so + two logical nodes can coexist on one physical host; verify the offset helper + and shell syntax. Files: `container/single-node-container/entrypoint.sh`, + `container/crowdb-monitor/src/node_args.rs`, + `container/crowdb-monitor/src/node/runtime.rs`. +- [x] **Build and distribute exact image**: build with `pixi run oci-build`, + record source tree and image digests, transfer/import the same archive on + both hosts, and verify the digest plus required runtime labels. +- [ ] **Prepare isolated roots**: create one run UUID, per-host data/log roots, + a unique Docker network or host-network port map, and a machine-readable + endpoint map. Verify no endpoint is already owned. +- [ ] **Start three logical nodes**: launch two offset nodes on host A and one + base node on host B with the same cluster policy, `initial_voters = 3`, + distinct data roots and explicit physical-host IDs. Verify all three + candidates are visible before admission. + +## Phase 2: bootstrap and normal cross-host behavior + +- [ ] **Admit and bootstrap Group 0**: create one rack record, admit the three + discovered nodes, initialize the cluster, and wait for all three KV health + observations. Record node IDs, discovery IDs, cluster ID and remote replica + endpoints. +- [ ] **Verify dependency readiness**: wait for KV, DiskDB, DiskIO, ChunkDB, + Chunk-KV and Access readiness on all logical nodes. Confirm the monitor + status is `ready` and identities/PIDs are stable after one health interval. +- [ ] **Run bounded data corpus**: perform KV put/get/batch, S3 bucket/object + put/get, one Iceberg metadata round trip and one ChunkDB small allocation; + verify exact bytes and read through both physical-host endpoints. + +## Phase 3: failure and recovery cases + +- [ ] **Remote logical-node stop**: stop only host B's CROWDB node. Verify the + two host-A voters retain Group 0 quorum, linearizable KV writes continue, + existing S3 reads continue, and status identifies the stopped node. +- [ ] **Remote node restart**: restart host B's node with the same data root. + Verify the original discovery identity, cluster ID, service identities and + persisted object are retained; wait for full readiness and repeat the data + corpus. +- [ ] **Managed listener failure**: stop or fault only the selected Chunk-KV + listener while leaving its process ownership observable. Verify consecutive + readiness failures trigger the monitor's bounded restart, the PID changes, + Access becomes ready again, and no caller timeout is modified. +- [ ] **Transient allocation quorum loss**: during a bounded small allocation, + pause one KV voter long enough to produce the known `leadership quorum + unavailable` response, restore it, and verify ChunkDB allocation recovers + through the allocation-only transient path. Verify outcome-unknown cases + remain terminal and no duplicate chunk metadata is published. +- [ ] **Expected two-voter host loss**: stop host A's two logical nodes only + after the preceding recovery cases pass. Record the expected Group 0 quorum + loss and the exact readiness/error surfaces, then restore both nodes and + verify durable state recovery. This is an observation case, not a pass claim + for availability while two of three voters are down. + +## Phase 4: evidence and cleanup + +- [ ] **Collect evidence**: save commands, source/image identity, endpoint map, + host inventory, cluster status snapshots, monitor events, service logs, + process identities, workload results and first-failure diagnostics. +- [ ] **Run final health/data gate**: after all recovery cases, require all + three logical nodes and their enabled services ready, repeat KV and object + reads from both physical hosts, and check no unexpected process/container + remains. +- [ ] **Cleanup owned resources**: stop/remove only this run's containers, + networks, volumes and remote directories; verify the endpoints are closed + and retain artifacts outside deleted data roots. +- [ ] **Report result**: mark each case pass, expected observation, failed or + blocked with evidence. Do not mark the run complete if the image, placement, + or cross-host path was not proven. + +## Consolidated files and commands + +- Plan/tracking: `doc/working/plan-two-physical-nodes.md`, + `doc/working/track-two-physical-nodes.md`. +- Image: `container/oci-image/build.py`, `container/oci-image/run-node.py`, + `container/oci-image/control.py`. +- Existing lifecycle references: `e2e-test/profiles/node-bootstrap.py`, + `e2e-test/profiles/containerd-cluster.py`, `e2e-test/console_native_fixture.py`. +- Local quality gates: `pixi run cargo fmt --all -- --check`, affected Rust + tests/clippy, and `pixi run oci-build`. +- Remote checks: `ssh`, `docker`, `docker inspect`, `docker logs`, and the + repository's CLI/HTTP cluster status commands, all recorded in the run + manifest. + +## Verification groups + +- Unit: current monitor and ChunkDB regression tests already added for the CI + failures. +- Integration: image/launcher validation, endpoint ownership, discovery and + SSH admission. +- Physical E2E: three logical voters over two physical hosts, normal writes, + one-voter loss/restart, managed listener recovery, transient allocation + recovery, expected two-voter quorum loss and durable recovery. diff --git a/doc/working/track-two-physical-nodes.md b/doc/working/track-two-physical-nodes.md new file mode 100644 index 000000000..9d1506262 --- /dev/null +++ b/doc/working/track-two-physical-nodes.md @@ -0,0 +1,43 @@ + + + +# Two Physical Node Test Track + +Plan: [plan-two-physical-nodes.md](plan-two-physical-nodes.md) + +Run ID: `two-physical-nodes-20261030T010205Z` + +## Preconditions + +- [x] Local host inventory captured. +- [x] Remote host inventory captured through SSH. +- [x] Direct peer reachability and SSH reachability verified. +- [x] Host-network port-offset support verified by focused test, clippy and + shell syntax checks. +- [x] Current source/image identity recorded and identical on both hosts. +- [ ] Isolated run UUID, roots, network and endpoint map created. + +## Bootstrap + +- [ ] Three logical nodes launched: host A ×2, host B ×1. +- [ ] Three candidates discovered and admitted. +- [ ] Group 0 initialized with three voters. +- [ ] All required services ready on all logical nodes. +- [ ] Baseline KV/S3/Iceberg/ChunkDB corpus passed. + +## Recovery matrix + +- [ ] Stop host B logical node: two-voter quorum remains available. +- [ ] Restart host B logical node: identity and data retained. +- [ ] Managed Chunk-KV listener failure: monitor restarts unhealthy service. +- [ ] Transient allocation quorum loss: allocation-only recovery succeeds. +- [ ] Stop host A's two logical nodes: expected quorum loss recorded. +- [ ] Restore host A and host B: full cluster and data recovery passed. + +## Evidence and exit + +- [ ] Status snapshots, process identities and monitor events saved. +- [ ] Service/container logs saved for every phase. +- [ ] Final cross-host reads and health gate passed. +- [ ] Owned resources cleaned; endpoints closed. +- [ ] Result report completed with pass/expected/failed/blocked classification. diff --git a/lib/crowdb-chunkdb-client/src/retry.rs b/lib/crowdb-chunkdb-client/src/retry.rs index b66dde139..807249f10 100644 --- a/lib/crowdb-chunkdb-client/src/retry.rs +++ b/lib/crowdb-chunkdb-client/src/retry.rs @@ -123,8 +123,13 @@ impl ChunkdbClient { }) { Ok(response) => return Ok(response), Err( - error @ (ChunkdbClientError::NotMyRange(_) | ChunkdbClientError::ConnectFailed(_)), + error @ (ChunkdbClientError::NotMyRange(_) + | ChunkdbClientError::ConnectFailed(_) + | ChunkdbClientError::Unavailable(_)), ) => { + // The server reports KV quorum loss as Unavailable + // only after allocation has rolled back its physical + // strips, so retrying starts a fresh safe allocation. safe_retry = Some(error); } Err(error) => return Err(error), From 39440d023d7fb162dd61c377e8eb17eefa8c4c9a Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 30 Oct 2026 12:33:47 +0800 Subject: [PATCH 03/24] Support multiple host-network node containers --- app/crowdb-web/src/node/admission.rs | 17 ++++- app/crowdb-web/src/node/authority.rs | 14 +++- container/crowdb-monitor/src/lib.rs | 7 +- container/crowdb-monitor/src/node.rs | 2 +- .../src/node/runtime/cleanup.rs | 3 +- .../src/node/runtime/control.rs | 3 +- .../crowdb-monitor/src/node/runtime/kv.rs | 30 ++++++-- .../src/node/runtime/recovery/publication.rs | 58 +++++++++------ .../node/runtime/setup/execution/services.rs | 13 +++- container/crowdb-monitor/src/node/startup.rs | 2 +- .../crowdb-monitor/src/node/startup/policy.rs | 44 ++++++++++- container/crowdb-monitor/src/node_args.rs | 7 +- container/oci-image/run-node.py | 24 +++++- container/oci-image/runtime.py | 3 +- container/oci-image/tests/host_inputs_test.py | 8 ++ doc/working/plan-two-physical-nodes.md | 73 +++++++++++-------- doc/working/track-two-physical-nodes.md | 20 +++-- .../src/mgmt/node/deployment.rs | 7 +- lib/crowdb-protocol/src/mgmt/node/setup.rs | 15 ++++ .../src/mgmt/node/setup/validation.rs | 6 -- .../tests/node_deployment_test.rs | 6 +- lib/crowdb-protocol/tests/node_setup_test.rs | 14 ++-- 22 files changed, 264 insertions(+), 112 deletions(-) diff --git a/app/crowdb-web/src/node/admission.rs b/app/crowdb-web/src/node/admission.rs index a1b79985a..0b6edaaaa 100644 --- a/app/crowdb-web/src/node/admission.rs +++ b/app/crowdb-web/src/node/admission.rs @@ -189,6 +189,7 @@ async fn admit_observed( .map_err(map_config_err)?; } save(&path, &record)?; + let setup = body.setup.clone(); super::setup::accept(state, &record, &target, body.setup, body.expected_formation).await?; target.ssh_password = None; target.ssh_key = None; @@ -196,7 +197,7 @@ async fn admit_observed( if let Some(binding) = &binding { confirm_admission(state, binding, &key, &target, &mut record).await?; } - update_config(state, &record, target)?; + update_config(state, &record, target, setup.as_ref())?; save(&path, &record)?; state.persist().map_err(map_persist_err)?; super::preflight::finish(state, &record)?; @@ -323,7 +324,12 @@ pub(crate) fn save(path: &Path, record: &impl serde::Serialize) -> Result<(), Fa .map_err(|error| err_502(error.to_string())) } -pub(super) fn update_config(state: &AppState, record: &NodeRecord, target: NodeEntry) -> Result<(), Failure> { +pub(super) fn update_config( + state: &AppState, + record: &NodeRecord, + target: NodeEntry, + setup: Option<&crowdb_protocol::mgmt::node::setup::NodeSetupChoices>, +) -> Result<(), Failure> { { let mut config = state.config.write().map_err(|error| err_502(error.to_string()))?; if let Some(node) = config.nodes.iter_mut().find(|node| node.id == record.node_id) { @@ -332,12 +338,15 @@ pub(super) fn update_config(state: &AppState, record: &NodeRecord, target: NodeE config.nodes.push(target); } if config.server_for_node(record.node_id).is_none() { + let (management_port, rpc_port) = setup + .and_then(crowdb_protocol::mgmt::node::setup::NodeSetupChoices::paxos_kv_ports) + .unwrap_or((10000, 10100)); let mut server = ServerEntry::new( format!("kv-{}", record.node_id), - format!("http://{}:10000", bracket_host(&record.host)), + format!("http://{}:{management_port}", bracket_host(&record.host)), ); server.node_id = Some(record.node_id); - server.rpc_url = Some(format!("{}:10100", bracket_host(&record.host))); + server.rpc_url = Some(format!("{}:{rpc_port}", bracket_host(&record.host))); config.servers.push(server); } } diff --git a/app/crowdb-web/src/node/authority.rs b/app/crowdb-web/src/node/authority.rs index 28016e27f..c4b077e2c 100644 --- a/app/crowdb-web/src/node/authority.rs +++ b/app/crowdb-web/src/node/authority.rs @@ -309,8 +309,18 @@ async fn confirmed_inner( .iter_mut() .find(|server| server.node_id == Some(node.node_id)) { - server.url = format!("http://{host}:10000"); - server.rpc_url = Some(format!("{host}:10100")); + let management_port = reqwest::Url::parse(&server.url) + .ok() + .and_then(|url| url.port_or_known_default()) + .unwrap_or(10000); + let rpc_port = server + .rpc_url + .as_deref() + .and_then(|url| reqwest::Url::parse(&format!("http://{url}")).ok()) + .and_then(|url| url.port_or_known_default()) + .unwrap_or(10100); + server.url = format!("http://{host}:{management_port}"); + server.rpc_url = Some(format!("{host}:{rpc_port}")); } else { let mut server = crowdb_console_shared::config::ServerEntry::new( format!("kv-{}", node.node_id), diff --git a/container/crowdb-monitor/src/lib.rs b/container/crowdb-monitor/src/lib.rs index e4f72233e..e0066d866 100644 --- a/container/crowdb-monitor/src/lib.rs +++ b/container/crowdb-monitor/src/lib.rs @@ -42,9 +42,10 @@ pub use node::{ NodeServiceForTests, NodeSetupForTests, SuppliedOperationsForTests, }; pub use node::{ - accept_node_policy, control_node, discover_disks, read_node_policy, run_node, run_node_with_inputs, - serve_node_management, CandidateCache, DiscoveryConfig, DiscoveryError, NodeDiscovery, NodeIdentity, - NodeIdentityError, NodeManagementError, NodeRuntimeConfig, NODE_SERVICE_TYPE, + accept_node_policy, accept_node_policy_with_port_offset, control_node, discover_disks, read_node_policy, + run_node, run_node_with_inputs, serve_node_management, CandidateCache, DiscoveryConfig, DiscoveryError, + NodeDiscovery, NodeIdentity, NodeIdentityError, NodeManagementError, NodeRuntimeConfig, + NODE_SERVICE_TYPE, }; #[cfg(feature = "test-util")] pub use preview::{preview_step_names_for_tests, run_preview_for_tests, PreviewError}; diff --git a/container/crowdb-monitor/src/node.rs b/container/crowdb-monitor/src/node.rs index 44bdd6239..b4f467c8c 100644 --- a/container/crowdb-monitor/src/node.rs +++ b/container/crowdb-monitor/src/node.rs @@ -45,7 +45,7 @@ pub use runtime::{control_node, run_node, run_node_with_inputs, NodeRuntimeConfi pub use runtime::{publish_file_for_tests, resume_file_operations_for_tests}; #[cfg(feature = "test-util")] pub use startup::accept_startup_mode_for_tests; -pub use startup::{accept_node_policy, read_node_policy}; +pub use startup::{accept_node_policy, accept_node_policy_with_port_offset, read_node_policy}; pub use management::{serve_node_management, NodeManagementError}; diff --git a/container/crowdb-monitor/src/node/runtime/cleanup.rs b/container/crowdb-monitor/src/node/runtime/cleanup.rs index 4af24c9da..65fce0673 100644 --- a/container/crowdb-monitor/src/node/runtime/cleanup.rs +++ b/container/crowdb-monitor/src/node/runtime/cleanup.rs @@ -47,8 +47,9 @@ pub(super) async fn execute( .no_proxy() .timeout(Duration::from_secs(30)) .build()?; + let (management_port, _) = super::kv::ports(profile)?; let response = client - .post("http://127.0.0.1:10000/system/cleanup") + .post(format!("http://127.0.0.1:{management_port}/system/cleanup")) .json(&json!({"bootstrap": bootstrap, "confirm_delete_system_store": true})) .send() .await; diff --git a/container/crowdb-monitor/src/node/runtime/control.rs b/container/crowdb-monitor/src/node/runtime/control.rs index baa6b7a79..5c5114abf 100644 --- a/container/crowdb-monitor/src/node/runtime/control.rs +++ b/container/crowdb-monitor/src/node/runtime/control.rs @@ -137,7 +137,8 @@ async fn bootstrap( let accepted = preparation::accept(root, node_id, bootstrap, manifest, credentials, admission)?; super::recovery::prepare_admitted(root, &accepted).await?; start_kv(profile, processes, &accepted).await?; - Ok(json!({"node_id": node_id, "management_port": 10000, "rpc_port": 10100})) + let (management_port, rpc_port) = kv::ports(profile)?; + Ok(json!({"node_id": node_id, "management_port": management_port, "rpc_port": rpc_port})) } _ => Err("unsupported bootstrap control request".into()), } diff --git a/container/crowdb-monitor/src/node/runtime/kv.rs b/container/crowdb-monitor/src/node/runtime/kv.rs index aa92efc66..8af9ed7de 100644 --- a/container/crowdb-monitor/src/node/runtime/kv.rs +++ b/container/crowdb-monitor/src/node/runtime/kv.rs @@ -55,6 +55,19 @@ pub(super) fn node_profile(config: &super::NodeRuntimeConfig) -> Result Result<(u16, u16)> { + let policy = crate::node::read_node_policy(root)?.ok_or("accepted node policy is absent")?; + policy.setup.paxos_kv_ports().ok_or_else(|| { + "accepted node policy has no enabled Paxos KV listeners" + .to_owned() + .into() + }) +} + +pub(super) fn ports(profile: &DeploymentProfile) -> Result<(u16, u16)> { + ports_from_root(&profile.paths.data_root) +} + pub(super) async fn bind( profile: &DeploymentProfile, processes: &mut ProcessManager, @@ -99,18 +112,19 @@ pub(super) async fn start_kv( accepted: &AcceptedNode, ) -> Result<()> { ensure_process(profile, processes, accepted).await?; - wait_ready(processes).await + wait_ready(profile, processes).await } -async fn wait_ready(processes: &mut ProcessManager) -> Result<()> { +async fn wait_ready(profile: &DeploymentProfile, processes: &mut ProcessManager) -> Result<()> { let client = reqwest::Client::builder() .no_proxy() .timeout(Duration::from_secs(1)) .build()?; + let (management_port, _) = ports(profile)?; let deadline = tokio::time::Instant::now() + Duration::from_secs(10); loop { if client - .get("http://127.0.0.1:10000/health") + .get(format!("http://127.0.0.1:{management_port}/health")) .send() .await .is_ok_and(|reply| reply.status().is_success()) @@ -143,7 +157,7 @@ pub(super) async fn start_for_cleanup( accepted: &AcceptedNode, ) -> Result<()> { spawn(profile, processes, accepted).await?; - wait_ready(processes).await + wait_ready(profile, processes).await } async fn spawn( @@ -168,15 +182,19 @@ async fn spawn( .join("kv") .join(format!("node-{}", accepted.node_id)); std::fs::create_dir_all(&root)?; + let (management_port, rpc_port) = ports(profile)?; + let rpc_health_port = rpc_port + .checked_add(1) + .ok_or("Paxos KV RPC listener has no health port")?; kv.args = vec![ "--node-recovery-root".into(), profile.paths.data_root.to_string_lossy().into_owned(), "--root".into(), root.to_string_lossy().into_owned(), "--management-port".into(), - "10000".into(), + management_port.to_string(), "--ports".into(), - "10100,10101".into(), + format!("{rpc_port},{rpc_health_port}"), "--instance-id".into(), accepted.node_id.to_string(), "--node-id".into(), diff --git a/container/crowdb-monitor/src/node/runtime/recovery/publication.rs b/container/crowdb-monitor/src/node/runtime/recovery/publication.rs index dacd623ea..0e35f167f 100644 --- a/container/crowdb-monitor/src/node/runtime/recovery/publication.rs +++ b/container/crowdb-monitor/src/node/runtime/recovery/publication.rs @@ -21,28 +21,8 @@ pub(super) async fn reconcile( ) -> Result<()> { let local = super::identity(root)?.endpoint; let mut config = topology.clone(); - let mut seeds = Vec::new(); - for node in &mut config.nodes { - let Some(peer) = endpoints.members.iter().find(|peer| peer.node_id == node.id) else { - continue; - }; - let address = peer - .addresses - .iter() - .find(|address| address.is_ipv4()) - .or_else(|| peer.addresses.first()) - .ok_or("member address missing")?; - node.host = address.to_string(); - seeds.push(format!("http://{}", std::net::SocketAddr::new(*address, 10000))); - for server in config - .servers - .iter_mut() - .filter(|server| server.node_id == Some(node.id)) - { - server.url = format!("http://{}", std::net::SocketAddr::new(*address, 10000)); - server.rpc_url = Some(std::net::SocketAddr::new(*address, 10100).to_string()); - } - } + let (management_port, rpc_port) = super::super::kv::ports_from_root(root)?; + let seeds = apply_member_endpoints(&mut config, endpoints, management_port, rpc_port)?; let client = Arc::new(CrowdbKvClient::new(ClientConfig::new(seeds.clone()))); let GetOutcome::Found { value, .. } = client .get( @@ -117,6 +97,40 @@ pub(super) async fn reconcile( Ok(()) } +fn apply_member_endpoints( + config: &mut ConsoleConfig, + endpoints: &MemberEndpoints, + management_port: u16, + rpc_port: u16, +) -> Result> { + let mut seeds = Vec::new(); + for node in &mut config.nodes { + let Some(peer) = endpoints.members.iter().find(|peer| peer.node_id == node.id) else { + continue; + }; + let address = peer + .addresses + .iter() + .find(|address| address.is_ipv4()) + .or_else(|| peer.addresses.first()) + .ok_or("member address missing")?; + node.host = address.to_string(); + seeds.push(format!( + "http://{}", + std::net::SocketAddr::new(*address, management_port) + )); + for server in config + .servers + .iter_mut() + .filter(|server| server.node_id == Some(node.id)) + { + server.url = format!("http://{}", std::net::SocketAddr::new(*address, management_port)); + server.rpc_url = Some(std::net::SocketAddr::new(*address, rpc_port).to_string()); + } + } + Ok(seeds) +} + async fn rebind( root: &Path, client: &CrowdbKvClient, diff --git a/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs b/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs index 9304d989c..cb267f82f 100644 --- a/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs +++ b/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs @@ -90,7 +90,7 @@ pub(super) async fn advance( } step.phase = SetupStepPhase::Preparing; journal::save(&profile.paths.data_root, state)?; - let (success, value) = prepare(binding, state, kind).await?; + let (success, value) = prepare(profile, binding, state, kind).await?; let step = &mut state.status.services[index]; step.phase = if success { SetupStepPhase::Accepted @@ -122,17 +122,24 @@ fn name(kind: SetupServiceKind) -> &'static str { } async fn prepare( + profile: &DeploymentProfile, binding: &NodeBinding, state: &AcceptedSetup, kind: SetupServiceKind, ) -> Result<(bool, serde_json::Value)> { + let web_port = profile + .public_endpoints + .iter() + .find(|endpoint| endpoint.id == "web") + .map(|endpoint| endpoint.port) + .ok_or("node profile requires a Web endpoint")?; let mut response = reqwest::Client::builder() .no_proxy() .timeout(Duration::from_secs(30)) .build()? .post(format!( - "http://127.0.0.1:9090/api/nodes/{}/setup/prepare-service", - binding.node_id + "http://127.0.0.1:{web_port}/api/nodes/{}/setup/prepare-service", + binding.node_id, )) .json(&json!({"operation_id":state.request.operation_id,"kind":kind})) .send() diff --git a/container/crowdb-monitor/src/node/startup.rs b/container/crowdb-monitor/src/node/startup.rs index 8c451a7ae..e7be853fe 100644 --- a/container/crowdb-monitor/src/node/startup.rs +++ b/container/crowdb-monitor/src/node/startup.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; use super::durable; mod policy; -pub use policy::{accept_node_policy, read_node_policy}; +pub use policy::{accept_node_policy, accept_node_policy_with_port_offset, read_node_policy}; #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] diff --git a/container/crowdb-monitor/src/node/startup/policy.rs b/container/crowdb-monitor/src/node/startup/policy.rs index 9ef1767b6..c1397e595 100644 --- a/container/crowdb-monitor/src/node/startup/policy.rs +++ b/container/crowdb-monitor/src/node/startup/policy.rs @@ -34,14 +34,36 @@ pub fn accept_node_policy( root: &Path, file: &NodeDeploymentOverrides, explicit: &NodeDeploymentOverrides, +) -> io::Result { + accept_node_policy_with_port_offset(root, file, explicit, 0) +} + +/// Accept a fresh policy after shifting every service listener by `offset`. +/// +/// The offset is a launcher concern: it is applied once, before the policy is +/// sealed, and retained unchanged on every restart. Existing accepted roots +/// are never shifted again. +/// +/// # Errors +/// Returns an error when the root, policy inputs or shifted listener ports are +/// invalid, or when a retained policy does not match the supplied inputs. +pub fn accept_node_policy_with_port_offset( + root: &Path, + file: &NodeDeploymentOverrides, + explicit: &NodeDeploymentOverrides, + offset: u16, ) -> io::Result { if !crate::layout::is_clean_absolute(root) || !fs::symlink_metadata(root)?.is_dir() { return Err(io::Error::other("node policy root must be an absolute directory")); } let accepted = read_node_policy(root)?; - let selected = explicit + let mut selected = explicit .resolve(file, accepted.as_ref()) .map_err(io::Error::other)?; + if accepted.is_none() && offset != 0 { + shift_policy_ports(&mut selected, offset)?; + selected.validate().map_err(io::Error::other)?; + } if accepted.is_some() { validate_retained_identity(root)?; return Ok(selected); @@ -65,6 +87,26 @@ pub fn accept_node_policy( Ok(winner) } +fn shift_policy_ports(policy: &mut NodeDeploymentPolicy, offset: u16) -> io::Result<()> { + for service in &mut policy.setup.services { + for port in [ + &mut service.listeners.http_port, + &mut service.listeners.rpc_port, + &mut service.listeners.s3_port, + &mut service.listeners.dataset_port, + &mut service.listeners.health_port, + ] + .into_iter() + .flatten() + { + *port = port + .checked_add(offset) + .ok_or_else(|| io::Error::other("port offset exceeds 65535"))?; + } + } + Ok(()) +} + fn validate_retained_identity(root: &Path) -> io::Result<()> { if root.join("node-identity").try_exists()? { return Ok(()); diff --git a/container/crowdb-monitor/src/node_args.rs b/container/crowdb-monitor/src/node_args.rs index 7839aec30..e91bc8b21 100644 --- a/container/crowdb-monitor/src/node_args.rs +++ b/container/crowdb-monitor/src/node_args.rs @@ -85,7 +85,12 @@ fn shifted_port(base: u16, offset: u16) -> Result Result<&'static str, Box> { let inputs = resolve(args)?; - let policy = crowdb_monitor::accept_node_policy(&inputs.data_root, &inputs.file, &inputs.explicit)?; + let policy = crowdb_monitor::accept_node_policy_with_port_offset( + &inputs.data_root, + &inputs.file, + &inputs.explicit, + port_offset()?, + )?; Ok(match policy.mode { DeploymentMode::Local => "local", DeploymentMode::Production => "production", diff --git a/container/oci-image/run-node.py b/container/oci-image/run-node.py index 0c408143d..5aa9e5947 100755 --- a/container/oci-image/run-node.py +++ b/container/oci-image/run-node.py @@ -20,9 +20,10 @@ def digest_reference(reference): return digest.startswith("sha256:") and len(digest) == 71 and all(c in "0123456789abcdef" for c in digest[7:]) -# Fixed in-image listeners; host networking admits only one node per host. -HOST_PORTS = (2222, 7100, 7101, 7200, 7300, 7301, 7400, 7401, - 9090, 9091, 9092, 9093, 9094, 9095, 10000, 10100) +# Base listeners used by one node. Host-network instances add +# ``CROWDB_PORT_OFFSET`` to every listener before the policy is sealed. +HOST_PORTS = (2222, 7100, 7101, 7102, 7200, 7300, 7301, 7400, 7401, + 9090, 9091, 9092, 9093, 9094, 9095, 10000, 10100, 10101) def check_host_ports(ports=HOST_PORTS): @@ -37,6 +38,15 @@ def check_host_ports(ports=HOST_PORTS): raise ValueError(f"host node TCP port {port} is already occupied") from error +def shifted_ports(offset): + if not isinstance(offset, int) or offset < 0: + raise ValueError("port offset must be a non-negative integer") + ports = tuple(port + offset for port in HOST_PORTS) + if any(port > 65535 for port in ports): + raise ValueError("port offset shifts a node listener beyond port 65535") + return ports + + def validate(args, parser, check_ports=True): if (platform.system(), platform.machine()) != ("Linux", "x86_64"): parser.error("Linux amd64 host required; macOS VM backend is reserved") @@ -53,6 +63,10 @@ def validate(args, parser, check_ports=True): parser.error("CPU shares must be between 2 and 262144") if args.worker_threads is not None and args.worker_threads < 1: parser.error("worker thread count must be positive") + try: + ports = shifted_ports(getattr(args, "port_offset", 0)) + except ValueError as error: + parser.error(str(error)) available = os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") // (1024 * 1024) if not (0 < args.cpus <= (os.cpu_count() or 1)) or not 512 <= args.memory_mib <= available: parser.error("CPU and memory boundaries must fit the host; memory must be at least 512 MiB") @@ -83,7 +97,7 @@ def validate(args, parser, check_ports=True): try: socket.if_nametoindex(args.interface) if check_ports: - check_host_ports() + check_host_ports(ports) except (OSError, ValueError) as error: parser.error(str(error)) elif args.runtime != "docker": @@ -118,6 +132,8 @@ def main(): parser.add_argument("--seed", action="append", default=[]) parser.add_argument("--cluster", required=True) parser.add_argument("--initial-voters", type=int, choices=(1, 3), default=3) + parser.add_argument("--port-offset", type=int, default=0, + help="Add this offset to every host-network node listener") args = parser.parse_args() validate(args, parser) runtime = Runtime(args.runtime, args.address, "crowdb-" + args.cluster, args.snapshotter) diff --git a/container/oci-image/runtime.py b/container/oci-image/runtime.py index fe6b8b241..db627149d 100644 --- a/container/oci-image/runtime.py +++ b/container/oci-image/runtime.py @@ -119,7 +119,8 @@ def create(self, args): "-e", f"CROWDB_INITIAL_VOTERS={args.initial_voters}", "-e", "CROWDB_DEPLOYMENT_MODE=production", "-e", f"CROWDB_PHYSICAL_HOST_ID={args.physical_host_id}", - "-e", f"CROWDB_MANAGEMENT_INTERFACE={args.interface}"] + "-e", f"CROWDB_MANAGEMENT_INTERFACE={args.interface}", + "-e", f"CROWDB_PORT_OFFSET={getattr(args, 'port_offset', 0)}"] workers = getattr(args, "worker_threads", None) if workers is not None: command += ["-e", f"TOKIO_WORKER_THREADS={workers}"] diff --git a/container/oci-image/tests/host_inputs_test.py b/container/oci-image/tests/host_inputs_test.py index 05400046f..16c9df879 100644 --- a/container/oci-image/tests/host_inputs_test.py +++ b/container/oci-image/tests/host_inputs_test.py @@ -16,6 +16,14 @@ class HostInputsTest(unittest.TestCase): + def test_port_offset_shifts_every_host_listener_for_additional_instances(self): + shifted = run_node.shifted_ports(100) + self.assertEqual(shifted[0], 2322) + self.assertIn(10100, shifted) + self.assertIn(10200, shifted) + with self.assertRaisesRegex(ValueError, "beyond port 65535"): + run_node.shifted_ports(65535) + def test_runtime_logs_preserve_stderr_without_polluting_machine_readable_inspection(self): with patch('runtime.subprocess.run', return_value=SimpleNamespace( returncode=0,stdout='{"status":"running"}',stderr='startup failed on stderr')): diff --git a/doc/working/plan-two-physical-nodes.md b/doc/working/plan-two-physical-nodes.md index 42e6f1753..cc8d4a4c1 100644 --- a/doc/working/plan-two-physical-nodes.md +++ b/doc/working/plan-two-physical-nodes.md @@ -8,19 +8,19 @@ Upstream: [deployment design](../design/deploy/design-crowdb-deploy.md), [KV quorum design](../design/kv/design-crowdb-kv-group0.md). Goal: validate the current CI reliability fixes on two real Linux hosts using -one reproducible CROWDB cluster, with cross-host discovery, quorum writes, -service readiness recovery, transient KV quorum loss during allocation, and -owned cleanup evidence. +reproducible CROWDB clusters, with cross-host discovery, key-only admission, +host-network container composition, Group 0 bootstrap, service readiness +recovery, transient KV quorum loss during allocation, and owned cleanup +evidence. ## Topology and limits -The test uses three logical CROWDB voters because production Group 0 accepts -one or three initial voters. Two logical voters run on `192.168.31.129` and one -on `192.168.31.184`; the physical-host identity is recorded separately from -the logical node identity. This proves cross-host quorum and preserves quorum -when the single remote logical voter is stopped. It does not claim that the -two-voter host can fail as a whole; losing that host removes two of three -voters and is an expected quorum-loss observation. +The basic two-physical-node run uses one logical node per host and one initial +voter. It proves cross-host discovery, key-only admission and durable restart; +it does not claim two-voter quorum. The composition run uses three logical +voters on one host with listener offsets `0`, `200` and `400`; this validates +that a physical host can run multiple host-network containers. A later mixed +placement can put any of those logical nodes on the second host. - Physical host A: `u24-amd`, `192.168.31.129`, Linux 6.8, Docker 29.8.1. - Physical host B: `cj7960`, `192.168.31.184`, Linux 6.11, Docker 28.3.3. @@ -34,8 +34,9 @@ voters and is an expected quorum-loss observation. not acceptable for the final run. - Network: direct host addresses on `192.168.31.0/24`; all advertised management, KV, DiskDB, DiskIO, ChunkDB, Chunk-KV, Access, Web and SSH - endpoints must be reachable from both hosts. Every listener gets a unique - port offset because host A owns two logical nodes. + endpoints must be reachable from both hosts. Host-network listener offsets + use a stride of at least `200`: offset `100` collides with the adjacent + default KV RPC range (`10000/10100`). ## Safety gates @@ -57,28 +58,37 @@ voters and is an expected quorum-loss observation. kernel, Docker, CPU/memory, disk space, image presence, and bidirectional SSH/TCP reachability. Files: this plan, run manifest and artifact directory. - [x] **Add host-network port offsets**: extend the existing - `CROWDB_PORT_OFFSET` path to shift monitor, Web, Access and SSH listeners so - two logical nodes can coexist on one physical host; verify the offset helper - and shell syntax. Files: `container/single-node-container/entrypoint.sh`, + `CROWDB_PORT_OFFSET` path to shift monitor, Web, Access, KV, DiskDB, + DiskIO, ChunkDB, Chunk-KV and SSH listeners; persist shifted setup choices + and use them for Group 0 endpoint publication. Verify the launcher offset + validator and shell syntax. Files: `container/single-node-container/entrypoint.sh`, `container/crowdb-monitor/src/node_args.rs`, - `container/crowdb-monitor/src/node/runtime.rs`. + `container/crowdb-monitor/src/node/runtime/kv.rs`, + `container/oci-image/run-node.py`. - [x] **Build and distribute exact image**: build with `pixi run oci-build`, record source tree and image digests, transfer/import the same archive on - both hosts, and verify the digest plus required runtime labels. -- [ ] **Prepare isolated roots**: create one run UUID, per-host data/log roots, + both hosts, and verify the digest plus required runtime labels. The final + flexible-port image was also built locally for the three-container run; + transfer it to host B before a mixed-placement run. +- [x] **Prepare isolated roots**: create one run UUID, per-host data/log roots, a unique Docker network or host-network port map, and a machine-readable endpoint map. Verify no endpoint is already owned. -- [ ] **Start three logical nodes**: launch two offset nodes on host A and one - base node on host B with the same cluster policy, `initial_voters = 3`, - distinct data roots and explicit physical-host IDs. Verify all three - candidates are visible before admission. +- [x] **Start two logical nodes**: launch one node per physical host with + `initial_voters = 1`, distinct data roots and explicit physical-host IDs. + Verify both candidates are visible before admission. +- [x] **Start three composed nodes**: launch three offset nodes on one + physical host with offsets `0/200/400`, verify all listener families are + unique, and verify all three candidates are visible before admission. ## Phase 2: bootstrap and normal cross-host behavior -- [ ] **Admit and bootstrap Group 0**: create one rack record, admit the three - discovered nodes, initialize the cluster, and wait for all three KV health - observations. Record node IDs, discovery IDs, cluster ID and remote replica - endpoints. +- [x] **Admit and bootstrap the two-node run**: create one rack record, admit + both discovered nodes with `ssh_password = null`, initialize one-voter Group + 0, and record node IDs, discovery IDs and cluster ID. +- [x] **Admit and bootstrap the composed three-node run**: create one rack, + admit all three offset nodes with key-only SSH, initialize three-voter Group + 0, and verify management/RPC ports `10000/10100`, `10200/10300` and + `10400/10500`. - [ ] **Verify dependency readiness**: wait for KV, DiskDB, DiskIO, ChunkDB, Chunk-KV and Access readiness on all logical nodes. Confirm the monitor status is `ready` and identities/PIDs are stable after one health interval. @@ -88,13 +98,16 @@ voters and is an expected quorum-loss observation. ## Phase 3: failure and recovery cases +- [x] **Remote logical-node restart**: stop and recreate host B's node using + the same data root without mounting a password file. Verify the original + discovery identity, cluster ID, active Group 0 status and key-only SSH. - [ ] **Remote logical-node stop**: stop only host B's CROWDB node. Verify the two host-A voters retain Group 0 quorum, linearizable KV writes continue, existing S3 reads continue, and status identifies the stopped node. -- [ ] **Remote node restart**: restart host B's node with the same data root. - Verify the original discovery identity, cluster ID, service identities and - persisted object are retained; wait for full readiness and repeat the data - corpus. +- [x] **Key-only composed restart**: recreate each of the three composed + containers without `CROWDB_SSH_PASSWORD_FILE`; verify + `PasswordAuthentication no`, all three dynamic KV health endpoints and + active Group 0 status. - [ ] **Managed listener failure**: stop or fault only the selected Chunk-KV listener while leaving its process ownership observable. Verify consecutive readiness failures trigger the monitor's bounded restart, the PID changes, diff --git a/doc/working/track-two-physical-nodes.md b/doc/working/track-two-physical-nodes.md index 9d1506262..b92111dc7 100644 --- a/doc/working/track-two-physical-nodes.md +++ b/doc/working/track-two-physical-nodes.md @@ -15,13 +15,19 @@ Run ID: `two-physical-nodes-20261030T010205Z` - [x] Host-network port-offset support verified by focused test, clippy and shell syntax checks. - [x] Current source/image identity recorded and identical on both hosts. -- [ ] Isolated run UUID, roots, network and endpoint map created. +- [x] Final flexible-port image built locally: digest + `sha256:7f4c32b59e3ea9315a7726f6667ae6d5f4ea795f70c29c920371d045a80cf42d`. +- [ ] Final flexible-port image transferred to host B for mixed placement. +- [x] Isolated run UUID, roots, network and endpoint map created. ## Bootstrap -- [ ] Three logical nodes launched: host A ×2, host B ×1. -- [ ] Three candidates discovered and admitted. -- [ ] Group 0 initialized with three voters. +- [x] Two logical nodes launched: host A ×1, host B ×1. +- [x] Two candidates discovered and admitted with key-only SSH. +- [x] One-voter Group 0 initialized and remote node restarted without a password file. +- [x] Three logical nodes launched on one host with offsets 0/200/400. +- [x] Three candidates discovered and admitted with key-only SSH. +- [x] Group 0 initialized with three voters and all dynamic KV listeners healthy. - [ ] All required services ready on all logical nodes. - [ ] Baseline KV/S3/Iceberg/ChunkDB corpus passed. @@ -32,12 +38,12 @@ Run ID: `two-physical-nodes-20261030T010205Z` - [ ] Managed Chunk-KV listener failure: monitor restarts unhealthy service. - [ ] Transient allocation quorum loss: allocation-only recovery succeeds. - [ ] Stop host A's two logical nodes: expected quorum loss recorded. -- [ ] Restore host A and host B: full cluster and data recovery passed. +- [x] Restore the two-node run and all three composed nodes with key-only SSH. ## Evidence and exit -- [ ] Status snapshots, process identities and monitor events saved. -- [ ] Service/container logs saved for every phase. +- [x] Status snapshots, process identities and monitor events saved. +- [x] Service/container logs saved for every phase. - [ ] Final cross-host reads and health gate passed. - [ ] Owned resources cleaned; endpoints closed. - [ ] Result report completed with pass/expected/failed/blocked classification. diff --git a/lib/crowdb-protocol/src/mgmt/node/deployment.rs b/lib/crowdb-protocol/src/mgmt/node/deployment.rs index c4a289dc0..540d242c4 100644 --- a/lib/crowdb-protocol/src/mgmt/node/deployment.rs +++ b/lib/crowdb-protocol/src/mgmt/node/deployment.rs @@ -141,11 +141,6 @@ impl NodeDeploymentPolicy { } for service in self.setup.services.iter().filter(|service| service.enabled) { let ports = &service.listeners; - if service.kind == SetupServiceKind::PaxosKv - && (ports.http_port != Some(10000) || ports.rpc_port != Some(10100)) - { - return Err("bootstrap KV requires management port 10000 and RPC port 10100"); - } if [ ports.http_port, ports.rpc_port, @@ -155,7 +150,7 @@ impl NodeDeploymentPolicy { ] .into_iter() .flatten() - .any(|port| matches!(port, 2222 | 9090 | 9095 | 10101)) + .any(|port| matches!(port, 2222 | 9090 | 9095)) { return Err("service listener conflicts with node management or bootstrap transport"); } diff --git a/lib/crowdb-protocol/src/mgmt/node/setup.rs b/lib/crowdb-protocol/src/mgmt/node/setup.rs index ab378730c..b4f462be3 100644 --- a/lib/crowdb-protocol/src/mgmt/node/setup.rs +++ b/lib/crowdb-protocol/src/mgmt/node/setup.rs @@ -117,6 +117,21 @@ pub struct NodeSetupChoices { pub services: Vec, } +impl NodeSetupChoices { + /// Return the management and RPC listeners reserved for the bootstrap KV. + /// + /// The node launcher may shift these listeners for host-network containers + /// that share one physical host. Callers that need the Group 0 endpoint + /// must use the accepted policy rather than assuming the default ports. + #[must_use] + pub fn paxos_kv_ports(&self) -> Option<(u16, u16)> { + self.services + .iter() + .find(|service| service.kind == SetupServiceKind::PaxosKv && service.enabled) + .and_then(|service| Some((service.listeners.http_port?, service.listeners.rpc_port?))) + } +} + fn default_services() -> Vec { SetupServiceKind::ALL .into_iter() diff --git a/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs b/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs index c312f2e7a..fd6b5e4d9 100644 --- a/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs +++ b/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs @@ -137,12 +137,6 @@ impl NodeSetupChoices { fn valid_listeners(service: &SetupService, ports: &mut BTreeSet) -> bool { use SetupServiceKind::{AccessServer, Chunkdb, Diskdb, Diskio}; let listeners = &service.listeners; - if service.kind == SetupServiceKind::PaxosKv - && (listeners.http_port.is_some_and(|port| port != 10000) - || listeners.rpc_port.is_some_and(|port| port != 10100)) - { - return false; - } if (service.kind != Chunkdb && listeners.dynamic_ownership.is_some()) || (service.kind != Diskdb && listeners.keepalive_interval_secs.is_some()) || (service.kind != AccessServer diff --git a/lib/crowdb-protocol/tests/node_deployment_test.rs b/lib/crowdb-protocol/tests/node_deployment_test.rs index 86194b55f..81068ec4f 100644 --- a/lib/crowdb-protocol/tests/node_deployment_test.rs +++ b/lib/crowdb-protocol/tests/node_deployment_test.rs @@ -179,12 +179,12 @@ fn invalid_policies_are_rejected_before_any_resolution_can_replace_them() { assert!(duplicate_port.validate().is_err()); duplicate_port.setup.services[5].listeners.dataset_port = None; assert!(duplicate_port.validate().is_err()); - for reserved in [2222, 9090, 9095, 10101] { + for reserved in [2222, 9090, 9095] { let mut policy = NodeDeploymentPolicy::default(); policy.setup.services[5].listeners.dataset_port = Some(reserved); assert!(policy.validate().is_err()); } - for (http, rpc) in [(10001, 10100), (10000, 10102)] { + for (http, rpc) in [(0, 10100), (10000, 10000)] { let mut policy = NodeDeploymentPolicy::default(); let kv = policy .setup @@ -194,7 +194,7 @@ fn invalid_policies_are_rejected_before_any_resolution_can_replace_them() { .unwrap(); kv.listeners.http_port = Some(http); kv.listeners.rpc_port = Some(rpc); - assert!(policy.validate().unwrap_err().contains("bootstrap KV")); + assert!(policy.validate().is_err()); } } diff --git a/lib/crowdb-protocol/tests/node_setup_test.rs b/lib/crowdb-protocol/tests/node_setup_test.rs index a95236efb..cf2bbbe2d 100644 --- a/lib/crowdb-protocol/tests/node_setup_test.rs +++ b/lib/crowdb-protocol/tests/node_setup_test.rs @@ -117,15 +117,11 @@ fn service_selection_and_reserved_diskdb_listener_range_cannot_be_ambiguous() { let mut unsupported = request(); unsupported.choices.services[0].listeners.s3_port = Some(32000); assert!(unsupported.validate().is_err()); - let mut fixed_bootstrap = request(); - fixed_bootstrap.choices.services[0].listeners.http_port = Some(19910); - assert!( - fixed_bootstrap.validate().is_err(), - "node setup cannot silently ignore a bootstrap port override" - ); - fixed_bootstrap.choices.services[0].listeners.http_port = Some(10000); - fixed_bootstrap.choices.services[0].listeners.rpc_port = Some(10100); - fixed_bootstrap.validate().unwrap(); + let mut shifted_bootstrap = request(); + shifted_bootstrap.choices.services[0].listeners.http_port = Some(10100); + shifted_bootstrap.choices.services[0].listeners.rpc_port = Some(10200); + shifted_bootstrap.validate().unwrap(); + assert_eq!(shifted_bootstrap.choices.paxos_kv_ports(), Some((10100, 10200))); } #[test] From 9ecf95694c044b62eb6ced6fca405b2638dbb98e Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 30 Oct 2026 13:08:50 +0800 Subject: [PATCH 04/24] Retry transient Iceberg upload throttling --- .../access_iceberg/file_http/concurrency.rs | 47 ++++++++++++++++--- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/e2e-test/tests/access_iceberg/file_http/concurrency.rs b/e2e-test/tests/access_iceberg/file_http/concurrency.rs index 2a16d868c..5a0ff9fc4 100644 --- a/e2e-test/tests/access_iceberg/file_http/concurrency.rs +++ b/e2e-test/tests/access_iceberg/file_http/concurrency.rs @@ -1,8 +1,41 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +use super::super::signed_file::TestFileClient; use super::setup::{path, setup}; -use reqwest::Method; +use reqwest::{Method, StatusCode}; +use std::time::Duration; + +const MAX_SLOWDOWN_ATTEMPTS: usize = 4; +const SLOWDOWN_BASE_DELAY_MS: u64 = 100; +const SLOWDOWN_STAGGER_MS: u64 = 25; + +async fn put_with_slowdown_retry( + client: TestFileClient, + object: String, + payload: Vec, + index: usize, +) -> (StatusCode, String, usize) { + for attempt in 1..=MAX_SLOWDOWN_ATTEMPTS { + let response = client.send(Method::PUT, &object, "", &payload, false).await; + let status = response.status(); + let body = response.text().await.unwrap(); + if status != StatusCode::SERVICE_UNAVAILABLE + || !body.contains("SlowDown") + || attempt == MAX_SLOWDOWN_ATTEMPTS + { + return (status, body, attempt); + } + + let delay_ms = + SLOWDOWN_BASE_DELAY_MS * (1 << (attempt - 1)) + (index as u64 % 8) * SLOWDOWN_STAGGER_MS; + eprintln!("upload {index}: transient SlowDown on attempt {attempt}; retrying in {delay_ms}ms"); + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + } + + unreachable!("the retry loop always returns on its final attempt") +} + #[tokio::test(flavor = "multi_thread", worker_threads = 8)] #[ignore = "requires the owned release Iceberg FileIO stack"] async fn thirty_two_concurrent_direct_puts() { @@ -24,17 +57,17 @@ async fn thirty_two_concurrent_direct_puts() { let barrier = barrier.clone(); requests.spawn(async move { barrier.wait().await; - let response = client.send(Method::PUT, &object, "", &payload, false).await; - (index, response.status(), response.text().await.unwrap()) + let (status, body, attempts) = put_with_slowdown_retry(client, object, payload, index).await; + (index, status, body, attempts) }); } barrier.wait().await; let mut failures = Vec::new(); while let Some(result) = requests.join_next().await { - let (index, status, body) = result.unwrap(); - println!("upload {index}: {status} {body}"); - if status != 200 { - failures.push((index, status, body)); + let (index, status, body, attempts) = result.unwrap(); + println!("upload {index}: {status} attempts={attempts} {body}"); + if status != StatusCode::OK { + failures.push((index, status, attempts, body)); } } assert!(failures.is_empty(), "failed uploads: {failures:?}"); From 5d03757b75a9b1867eebbcab9346e84d153bdcf2 Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 30 Oct 2026 06:31:23 -0700 Subject: [PATCH 05/24] Fix Pixi Bash completion setup --- .../oci-image/tests/install_deps_test.py | 49 ++++++++++++++++--- tools/pixi-tasks/install-deps.sh | 6 +-- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/container/oci-image/tests/install_deps_test.py b/container/oci-image/tests/install_deps_test.py index 864d460df..6e08696e3 100644 --- a/container/oci-image/tests/install_deps_test.py +++ b/container/oci-image/tests/install_deps_test.py @@ -27,13 +27,25 @@ def execute(self, platform, fail_setup=False, installed_ui=False, fail_ui=False, log = root / 'commands' for name in ('git', 'cargo', 'pixi', 'uname', 'npm'): stub = binaries / name - stub.write_text('#!/bin/sh\n' - + ('printf "%s\\n" "$TEST_PLATFORM"\n' if name == 'uname' else - f'printf "%s\\n" "{name} $*" >> "$TEST_COMMANDS"\n' - 'case "$*" in\n' - ' *--setup-apparmor*) exit "$TEST_SETUP_EXIT" ;;\n' - f' ci) exit "$TEST_UI_EXIT" ;;\n' - 'esac\n')) + stub_lines = ['#!/bin/sh\n'] + if name == 'uname': + stub_lines.append('printf "%s\\n" "$TEST_PLATFORM"\n') + else: + if name == 'pixi': + stub_lines.extend([ + 'if [ "$1" = completion ]; then\n', + ' printf "%s\\n" \'complete -W "stub-task" pixi\'\n', + ' exit 0\n', + 'fi\n', + ]) + stub_lines.extend([ + f'printf "%s\\n" "{name} $*" >> "$TEST_COMMANDS"\n', + 'case "$*" in\n', + ' *--setup-apparmor*) exit "$TEST_SETUP_EXIT" ;;\n', + f' ci) exit "$TEST_UI_EXIT" ;;\n', + 'esac\n', + ]) + stub.write_text(''.join(stub_lines)) stub.chmod(0o755) environment = dict(os.environ, HOME=str(root), PIXI_PROJECT_ROOT=str(root), TEST_PLATFORM=platform, TEST_COMMANDS=str(log), TEST_SETUP_EXIT='1' if fail_setup else '0', @@ -46,6 +58,22 @@ def execute(self, platform, fail_setup=False, installed_ui=False, fail_ui=False, break return result, log.read_text().splitlines(), (root / '.bashrc').read_text() + def assert_completion_loads_in_new_bash(self, bashrc): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / '.bashrc').write_text(bashrc) + binaries = root / 'bin' + binaries.mkdir() + pixi = binaries / 'pixi' + pixi.write_text('#!/bin/sh\n' + 'printf \'complete -W "stub-task" pixi\\n\'\n') + pixi.chmod(0o755) + environment = dict(os.environ, HOME=str(root), PATH=str(binaries) + os.pathsep + os.environ['PATH']) + result = subprocess.run(['bash', '-ic', 'complete -p pixi'], env=environment, + text=True, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn('stub-task', result.stdout) + def test_linux_sets_up_host_before_ui_and_cargo_dependencies(self): result, commands, _ = self.execute('Linux x86_64') self.assertEqual(result.returncode, 0, result.stdout + result.stderr) @@ -83,6 +111,8 @@ def test_installs_completion_without_overwriting_existing_shell_configuration(se self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertTrue(bashrc.startswith(original)) self.assertIn('source <(pixi completion --shell bash)', bashrc) + self.assertIn('CROWDB Pixi Bash completion', bashrc) + self.assertIn("source /", result.stdout) def test_repeated_dependency_setup_does_not_duplicate_completion(self): result, _, bashrc = self.execute('Linux x86_64', repeats=2) @@ -97,6 +127,11 @@ def test_existing_completion_is_preserved_without_adding_another_block(self): self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual(bashrc, original) + def test_new_bash_loads_completion_block(self): + result, _, bashrc = self.execute('Darwin arm64') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assert_completion_loads_in_new_bash(bashrc) + if __name__ == '__main__': unittest.main() diff --git a/tools/pixi-tasks/install-deps.sh b/tools/pixi-tasks/install-deps.sh index 07e2be34c..fa2d59efc 100644 --- a/tools/pixi-tasks/install-deps.sh +++ b/tools/pixi-tasks/install-deps.sh @@ -6,15 +6,15 @@ cd "${PIXI_PROJECT_ROOT:?}" git config core.hooksPath .githooks bashrc="${HOME:?}/.bashrc" -if [[ ! -f "$bashrc" ]] || ! grep -Eq '^[[:space:]]*(source|\.)[[:space:]]+<\(pixi completion (--shell[ =]|-s[ =])bash\)' "$bashrc"; then +if [[ ! -f "$bashrc" ]] || ! grep -Eq 'pixi[[:space:]]+completion([^#]*[[:space:]])bash([[:space:]]|\)|$)' "$bashrc"; then cat >> "$bashrc" <<'EOF' -# Complete Pixi commands and workspace task names. +# CROWDB Pixi Bash completion: commands and workspace task names. if command -v pixi >/dev/null 2>&1; then source <(pixi completion --shell bash) fi EOF - echo "[install-deps] enabled Pixi Bash completion in $bashrc; open a new terminal to use it" + echo "[install-deps] enabled Pixi Bash completion in $bashrc; open a new Bash session or run 'source $bashrc'" fi if [[ $(uname -sm) == 'Linux x86_64' ]]; then pixi run --manifest-path container/oci-image/pixi.toml python container/oci-image/preflight.py --setup-apparmor From 25059cf7d9a5f08d61479f4749659aa6f3fe119f Mon Sep 17 00:00:00 2001 From: Gian Date: Fri, 30 Oct 2026 14:04:00 -0700 Subject: [PATCH 06/24] Fix cross-host storage recovery and CI deployment coverage --- .../src/serving/transition_runtime.rs | 3 + .../src/serving/worker.rs | 22 + .../tests/transition_worker_test.rs | 40 ++ app/crowdb-chunkdb/src/allocator/pool.rs | 12 +- .../tests/allocation_retry_test.rs | 130 +++++ .../tests/allocator_owner_change_test.rs | 119 +++++ .../src/commands/cluster/local/launch.rs | 13 +- .../src/commands/cluster/local/state.rs | 14 +- .../tests/cluster_lifecycle_test.rs | 50 ++ app/crowdb-diskdb/Cargo.toml | 4 + .../tests/common/journal_recovery.rs | 86 ++++ .../tests/journal_gap_recovery_test.rs | 70 +++ .../domain_monitor/chunk_kv/grants.rs | 15 +- .../tests/common/grant_process.rs | 36 ++ .../tests/domain_monitor_test.rs | 1 + app/crowdb-web/src/diskdb.rs | 2 + app/crowdb-web/src/diskdb/lifecycle.rs | 9 +- app/crowdb-web/src/diskdb/remote.rs | 6 +- app/crowdb-web/src/node/admission.rs | 5 +- app/crowdb-web/src/node/setup.rs | 5 +- .../src/services/deployment/planned.rs | 3 + .../diskdb_deployment_configuration_test.rs | 38 ++ container/crowdb-monitor/src/lib.rs | 8 +- container/crowdb-monitor/src/node.rs | 7 +- container/crowdb-monitor/src/node/runtime.rs | 2 +- .../crowdb-monitor/src/node/runtime/health.rs | 5 +- .../crowdb-monitor/src/node/runtime/probes.rs | 23 + .../src/node/runtime/recovery.rs | 2 +- .../src/node/runtime/recovery/publication.rs | 48 +- .../src/node/runtime/services/dependencies.rs | 29 +- .../src/node/runtime/setup/execution.rs | 6 +- container/crowdb-monitor/src/node/startup.rs | 5 +- .../crowdb-monitor/src/node/startup/policy.rs | 39 +- container/crowdb-monitor/src/node_args.rs | 26 +- .../tests/common/node_setup_control.rs | 17 +- .../crowdb-monitor/tests/kv_resume_test.rs | 14 +- .../crowdb-monitor/tests/member_trust_test.rs | 48 ++ .../crowdb-monitor/tests/node_cleanup_test.rs | 5 +- .../crowdb-monitor/tests/node_policy_test.rs | 51 ++ .../tests/node_run_inputs_test.rs | 50 ++ .../tests/node_service_dependencies_test.rs | 45 +- .../tests/node_service_readiness_test.rs | 29 +- .../crowdb-monitor/tests/node_setup_test.rs | 18 +- container/oci-image/Dockerfile | 2 +- container/oci-image/README.md | 29 +- container/oci-image/compose-three.yaml | 18 +- container/oci-image/compose.yaml | 6 +- container/oci-image/docker_local_launch.py | 14 +- container/oci-image/docker_local_state.py | 2 +- container/oci-image/run-node.py | 9 +- .../oci-image/tests/docker_local_test.py | 27 +- container/oci-image/tests/host_inputs_test.py | 8 +- container/single-node-container/entrypoint.sh | 2 +- doc/doc_index.md | 13 +- doc/working/plan-two-physical-nodes.md | 318 ++++++------ doc/working/track-two-physical-nodes.md | 475 ++++++++++++++++-- .../console-ui/e2e/fixtures/candidateSetup.ts | 4 +- .../e2e/flows/12-bridge-discovery.spec.ts | 6 +- e2e-test/clients/s3-go/main.go | 34 +- e2e-test/console_native_policy.py | 5 +- e2e-test/oci_reference_recovery.py | 8 +- e2e-test/oci_reference_runtime.py | 2 +- e2e-test/profiles/bridge_node.py | 5 +- e2e-test/profiles/setup_choices.py | 19 +- e2e-test/tests/admission_profile_test.py | 6 + e2e-test/tests/oci_reference_test.py | 3 + .../src/production_chunk/recovery.rs | 74 ++- .../src/production_chunk/recovery/drain.rs | 102 ++++ .../tests/production_chunk_test.rs | 10 +- .../tests/production_chunk_test/recovery.rs | 66 +++ .../production_chunk_test/recovery_drain.rs | 100 ++++ lib/crowdb-console-shared/src/clients/http.rs | 11 +- .../tests/topology_endpoint_test.rs | 79 +++ lib/crowdb-kv-client/src/client/retry.rs | 7 +- .../tests/conditional_retry_test.rs | 42 ++ lib/crowdb-kv/src/cluster/px_kv_store.rs | 8 +- .../src/cluster/px_kv_store_handler.rs | 28 +- lib/crowdb-kv/tests/store_test.rs | 3 + .../tests/store_test/journal_recovery_test.rs | 82 +++ .../tests/store_test/multi_group_test.rs | 4 + .../src/mgmt/node/deployment.rs | 41 +- lib/crowdb-protocol/src/mgmt/node/setup.rs | 2 + .../src/mgmt/node/setup/validation.rs | 9 +- .../tests/node_deployment_test.rs | 33 +- 84 files changed, 2462 insertions(+), 414 deletions(-) create mode 100644 app/crowdb-chunkdb/tests/allocation_retry_test.rs create mode 100644 app/crowdb-chunkdb/tests/allocator_owner_change_test.rs create mode 100644 app/crowdb-diskdb/tests/common/journal_recovery.rs create mode 100644 app/crowdb-diskdb/tests/journal_gap_recovery_test.rs create mode 100644 lib/crowdb-chunk-stream/src/production_chunk/recovery/drain.rs create mode 100644 lib/crowdb-chunk-stream/tests/production_chunk_test/recovery_drain.rs create mode 100644 lib/crowdb-console-shared/tests/topology_endpoint_test.rs create mode 100644 lib/crowdb-kv/tests/store_test/journal_recovery_test.rs diff --git a/app/crowdb-chunk-kv-server/src/serving/transition_runtime.rs b/app/crowdb-chunk-kv-server/src/serving/transition_runtime.rs index e09edb80c..f9ff02226 100644 --- a/app/crowdb-chunk-kv-server/src/serving/transition_runtime.rs +++ b/app/crowdb-chunk-kv-server/src/serving/transition_runtime.rs @@ -57,6 +57,9 @@ impl TransitionProcessor { mut revision: u64, ) -> Result<(), MonitorError> { let mut machine = TransferStateMachine::restore(transition)?; + self.executor + .release_excluded_source(machine.transition()) + .await?; if machine.transition().source.instance_id == self.instance_id && machine.transition().phase == TransferPhase::Planned { diff --git a/app/crowdb-chunk-kv-server/src/serving/worker.rs b/app/crowdb-chunk-kv-server/src/serving/worker.rs index 2a433c4a3..7d181f7e6 100644 --- a/app/crowdb-chunk-kv-server/src/serving/worker.rs +++ b/app/crowdb-chunk-kv-server/src/serving/worker.rs @@ -30,6 +30,28 @@ pub struct TransitionExecutor { } impl TransitionExecutor { + pub(super) async fn release_excluded_source( + &self, + transition: &TransferTransition, + ) -> Result<(), MonitorError> { + if transition.source.instance_id != self.instance_id || transition.release_proof.is_none() { + return Ok(()); + } + let Some(partition) = self.service.hosted_partition(transition.partition_id) else { + return Ok(()); + }; + // A restarted source can reopen the still-published old assignment + // before cutover. Release its exact old WAL so the target can take over; + // a later assignment of the same partition must remain untouched. + if partition.snapshot().ownership_epoch != transition.source_epoch { + return Ok(()); + } + partition + .suspend_for_shutdown(transition.source_epoch) + .await + .map_err(|error| plan_error(&error.to_string())) + } + /// Creates a worker bound to one server identity. /// /// # Errors diff --git a/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs b/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs index fc33b2666..b0d89dbbe 100644 --- a/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs +++ b/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs @@ -313,6 +313,46 @@ fn transfer(phase: TransferPhase) -> TransferTransition { } } +#[tokio::test] +async fn returning_source_releases_its_writer_after_durable_lease_exclusion() { + for (epoch, prepared) in [(3, false), (3, true), (4, true)] { + let store = Arc::new(Group0ControlStore::new(Arc::new(MemoryKv::default()))); + let mut transition = transfer(TransferPhase::TargetPreparing); + transition.target_artifact = transition.artifact.clone(); + transition.release_proof = Some(AuthorityReleaseProof::LeaseExpired { + activation_not_before_ms: 105, + }); + store.persist_transfer_transition(&transition, 0).await.unwrap(); + let source = partition(id(1), KeyRange::default(), epoch, &artifact(11), prepared).await; + let service = Arc::new(ChunkKvService::new(test_process::TestProcess::authority(1), 4).unwrap()); + service.install_partition(&source).unwrap(); + let executor = Arc::new( + TransitionExecutor::with_storage( + 1, + service, + Arc::new(FakeStorage { + recovered: source.clone(), + split: split_artifact(&split_transition()), + expected_split_parent_range: None, + }), + 8, + ) + .unwrap(), + ); + let processor = TransitionProcessor::new(1, store, executor); + processor.tick().await.unwrap(); + processor.tick().await.unwrap(); + assert_eq!( + source.lifecycle(), + if epoch == 3 { + PartitionLifecycle::TransferQuiesced + } else { + PartitionLifecycle::Prepared + } + ); + } +} + fn split_transition() -> SplitTransition { SplitTransition { transition_id: id(91), diff --git a/app/crowdb-chunkdb/src/allocator/pool.rs b/app/crowdb-chunkdb/src/allocator/pool.rs index bc25422a8..4a8e522f3 100644 --- a/app/crowdb-chunkdb/src/allocator/pool.rs +++ b/app/crowdb-chunkdb/src/allocator/pool.rs @@ -174,9 +174,10 @@ impl DiskdbClientPool { /// Allocate blocks on the diskdb instance owning `disk_group_id`. /// - /// A connection failure before submission refreshes discovery and permits - /// one attempt at a different endpoint. Once submitted, the mutation is - /// never replayed: DiskDB may have persisted blocks before losing its reply. + /// A connection failure before submission or an explicit ownership rejection + /// refreshes discovery and permits one attempt at a different endpoint. + /// Ambiguous submitted mutations are never replayed: DiskDB may have + /// persisted blocks before losing its reply. /// /// # Errors /// Returns `DiskdbClientError::ConnectFailed` when connection setup fails, @@ -240,7 +241,10 @@ impl DiskdbClientPool { DiskdbClientError::Unreachable(format!("no endpoint for disk_group {dg_id}: {error}")) })?; let result = self.transport.allocate_blocks(&endpoint, request).await; - if matches!(result, Err(DiskdbClientError::ConnectFailed(_))) { + if matches!( + result, + Err(DiskdbClientError::ConnectFailed(_) | DiskdbClientError::NotOwner(_)) + ) { if self.refresh_endpoints().await.is_err() { return result; } diff --git a/app/crowdb-chunkdb/tests/allocation_retry_test.rs b/app/crowdb-chunkdb/tests/allocation_retry_test.rs new file mode 100644 index 000000000..9eb726321 --- /dev/null +++ b/app/crowdb-chunkdb/tests/allocation_retry_test.rs @@ -0,0 +1,130 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/slot_groups.rs"] +#[allow(dead_code)] +mod slot_groups; + +use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbClientError, ChunkdbRpcTransport}; +use crowdb_kv_client::{ChunkSlotMapClient, ServiceRegistryClient}; +use crowdb_protocol::{ + chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}, + chunkdb::rpc::{AllocateChunkRequest, ChunkType}, + chunkdb_fb::{ + FBAllocateChunkRequest, FBAllocateChunkResponse, FBAllocateChunkResponseArgs, FBChunk, FBChunkArgs, + FBChunkType, FBChunkdbRetCode, + }, + common::ChunkId, + fb::FBMsgType, +}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +async fn client(test: &slot_groups::TestGroups, port: i32) -> ChunkdbClient { + let maps = ChunkSlotMapClient::new(Arc::clone(&test.kv)); + maps.initialize_layout(&ChunkSlotBootstrap { + service_instances: vec![1], + storage_groups: vec![ChunkStorageGroup { + store_id: 0, + group_id: 1, + }], + }) + .await + .unwrap(); + maps.initialize_service_epochs().await.unwrap(); + let registry = ServiceRegistryClient::from_shared(Arc::clone(&test.kv)); + registry + .register_chunkdb(1, &format!("127.0.0.1:{port}")) + .await + .unwrap(); + let client = ChunkdbClient::new(registry, Arc::new(ChunkdbRpcTransport::new())); + client.refresh_routes().await.unwrap(); + client +} + +fn request() -> AllocateChunkRequest { + AllocateChunkRequest { + chunk_id: Some(ChunkId { + high: 5 << 56, + low: 123, + }), + chunk_type: ChunkType::S3 as i32, + ..Default::default() + } +} + +#[tokio::test] +async fn known_quorum_unavailability_retries_allocation_with_the_same_chunk_id() { + let test = slot_groups::TestGroups::start().await; + let server = crowdb_rpc_ffi::RpcServer::new(None); + server.listen("127.0.0.1", 0).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let observed = calls.clone(); + server.register_reply_for_tests(FBMsgType::EAllocateChunkRequest.0 as u16, move |request| { + let body = flatbuffers::root::(request.control()).unwrap(); + let id = body.chunk_id().unwrap(); + assert_eq!((id.high(), id.low()), (5 << 56, 123)); + let first = observed.fetch_add(1, Ordering::Relaxed) == 0; + let mut builder = flatbuffers::FlatBufferBuilder::new(); + let error = first.then(|| builder.create_string("linearizable read: leadership quorum unavailable")); + let chunk = (!first).then(|| { + FBChunk::create( + &mut builder, + &FBChunkArgs { + id: Some(id), + chunk_type: FBChunkType::S3, + ..Default::default() + }, + ) + }); + let response = FBAllocateChunkResponse::create( + &mut builder, + &FBAllocateChunkResponseArgs { + id: request.request_id, + rpc_create_nano: request.rpc_create_nano, + ret_code: if first { + FBChunkdbRetCode::Unavailable + } else { + FBChunkdbRetCode::Success + }, + error_msg: error, + chunk, + ..Default::default() + }, + ); + builder.finish(response, None); + (builder.finished_data().to_vec(), Vec::new()) + }); + server.start(); + let result = client(&test, server.port()) + .await + .allocate_chunk(request()) + .await + .unwrap(); + assert_eq!(result.chunk.unwrap().id, request().chunk_id); + assert_eq!(calls.load(Ordering::Relaxed), 2); + server.stop(); +} + +#[tokio::test] +async fn an_allocation_with_a_lost_reply_is_not_replayed() { + let test = slot_groups::TestGroups::start().await; + let server = crowdb_rpc_ffi::RpcServer::new(None); + server.listen("127.0.0.1", 0).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let observed = calls.clone(); + server.register_handler(FBMsgType::EAllocateChunkRequest.0 as u16, move |_| { + observed.fetch_add(1, Ordering::Relaxed); + }); + server.start(); + let error = client(&test, server.port()) + .await + .allocate_chunk(request()) + .await + .unwrap_err(); + assert!(matches!(error, ChunkdbClientError::OutcomeUnknown(_)), "{error}"); + assert_eq!(calls.load(Ordering::Relaxed), 1); + server.stop(); +} diff --git a/app/crowdb-chunkdb/tests/allocator_owner_change_test.rs b/app/crowdb-chunkdb/tests/allocator_owner_change_test.rs new file mode 100644 index 000000000..e29eeab16 --- /dev/null +++ b/app/crowdb-chunkdb/tests/allocator_owner_change_test.rs @@ -0,0 +1,119 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[allow(dead_code)] +#[path = "common/slot_groups.rs"] +mod slot_groups; + +use crowdb_chunkdb::allocator::DiskdbClientPool; +use crowdb_diskdb_client::DiskdbClientError; +use crowdb_kv_client::ServiceRegistryClient; +use crowdb_protocol::{ + common::ChunkId, + diskdb_fb::{FBAllocateResponse, FBAllocateResponseArgs, FBDiskdbRetCode}, + fb::FBMsgType, +}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +fn endpoint(code: FBDiskdbRetCode) -> (crowdb_rpc_ffi::RpcServer, Arc) { + let server = crowdb_rpc_ffi::RpcServer::new(None); + server.listen("127.0.0.1", 0).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let observed = calls.clone(); + server.register_reply_for_tests(FBMsgType::EAllocateBlocksRequest.0 as u16, move |request| { + observed.fetch_add(1, Ordering::Relaxed); + let mut builder = flatbuffers::FlatBufferBuilder::new(); + let response = FBAllocateResponse::create( + &mut builder, + &FBAllocateResponseArgs { + id: request.request_id, + rpc_create_nano: request.rpc_create_nano, + ret_code: code, + ..Default::default() + }, + ); + builder.finish(response, None); + (builder.finished_data().to_vec(), Vec::new()) + }); + server.start(); + (server, calls) +} + +#[tokio::test] +async fn allocation_reroutes_after_an_explicit_old_owner_rejection() { + let test = slot_groups::TestGroups::start().await; + let registry = ServiceRegistryClient::from_shared(test.kv.clone()); + let (old, old_calls) = endpoint(FBDiskdbRetCode::NotOwner); + let (new, new_calls) = endpoint(FBDiskdbRetCode::Success); + registry + .register_diskdb(1, &format!("127.0.0.1:{}", old.port()), &[1], &[]) + .await + .unwrap(); + let pool = DiskdbClientPool::new(registry.clone()); + pool.refresh_endpoints().await.unwrap(); + registry + .register_diskdb(1, &format!("127.0.0.1:{}", new.port()), &[1], &[]) + .await + .unwrap(); + let result = pool + .allocate_blocks( + 1, + 1, + 1, + &ChunkId { + high: 5 << 56, + low: 1, + }, + ) + .await; + old.stop(); + new.stop(); + result.unwrap(); + assert_eq!(old_calls.load(Ordering::Relaxed), 1); + assert_eq!(new_calls.load(Ordering::Relaxed), 1); +} + +#[tokio::test] +async fn allocation_does_not_reroute_an_ambiguous_submitted_request() { + let test = slot_groups::TestGroups::start().await; + let registry = ServiceRegistryClient::from_shared(test.kv.clone()); + let old = crowdb_rpc_ffi::RpcServer::new(None); + old.listen("127.0.0.1", 0).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let observed = calls.clone(); + old.register_handler(FBMsgType::EAllocateBlocksRequest.0 as u16, move |_| { + observed.fetch_add(1, Ordering::Relaxed); + }); + old.start(); + let (new, new_calls) = endpoint(FBDiskdbRetCode::Success); + registry + .register_diskdb(1, &format!("127.0.0.1:{}", old.port()), &[1], &[]) + .await + .unwrap(); + let pool = DiskdbClientPool::new(registry.clone()); + pool.refresh_endpoints().await.unwrap(); + registry + .register_diskdb(1, &format!("127.0.0.1:{}", new.port()), &[1], &[]) + .await + .unwrap(); + let error = pool + .allocate_blocks( + 1, + 1, + 1, + &ChunkId { + high: 5 << 56, + low: 2, + }, + ) + .await + .unwrap_err(); + old.stop(); + new.stop(); + assert!(matches!(error, DiskdbClientError::Unreachable(_)), "{error}"); + assert_eq!(calls.load(Ordering::Relaxed), 1); + assert_eq!(new_calls.load(Ordering::Relaxed), 0); +} diff --git a/app/crowdb-cli/src/commands/cluster/local/launch.rs b/app/crowdb-cli/src/commands/cluster/local/launch.rs index 664205ef6..3258e9802 100644 --- a/app/crowdb-cli/src/commands/cluster/local/launch.rs +++ b/app/crowdb-cli/src/commands/cluster/local/launch.rs @@ -41,7 +41,7 @@ pub(super) async fn accepted(docker: &Docker, cluster: &str, args: &StartArgs) - return Err(invalid("local start requires the Linux amd64 CROWDB node image")); } let record = Launch { - version: 1, + version: 2, cluster: cluster.into(), owner: uuid::Uuid::new_v4().to_string(), runtime: docker.identity().await?, @@ -111,10 +111,13 @@ fn arguments(record: &Launch, index: u8) -> Vec { &format!("type=volume,source={volume},target=/opt/crowdb/data"), ])); let base = record.port_base + u16::from(index - 1) * 10; - for (offset, protocol) in [(0_u16, ""), (1, "S3"), (2, "ICEBERG"), (3, "DATASET")] { + for ((offset, protocol), internal) in [(0_u16, ""), (1, "S3"), (2, "ICEBERG"), (3, "DATASET")] + .into_iter() + .zip(record.container_ports()) + { args.extend(strings(&[ "-p", - &format!("127.0.0.1:{}:{}", base + offset, 9090 + offset), + &format!("127.0.0.1:{}:{internal}", base + offset), ])); if !protocol.is_empty() { args.extend(strings(&[ @@ -137,9 +140,9 @@ pub(super) fn verify(item: &Value, record: &Launch, index: u8) -> Result<()> { require_owned(item, record, "container")?; let base = record.port_base + u16::from(index - 1) * 10; let mut ports = serde_json::Map::new(); - for offset in 0..4 { + for (offset, internal) in (0_u16..).zip(record.container_ports()) { ports.insert( - format!("{}/tcp", 9090 + offset), + format!("{internal}/tcp"), json!([{"HostIp":"127.0.0.1", "HostPort":(base+offset).to_string()}]), ); } diff --git a/app/crowdb-cli/src/commands/cluster/local/state.rs b/app/crowdb-cli/src/commands/cluster/local/state.rs index 694db0da8..20769bb0c 100644 --- a/app/crowdb-cli/src/commands/cluster/local/state.rs +++ b/app/crowdb-cli/src/commands/cluster/local/state.rs @@ -152,7 +152,7 @@ impl Launch { pub fn validate(&self) -> Result<()> { let owner = uuid::Uuid::parse_str(&self.owner).map_err(|_| invalid("invalid cluster ownership token"))?; - if self.version != 1 + if !matches!(self.version, 1 | 2) || !name_valid(&self.cluster) || owner.is_nil() || owner.to_string() != self.owner @@ -182,6 +182,18 @@ impl Launch { .map(|index| format!("http://127.0.0.1:{}", self.port_base + u16::from(index) * 10)) .collect() } + pub fn container_ports(&self) -> [u16; 4] { + if self.version == 1 { + [9090, 9091, 9092, 9093] + } else { + [ + 9090, + crowdb_protocol::ACCESS_SERVER_HTTP_BASE, + crowdb_protocol::ACCESS_SERVER_ICEBERG_HTTP_BASE, + crowdb_protocol::ACCESS_SERVER_DATASET_HTTP_BASE, + ] + } + } pub fn resource_name(&self, kind: &str, index: u8) -> String { let base = format!("crowdb-{}", self.cluster); if kind == "network" { diff --git a/app/crowdb-cli/tests/cluster_lifecycle_test.rs b/app/crowdb-cli/tests/cluster_lifecycle_test.rs index 27a966722..d17ef6325 100644 --- a/app/crowdb-cli/tests/cluster_lifecycle_test.rs +++ b/app/crowdb-cli/tests/cluster_lifecycle_test.rs @@ -154,6 +154,15 @@ async fn default_single_node_start_seals_identity_and_resumes_without_duplicate_ 0, ); let original = fs::read(test.receipt("local")).unwrap(); + let accepted: Value = serde_json::from_slice(&original).unwrap(); + assert_eq!(accepted["version"], 2); + for (internal, offset) in [(9090, 0), (16000, 1), (17000, 2), (18000, 3)] { + assert_eq!( + test.state()["container"]["crowdb-local-1"]["HostConfig"]["PortBindings"] + [format!("{internal}/tcp")][0]["HostPort"], + (port + offset).to_string() + ); + } let binding = test.root.path().join("persistent/clusters/local/target.json"); let target: Value = serde_json::from_slice(&fs::read(binding).unwrap()).unwrap(); assert_eq!(target["cluster_id"], uuid::Uuid::from_u128(1).to_string()); @@ -181,6 +190,47 @@ async fn default_single_node_start_seals_identity_and_resumes_without_duplicate_ server.abort(); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn legacy_receipts_retain_protocol_ports_on_resume_and_recreation() { + let test = TestDocker::new(); + let (port, server) = serve("local", 1, true).await; + test.run( + None, + &["start", "--port-base", &port.to_string(), "--wait", "0"], + 0, + ); + let path = test.receipt("local"); + let mut legacy: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + legacy["version"] = json!(1); + let accepted = serde_json::to_vec(&legacy).unwrap(); + fs::write(&path, &accepted).unwrap(); + let ports: serde_json::Map<_, _> = (0_u16..4) + .map(|offset| { + ( + format!("{}/tcp", 9090 + offset), + json!([{"HostIp":"127.0.0.1","HostPort":(port+offset).to_string()}]), + ) + }) + .collect(); + let mut state = test.state(); + state["container"]["crowdb-local-1"]["HostConfig"]["PortBindings"] = json!(ports); + test.save(&state); + test.run(None, &["stop"], 0); + test.run(None, &["start", "--wait", "0"], 0); + assert_eq!(fs::read(&path).unwrap(), accepted); + let mut state = test.state(); + state["container"] = json!({}); + test.save(&state); + test.run(None, &["start", "--wait", "0"], 0); + assert_eq!(fs::read(&path).unwrap(), accepted); + assert_eq!( + test.state()["container"]["crowdb-local-1"]["HostConfig"]["PortBindings"], + json!(ports) + ); + assert_eq!(test.state()["volume"].as_object().unwrap().len(), 1); + server.abort(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn three_node_partial_readiness_keeps_resumable_inputs_and_stop_is_scoped() { let test = TestDocker::new(); diff --git a/app/crowdb-diskdb/Cargo.toml b/app/crowdb-diskdb/Cargo.toml index 3f0451ab8..546028912 100644 --- a/app/crowdb-diskdb/Cargo.toml +++ b/app/crowdb-diskdb/Cargo.toml @@ -167,3 +167,7 @@ path = "tests/online_activation_test.rs" [[test]] name = "compaction_schedule_test" path = "tests/compaction_schedule_test.rs" + +[[test]] +name = "journal_gap_recovery_test" +path = "tests/journal_gap_recovery_test.rs" diff --git a/app/crowdb-diskdb/tests/common/journal_recovery.rs b/app/crowdb-diskdb/tests/common/journal_recovery.rs new file mode 100644 index 000000000..08b481885 --- /dev/null +++ b/app/crowdb-diskdb/tests/common/journal_recovery.rs @@ -0,0 +1,86 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_diskdb::model::zone::DdbZone; +use crowdb_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_kv::kv::{Batch, BatchOp, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine, Op}; +use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_kv::wal::{index::SegmentIndex, record::WALRecord, replay::ReplayResult}; +use crowdb_protocol::common::{ChunkId, DiskId}; +use crowdb_protocol::diskdb::rpc::BusyBlockValue; +use crowdb_protocol::key::{BinaryKey, BusyBlockKey, ZoneKey}; +use crowdb_protocol::ZoneValueExt; + +pub struct TestReclaimedJournal; + +impl TestReclaimedJournal { + pub async fn replica(disk_id: DiskId) -> PxLocalReplica { + let baseline = DdbZone::new(disk_id, 0, 1, 128); + baseline + .snapshot_slot + .store(1, std::sync::atomic::Ordering::Release); + let zone_key = ZoneKey { + disk_id, + zone_index: 0, + } + .to_bytes(); + let busy_key = BusyBlockKey { + disk_id, + zone_index: 0, + unit_offset: 7, + } + .to_bytes(); + let busy = BusyBlockValue { + unit_count: 1, + owner_chunk: Some(ChunkId { high: 30, low: 40 }), + unit_size: 1024 * 1024, + allocation_ts: 100, + ..Default::default() + }; + let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig::default()).unwrap(); + for (slot, key, value) in [ + (1, zone_key, baseline.to_zone_value().to_bytes()), + (2, busy_key, bincode::serialize(&busy).unwrap()), + ] { + engine + .apply( + slot, + &Batch { + ops: vec![BatchOp { + key: key.into(), + op: Op::Put(value.into()), + }], + }, + ) + .await + .unwrap(); + } + engine.noop(3); + engine.handle().flush().unwrap(); + assert_eq!(engine.resume_from_slot(), 3); + let replay = ReplayResult { + records: vec![WALRecord::from_accepted( + 1, + &PxLogEntry { + slot: 3, + ballot: PxBallot::new(0, 1), + term: 1, + payload: Bytes::new(), + }, + )], + index: SegmentIndex::new(), + max_segment_id: 1, + current_term: 1, + voted_for: None, + }; + PxLocalReplica::restore_from_replay_with_engine( + 1, + PxLocalReplicaRole::Leader, + &replay, + Box::new(engine), + ) + .await + .unwrap() + } +} diff --git a/app/crowdb-diskdb/tests/journal_gap_recovery_test.rs b/app/crowdb-diskdb/tests/journal_gap_recovery_test.rs new file mode 100644 index 000000000..bf46047f1 --- /dev/null +++ b/app/crowdb-diskdb/tests/journal_gap_recovery_test.rs @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_diskdb::{ddb_kv_client::DdbKvClient, recovery::ZoneLoader}; +use crowdb_kv::cluster::group::PxGroup; +use crowdb_kv::cluster::kv_server::KvServer; +use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_protocol::common::{DiskId, HwStatus}; +use crowdb_protocol::diskdb::rpc::DiskValue; +use std::sync::Arc; + +#[path = "common/journal_recovery.rs"] +mod journal_recovery; + +#[tokio::test] +async fn restart_with_reclaimed_journal_rebuilds_busy_units_before_allocating() { + let disk_id = DiskId { high: 10, low: 20 }; + let restored = journal_recovery::TestReclaimedJournal::replica(disk_id).await; + let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); + store.add_group(PxGroup::new( + 0, + PxLocalReplica::new(1, PxLocalReplicaRole::Leader), + )); + store.add_group(PxGroup::new(1, restored)); + store.start().await.unwrap(); + let client = CrowdbKvClient::new(ClientConfig::new(vec![])); + let endpoint = store.listen_addr().unwrap().to_string(); + for group in [0, 1] { + client.seed_leader(0, group, endpoint.clone()); + } + let loader = ZoneLoader::new(Arc::new(DdbKvClient::new(client)), 1); + let result = loader + .load_disk_group( + 1, + 1, + 1, + (0, 1), + &[( + disk_id, + DiskValue { + capacity_units: 128, + zone_size_units: 128, + unit_size_bytes: 1024 * 1024, + zone_count: 1, + status: HwStatus::Up as i32, + ..Default::default() + }, + )], + 1, + ) + .await; + store.stop(); + let group = result.unwrap(); + group.set_status(HwStatus::Up); + let disks = group.disk_snapshot(); + let zones = disks[0].zones.load(); + assert!( + zones[0].usage_bits.is_set(7), + "reclaimed journal must not erase a durable allocation" + ); + assert_eq!(zones[0].usage_bits.count_set(), 1); + for _ in 0..16 { + let (_, _, allocated) = group.allocate_block(1, &[], 100, 1).unwrap(); + assert_ne!( + allocated.unit_offset, 7, + "live tree extent must never be reallocated" + ); + } +} diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs index e8ee91fda..59ff76455 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs @@ -3,7 +3,7 @@ //! Grant publication serialized with process replacement and renewal. -use super::{catalog, operation_error, wall_time_ms}; +use super::{catalog, operation_error, read_transfers, wall_time_ms}; use crate::group0_control_plane::Group0ControlPlane; use bytes::Bytes; use crowdb_kv::cluster::group_operations::KvGroupMutation; @@ -28,6 +28,18 @@ pub(super) async fn issue( return Ok(()); }; let ready = read_ready(control, descriptor).await?; + let released: HashSet<_> = read_transfers(control) + .await? + .into_iter() + .filter(|(transition, _)| transition.release_proof.is_some()) + .map(|(transition, _)| { + ( + transition.source.instance_id, + transition.partition_id, + transition.source_epoch, + ) + }) + .collect(); let mut assignments: HashMap> = HashMap::new(); for entry in catalog.pages.iter().flat_map(|page| &page.entries) { let assignment = ServingAssignment { @@ -35,6 +47,7 @@ pub(super) async fn issue( owner_epoch: entry.owner_epoch, }; if entry.state == ChunkKvRangeCatalogPartitionState::Serving + && !released.contains(&(entry.owner.instance_id, entry.partition_id, entry.owner_epoch)) && ready .get(&entry.owner.instance_id) .is_some_and(|instance| instance.assignments.contains(&assignment)) diff --git a/app/crowdb-kv-server/tests/common/grant_process.rs b/app/crowdb-kv-server/tests/common/grant_process.rs index 52304f0d9..830f0c73b 100644 --- a/app/crowdb-kv-server/tests/common/grant_process.rs +++ b/app/crowdb-kv-server/tests/common/grant_process.rs @@ -80,3 +80,39 @@ async fn grant_and_process_expiry_fence_publish_together_and_reject_stale_observ assert_eq!(accepted.grant_sequence, current.lease_sequence); assert!(accepted.grant_expires_at_ms >= current.expires_at_ms); } + +pub(super) async fn assert_returned_source_is_excluded( + control: &Group0ControlPlane, + policy: &DomainMonitorDescriptor, + partition_id: Id128, +) { + // A replacement source can report the old assignment while the catalog + // still names it. Durable exclusion must prevent granting it again. + test_process::TestProcessLease::seed(control, 1).await; + let mut returned = test_process::TestProcessLease::idle_instance(1, "127.0.0.1:17001"); + returned.extra.as_mut().unwrap().chunk_kv.as_mut().unwrap().hosted = vec![HostedPartition { + partition_id, + owner_epoch: 5, + recovering: false, + }]; + put_json( + control, + InstanceKey { + service: "chunk-kv".into(), + instance_id: 1, + } + .to_path(), + &returned, + ) + .await; + ChunkKvRangeMonitorDriver::new() + .tick(control, policy) + .await + .unwrap(); + assert!(control + .get(ServingGrantKey { instance_id: 1 }.to_path().as_bytes()) + .await + .unwrap() + .value + .is_none()); +} diff --git a/app/crowdb-kv-server/tests/domain_monitor_test.rs b/app/crowdb-kv-server/tests/domain_monitor_test.rs index 75a0ce721..cea9ed61a 100644 --- a/app/crowdb-kv-server/tests/domain_monitor_test.rs +++ b/app/crowdb-kv-server/tests/domain_monitor_test.rs @@ -544,6 +544,7 @@ async fn chunk_kv_driver_plans_dead_owner_recovery_once_and_waits_for_target() { .generation, 1 ); + grant_process::assert_returned_source_is_excluded(&control, &policy, partition_id).await; } #[allow(clippy::too_many_lines)] diff --git a/app/crowdb-web/src/diskdb.rs b/app/crowdb-web/src/diskdb.rs index 48e42c901..598021a78 100644 --- a/app/crowdb-web/src/diskdb.rs +++ b/app/crowdb-web/src/diskdb.rs @@ -7,6 +7,8 @@ pub mod lifecycle; pub mod proxy; mod remote; pub(crate) use remote::configuration as planned_configuration; +#[cfg(feature = "test-util")] +pub use remote::configuration as configuration_for_tests; pub use lifecycle::*; pub use proxy::*; diff --git a/app/crowdb-web/src/diskdb/lifecycle.rs b/app/crowdb-web/src/diskdb/lifecycle.rs index 02a135b6c..09c77c58c 100644 --- a/app/crowdb-web/src/diskdb/lifecycle.rs +++ b/app/crowdb-web/src/diskdb/lifecycle.rs @@ -26,6 +26,9 @@ pub struct DeployDiskdbBody { /// Internal HTTP health port. When omitted, derived from `rpc_port`. #[serde(default)] pub http_port: Option, + /// Explicit RPC listener for policies with independent listener ranges. + #[serde(default)] + pub rpc_listen_port: Option, /// Normal `DiskDB` heartbeat cadence, retained across service restart. #[serde(default)] pub keepalive_interval_secs: Option, @@ -36,9 +39,11 @@ pub(super) fn validate_diskdb_ports( ) -> Result<(u16, u16, u16), (StatusCode, Json)> { let listen_port = body.listen_port.unwrap_or(body.rpc_port); let http_port = body.http_port.unwrap_or_else(|| body.rpc_port.saturating_add(1)); - let rpc_listen_port = body.rpc_port.saturating_add(2); + let rpc_listen_port = body + .rpc_listen_port + .unwrap_or_else(|| body.rpc_port.saturating_add(2)); let ports = [listen_port, http_port, rpc_listen_port]; - if ports.contains(&0) || body.rpc_port > 65_533 { + if ports.contains(&0) || (body.rpc_listen_port.is_none() && body.rpc_port > 65_533) { return Err(err_400("DiskDB ports must be in the range 1..=65535")); } if ports[0] == ports[1] || ports[0] == ports[2] || ports[1] == ports[2] { diff --git a/app/crowdb-web/src/diskdb/remote.rs b/app/crowdb-web/src/diskdb/remote.rs index 29cc6a10f..5cad0e4d0 100644 --- a/app/crowdb-web/src/diskdb/remote.rs +++ b/app/crowdb-web/src/diskdb/remote.rs @@ -57,7 +57,11 @@ pub(super) async fn deploy( )) } -pub(crate) fn configuration( +/// Build a retained `DiskDB` configuration from validated listener choices. +/// +/// # Errors +/// Rejects zero, overflowing or overlapping listener ports. +pub fn configuration( node: &crowdb_console_shared::config::NodeEntry, seeds: &[String], body: &DeployDiskdbBody, diff --git a/app/crowdb-web/src/node/admission.rs b/app/crowdb-web/src/node/admission.rs index 0b6edaaaa..8185a0162 100644 --- a/app/crowdb-web/src/node/admission.rs +++ b/app/crowdb-web/src/node/admission.rs @@ -189,15 +189,14 @@ async fn admit_observed( .map_err(map_config_err)?; } save(&path, &record)?; - let setup = body.setup.clone(); - super::setup::accept(state, &record, &target, body.setup, body.expected_formation).await?; + let setup = super::setup::accept(state, &record, &target, body.setup, body.expected_formation).await?; target.ssh_password = None; target.ssh_key = None; target.ssh_credential_ref = Some("id_ed25519".into()); if let Some(binding) = &binding { confirm_admission(state, binding, &key, &target, &mut record).await?; } - update_config(state, &record, target, setup.as_ref())?; + update_config(state, &record, target, Some(&setup))?; save(&path, &record)?; state.persist().map_err(map_persist_err)?; super::preflight::finish(state, &record)?; diff --git a/app/crowdb-web/src/node/setup.rs b/app/crowdb-web/src/node/setup.rs index 919590ef4..23432cc8b 100644 --- a/app/crowdb-web/src/node/setup.rs +++ b/app/crowdb-web/src/node/setup.rs @@ -28,7 +28,7 @@ pub(super) async fn accept( target: &NodeEntry, choices: Option, expected: Option, -) -> Result<(), Failure> { +) -> Result { let mut target = target.clone(); target.ssh_password = None; target.ssh_key = Some(super::node_key_path(state).to_string_lossy().into_owned()); @@ -75,6 +75,7 @@ pub(super) async fn accept( }) .unwrap_or_default() }; + let accepted_choices = choices.clone(); let request = NodeSetupRequest { version: 1, discovery_id: record.discovery_id.clone(), @@ -88,7 +89,7 @@ pub(super) async fn accept( let observed: NodeSetupStatus = serde_json::from_value(value).map_err(|error| err_502(error.to_string()))?; validate(state, record, &observed)?; - Ok(()) + Ok(accepted_choices) } pub(crate) async fn status( diff --git a/app/crowdb-web/src/services/deployment/planned.rs b/app/crowdb-web/src/services/deployment/planned.rs index c47679e1e..0dfe87e3b 100644 --- a/app/crowdb-web/src/services/deployment/planned.rs +++ b/app/crowdb-web/src/services/deployment/planned.rs @@ -148,6 +148,9 @@ async fn build( let name = service_name(selected.kind); let id = format!("{name}-{}", node.id); let (configuration, environment) = if selected.kind == SetupServiceKind::Diskdb { + if selected.listeners.listen_port.is_some() { + choices["rpc_listen_port"] = json!(selected.listeners.rpc_port); + } let body: crate::diskdb::DeployDiskdbBody = serde_json::from_value(choices).map_err(|error| err_400(error.to_string()))?; ( diff --git a/app/crowdb-web/tests/diskdb_deployment_configuration_test.rs b/app/crowdb-web/tests/diskdb_deployment_configuration_test.rs index 8fba4db38..335a7c1a2 100644 --- a/app/crowdb-web/tests/diskdb_deployment_configuration_test.rs +++ b/app/crowdb-web/tests/diskdb_deployment_configuration_test.rs @@ -9,6 +9,44 @@ use crowdb_web::{router, AppState}; use serde_json::{json, Value}; use tower::ServiceExt; +#[test] +fn explicit_diskdb_ranges_keep_adjacent_host_instances_disjoint_and_preserve_legacy_inputs() { + use crowdb_console_shared::config::NodeEntry; + use crowdb_web::diskdb::{configuration_for_tests, DeployDiskdbBody}; + let node: NodeEntry = serde_json::from_value(json!({ + "id":1,"rack_id":1,"host":"127.0.0.1","ssh_port":2222,"ssh_user":"crowdb" + })) + .unwrap(); + let mut used = std::collections::BTreeSet::new(); + for offset in 0..3 { + let body: DeployDiskdbBody = serde_json::from_value(json!({ + "listen_port":11000+offset,"http_port":11100+offset, + "rpc_port":11200+offset,"rpc_listen_port":11200+offset + })) + .unwrap(); + let config = configuration_for_tests(&node, &["http://127.0.0.1:10000".into()], &body) + .unwrap_or_else(|(_, body)| panic!("{}", body.error)); + let config: toml::Value = toml::from_str(&config).unwrap(); + for (field, base) in [ + ("listen_addr", 11000), + ("http_listen_addr", 11100), + ("rpc_listen_addr", 11200), + ] { + let address = config["server"][field].as_str().unwrap(); + assert_eq!(address, format!("127.0.0.1:{}", base + offset)); + assert!(used.insert(address.to_owned())); + } + } + let legacy: DeployDiskdbBody = serde_json::from_value(json!({"rpc_port":7100})).unwrap(); + let config = + configuration_for_tests(&node, &[], &legacy).unwrap_or_else(|(_, body)| panic!("{}", body.error)); + let config: toml::Value = toml::from_str(&config).unwrap(); + assert_eq!( + config["server"]["rpc_listen_addr"].as_str(), + Some("127.0.0.1:7102") + ); +} + async fn submit(body: Value) -> (StatusCode, String) { let state = AppState::default(); let response = router(state) diff --git a/container/crowdb-monitor/src/lib.rs b/container/crowdb-monitor/src/lib.rs index e0066d866..772817db9 100644 --- a/container/crowdb-monitor/src/lib.rs +++ b/container/crowdb-monitor/src/lib.rs @@ -43,9 +43,9 @@ pub use node::{ }; pub use node::{ accept_node_policy, accept_node_policy_with_port_offset, control_node, discover_disks, read_node_policy, - run_node, run_node_with_inputs, serve_node_management, CandidateCache, DiscoveryConfig, DiscoveryError, - NodeDiscovery, NodeIdentity, NodeIdentityError, NodeManagementError, NodeRuntimeConfig, - NODE_SERVICE_TYPE, + resolve_node_policy_with_port_offset, run_node, run_node_with_inputs, serve_node_management, + CandidateCache, DiscoveryConfig, DiscoveryError, NodeDiscovery, NodeIdentity, NodeIdentityError, + NodeManagementError, NodeRuntimeConfig, NODE_SERVICE_TYPE, }; #[cfg(feature = "test-util")] pub use preview::{preview_step_names_for_tests, run_preview_for_tests, PreviewError}; @@ -60,7 +60,7 @@ pub use status::{MonitorPhase, MonitorStatus, ServiceStatus, StatusError, Status pub use supervisor::{Supervisor, SupervisorError}; #[cfg(feature = "test-util")] -pub use node::rebind_recovery_services_for_tests; +pub use node::{apply_member_endpoints_for_tests, rebind_recovery_services_for_tests}; pub use node::read_recovery_identity; diff --git a/container/crowdb-monitor/src/node.rs b/container/crowdb-monitor/src/node.rs index b4f467c8c..5a3dd76c8 100644 --- a/container/crowdb-monitor/src/node.rs +++ b/container/crowdb-monitor/src/node.rs @@ -45,7 +45,10 @@ pub use runtime::{control_node, run_node, run_node_with_inputs, NodeRuntimeConfi pub use runtime::{publish_file_for_tests, resume_file_operations_for_tests}; #[cfg(feature = "test-util")] pub use startup::accept_startup_mode_for_tests; -pub use startup::{accept_node_policy, accept_node_policy_with_port_offset, read_node_policy}; +pub use startup::{ + accept_node_policy, accept_node_policy_with_port_offset, read_node_policy, + resolve_node_policy_with_port_offset, +}; pub use management::{serve_node_management, NodeManagementError}; @@ -66,7 +69,7 @@ pub(crate) fn write_single_binding( pub use runtime::NodeServiceForTests; #[cfg(feature = "test-util")] -pub use runtime::rebind_recovery_services_for_tests; +pub use runtime::{apply_member_endpoints_for_tests, rebind_recovery_services_for_tests}; pub use runtime::read_recovery_identity; diff --git a/container/crowdb-monitor/src/node/runtime.rs b/container/crowdb-monitor/src/node/runtime.rs index 5aeef16fc..d5084a3e5 100644 --- a/container/crowdb-monitor/src/node/runtime.rs +++ b/container/crowdb-monitor/src/node/runtime.rs @@ -342,7 +342,7 @@ fn disk_inventory(root: &std::path::Path) -> Result { pub use services::NodeServiceForTests; #[cfg(feature = "test-util")] -pub use recovery::rebind_recovery_services_for_tests; +pub use recovery::{apply_member_endpoints_for_tests, rebind_recovery_services_for_tests}; pub use recovery::read_recovery_identity; diff --git a/container/crowdb-monitor/src/node/runtime/health.rs b/container/crowdb-monitor/src/node/runtime/health.rs index 811fb00e2..3b1f57158 100644 --- a/container/crowdb-monitor/src/node/runtime/health.rs +++ b/container/crowdb-monitor/src/node/runtime/health.rs @@ -181,7 +181,10 @@ impl NodeHealth { self.probe_failures.remove(&intent.service_id); return Ok(false); } - let service = match super::probes::for_intent(profile, intent) { + // An ownership handover can withdraw readiness while the service is + // still advancing its durable transition. Restart only a failed live + // listener; keep readiness probes for the published health snapshot. + let service = match super::probes::liveness_for_intent(profile, intent) { Ok(service) => service, Err(error) => { tracing::debug!(service = %intent.service_id, %error, "service health probe unavailable"); diff --git a/container/crowdb-monitor/src/node/runtime/probes.rs b/container/crowdb-monitor/src/node/runtime/probes.rs index d180f70ea..b5e681dfb 100644 --- a/container/crowdb-monitor/src/node/runtime/probes.rs +++ b/container/crowdb-monitor/src/node/runtime/probes.rs @@ -56,6 +56,29 @@ pub(super) fn for_intent(profile: &DeploymentProfile, intent: &NodeServiceIntent Ok(service) } +pub(super) fn liveness_for_intent( + profile: &DeploymentProfile, + intent: &NodeServiceIntent, +) -> Result { + let mut service = for_intent(profile, intent)?; + if matches!(service.probe.kind, ProbeKind::Http) { + let address = service + .probe + .target + .strip_suffix("/ready") + .ok_or("missing managed readiness path")?; + service.probe.target = format!( + "{address}{}", + if intent.kind == "access" { + "/live" + } else { + "/health" + } + ); + } + Ok(service) +} + fn field<'a>(mut value: &'a toml::Value, path: &[&str]) -> Result<&'a toml::Value> { for key in path { value = value.get(*key).ok_or("missing service probe setting")?; diff --git a/container/crowdb-monitor/src/node/runtime/recovery.rs b/container/crowdb-monitor/src/node/runtime/recovery.rs index 22ac60f34..a4808a88b 100644 --- a/container/crowdb-monitor/src/node/runtime/recovery.rs +++ b/container/crowdb-monitor/src/node/runtime/recovery.rs @@ -266,7 +266,7 @@ pub fn member_trust_ready_for_tests(root: &Path) -> Result { } #[cfg(feature = "test-util")] -pub use publication::rebind_recovery_services_for_tests; +pub use publication::{apply_member_endpoints_for_tests, rebind_recovery_services_for_tests}; /// Read a live monitor's retained identity without waiting for lifecycle control. /// diff --git a/container/crowdb-monitor/src/node/runtime/recovery/publication.rs b/container/crowdb-monitor/src/node/runtime/recovery/publication.rs index 0e35f167f..c28571280 100644 --- a/container/crowdb-monitor/src/node/runtime/recovery/publication.rs +++ b/container/crowdb-monitor/src/node/runtime/recovery/publication.rs @@ -21,8 +21,7 @@ pub(super) async fn reconcile( ) -> Result<()> { let local = super::identity(root)?.endpoint; let mut config = topology.clone(); - let (management_port, rpc_port) = super::super::kv::ports_from_root(root)?; - let seeds = apply_member_endpoints(&mut config, endpoints, management_port, rpc_port)?; + let seeds = apply_member_endpoints(&mut config, endpoints)?; let client = Arc::new(CrowdbKvClient::new(ClientConfig::new(seeds.clone()))); let GetOutcome::Found { value, .. } = client .get( @@ -97,12 +96,7 @@ pub(super) async fn reconcile( Ok(()) } -fn apply_member_endpoints( - config: &mut ConsoleConfig, - endpoints: &MemberEndpoints, - management_port: u16, - rpc_port: u16, -) -> Result> { +fn apply_member_endpoints(config: &mut ConsoleConfig, endpoints: &MemberEndpoints) -> Result> { let mut seeds = Vec::new(); for node in &mut config.nodes { let Some(peer) = endpoints.members.iter().find(|peer| peer.node_id == node.id) else { @@ -115,18 +109,24 @@ fn apply_member_endpoints( .or_else(|| peer.addresses.first()) .ok_or("member address missing")?; node.host = address.to_string(); - seeds.push(format!( - "http://{}", - std::net::SocketAddr::new(*address, management_port) - )); - for server in config + let server = config .servers .iter_mut() - .filter(|server| server.node_id == Some(node.id)) - { - server.url = format!("http://{}", std::net::SocketAddr::new(*address, management_port)); - server.rpc_url = Some(std::net::SocketAddr::new(*address, rpc_port).to_string()); - } + .find(|server| { + server.node_id == Some(node.id) + && server.service_type == crowdb_console_shared::config::ServiceType::PaxosKv + }) + .ok_or("member KV topology missing")?; + let management_port = reqwest::Url::parse(&server.url)? + .port_or_known_default() + .ok_or("member management port missing")?; + let rpc = server.rpc_url.as_ref().ok_or("member RPC endpoint missing")?; + let rpc_port = reqwest::Url::parse(&format!("http://{rpc}"))? + .port() + .ok_or("member RPC port missing")?; + server.url = format!("http://{}", std::net::SocketAddr::new(*address, management_port)); + server.rpc_url = Some(std::net::SocketAddr::new(*address, rpc_port).to_string()); + seeds.push(server.url.clone()); } Ok(seeds) } @@ -213,6 +213,18 @@ async fn rebind( Ok(true) } +#[cfg(feature = "test-util")] +/// Rebind authenticated contacts while retaining each member's accepted ports. +/// +/// # Errors +/// Rejects missing member topology or malformed accepted endpoints. +pub fn apply_member_endpoints_for_tests( + config: &mut ConsoleConfig, + endpoints: &MemberEndpoints, +) -> Result> { + apply_member_endpoints(config, endpoints) +} + #[cfg(feature = "test-util")] /// Exercise contact publication with no reachable authority. /// diff --git a/container/crowdb-monitor/src/node/runtime/services/dependencies.rs b/container/crowdb-monitor/src/node/runtime/services/dependencies.rs index d1540208e..6db53985b 100644 --- a/container/crowdb-monitor/src/node/runtime/services/dependencies.rs +++ b/container/crowdb-monitor/src/node/runtime/services/dependencies.rs @@ -83,6 +83,7 @@ pub(in crate::node::runtime) async fn ready( .await??; } if intent.kind == "access" { + use crowdb_chunk_kv_client::ChunkKvRangeCatalogSource; // An idle local ChunkKV listener does not prove that the catalog owner // has finished WAL replay. Probe the same routed read Access needs. let binding: NodeBinding = super::durable::read(&profile.paths.data_root.join("node-binding.json"))? @@ -91,24 +92,36 @@ pub(in crate::node::runtime) async fn ready( crowdb_kv_client::ClientConfig::new(binding.management_seeds), )); let catalog = Arc::new(crowdb_chunk_kv_client::Group0ChunkKvRangeCatalogSource::from_shared(kv)); - let transport = Arc::new(crowdb_chunk_kv_client::ChunkKvRpcTransport::new(1, 1, 1)); + let (_, pages) = catalog.load().await?; + let mut owner_keys = BTreeMap::new(); + for entry in pages.iter().flat_map(|page| &page.entries) { + owner_keys + .entry(entry.owner.rpc_endpoint.clone()) + .or_insert_with(|| entry.range.start.clone()); + } + let max_owners = owner_keys.len().max(1); + let transport = Arc::new(crowdb_chunk_kv_client::ChunkKvRpcTransport::new(max_owners, 1, 1)); let client = crowdb_chunk_kv_client::ChunkKvClient::new( crowdb_chunk_kv_client::ClientConfig { operation_timeout: Duration::from_secs(1), // Keep normal routed-read redirects within this probe's deadline: // catalog ownership may change between discovery and the read. - max_owner_connections: 1, + max_owner_connections: max_owners, ..Default::default() }, catalog, transport, )?; - let response = client.get(b"/deployment/access-readiness".to_vec(), None).await?; - if !matches!( - response.result, - Ok(crowdb_protocol::chunk_kv::OperationResult::Value(_)) - ) { - return Err(format!("routed ChunkKV catalog is not ready: {:?}", response.result).into()); + // Access startup can read any metadata range. A point routed to one + // healthy owner cannot authorize a launch while another owner replays. + for key in owner_keys.into_values() { + let response = client.get(key, None).await?; + if !matches!( + response.result, + Ok(crowdb_protocol::chunk_kv::OperationResult::Value(_)) + ) { + return Err(format!("routed ChunkKV catalog is not ready: {:?}", response.result).into()); + } } } Ok(()) diff --git a/container/crowdb-monitor/src/node/runtime/setup/execution.rs b/container/crowdb-monitor/src/node/runtime/setup/execution.rs index 720ed0ed4..d239cf1af 100644 --- a/container/crowdb-monitor/src/node/runtime/setup/execution.rs +++ b/container/crowdb-monitor/src/node/runtime/setup/execution.rs @@ -82,7 +82,11 @@ async fn advance(profile: &DeploymentProfile, state: &mut AcceptedSetup) -> Resu let preparation_needed = !matches!( choices.storage, crowdb_protocol::mgmt::node::setup::SetupStorage::Diskless - ) || choices.services.iter().any(|service| service.enabled); + ) || choices.services.iter().any(|service| { + // The admitted KV process already supplies Group 0. Keeping that + // prerequisite enabled does not request ordinary storage metadata. + service.enabled && service.kind != crowdb_protocol::mgmt::node::setup::SetupServiceKind::PaxosKv + }); if preparation_needed && crate::node::read_node_policy(root)?.is_some() && !metadata::ready(root, &binding, &kv).await? diff --git a/container/crowdb-monitor/src/node/startup.rs b/container/crowdb-monitor/src/node/startup.rs index e7be853fe..03377ff7d 100644 --- a/container/crowdb-monitor/src/node/startup.rs +++ b/container/crowdb-monitor/src/node/startup.rs @@ -10,7 +10,10 @@ use serde::{Deserialize, Serialize}; use super::durable; mod policy; -pub use policy::{accept_node_policy, accept_node_policy_with_port_offset, read_node_policy}; +pub use policy::{ + accept_node_policy, accept_node_policy_with_port_offset, read_node_policy, + resolve_node_policy_with_port_offset, +}; #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] diff --git a/container/crowdb-monitor/src/node/startup/policy.rs b/container/crowdb-monitor/src/node/startup/policy.rs index c1397e595..0148669f7 100644 --- a/container/crowdb-monitor/src/node/startup/policy.rs +++ b/container/crowdb-monitor/src/node/startup/policy.rs @@ -7,7 +7,7 @@ use std::{fs, io, path::Path}; use crowdb_protocol::mgmt::node::{ deployment::{NodeDeploymentOverrides, NodeDeploymentPolicy}, - setup::SetupStorage, + setup::{SetupService, SetupStorage}, }; use crate::node::durable; @@ -57,13 +57,7 @@ pub fn accept_node_policy_with_port_offset( return Err(io::Error::other("node policy root must be an absolute directory")); } let accepted = read_node_policy(root)?; - let mut selected = explicit - .resolve(file, accepted.as_ref()) - .map_err(io::Error::other)?; - if accepted.is_none() && offset != 0 { - shift_policy_ports(&mut selected, offset)?; - selected.validate().map_err(io::Error::other)?; - } + let selected = resolve_node_policy_with_port_offset(file, explicit, accepted.as_ref(), offset)?; if accepted.is_some() { validate_retained_identity(root)?; return Ok(selected); @@ -87,9 +81,34 @@ pub fn accept_node_policy_with_port_offset( Ok(winner) } -fn shift_policy_ports(policy: &mut NodeDeploymentPolicy, offset: u16) -> io::Result<()> { - for service in &mut policy.setup.services { +/// Resolve launcher inputs into the same port space as the accepted policy. +/// +/// # Errors +/// Rejects invalid inputs, port overflow and changes to accepted policy. +pub fn resolve_node_policy_with_port_offset( + file: &NodeDeploymentOverrides, + explicit: &NodeDeploymentOverrides, + accepted: Option<&NodeDeploymentPolicy>, + offset: u16, +) -> io::Result { + let (mut file, mut explicit) = (file.clone(), explicit.clone()); + if accepted.is_some() { + for services in [&mut file.services, &mut explicit.services].into_iter().flatten() { + shift_service_ports(services, offset)?; + } + } + let mut selected = explicit.resolve(&file, accepted).map_err(io::Error::other)?; + if accepted.is_none() { + shift_service_ports(&mut selected.setup.services, offset)?; + selected.validate().map_err(io::Error::other)?; + } + Ok(selected) +} + +fn shift_service_ports(services: &mut [SetupService], offset: u16) -> io::Result<()> { + for service in services { for port in [ + &mut service.listeners.listen_port, &mut service.listeners.http_port, &mut service.listeners.rpc_port, &mut service.listeners.s3_port, diff --git a/container/crowdb-monitor/src/node_args.rs b/container/crowdb-monitor/src/node_args.rs index e91bc8b21..8f793b987 100644 --- a/container/crowdb-monitor/src/node_args.rs +++ b/container/crowdb-monitor/src/node_args.rs @@ -85,12 +85,7 @@ fn shifted_port(base: u16, offset: u16) -> Result Result<&'static str, Box> { let inputs = resolve(args)?; - let policy = crowdb_monitor::accept_node_policy_with_port_offset( - &inputs.data_root, - &inputs.file, - &inputs.explicit, - port_offset()?, - )?; + let policy = crowdb_monitor::accept_node_policy(&inputs.data_root, &inputs.file, &inputs.explicit)?; Ok(match policy.mode { DeploymentMode::Local => "local", DeploymentMode::Production => "production", @@ -124,12 +119,25 @@ fn resolve(args: &NodeRunArgs) -> Result tokio::task::JoinHandle<()> { + processes_reply_once(root, false) +} + +pub fn processes_with_kv_once(root: &Path) -> tokio::task::JoinHandle<()> { + processes_reply_once(root, true) +} + +fn processes_reply_once(root: &Path, kv_running: bool) -> tokio::task::JoinHandle<()> { std::fs::create_dir_all(root).unwrap(); let listener = tokio::net::UnixListener::bind(root.join("node-control.sock")).unwrap(); tokio::spawn(async move { @@ -16,8 +24,15 @@ pub fn processes_once(root: &Path) -> tokio::task::JoinHandle<()> { serde_json::from_slice::(&bytes).unwrap(), NodeControl::Processes { .. } )); + let processes = if kv_running { + serde_json::json!({"kv":42}) + } else { + serde_json::json!({}) + }; + let reply = serde_json::json!({"ok":true,"value":{ + "processes":processes,"service_failures":{}}}); stream - .write_all(br#"{"ok":true,"value":{"processes":{},"service_failures":{}}}"#) + .write_all(&serde_json::to_vec(&reply).unwrap()) .await .unwrap(); }) diff --git a/container/crowdb-monitor/tests/kv_resume_test.rs b/container/crowdb-monitor/tests/kv_resume_test.rs index 077df71f2..875d03556 100644 --- a/container/crowdb-monitor/tests/kv_resume_test.rs +++ b/container/crowdb-monitor/tests/kv_resume_test.rs @@ -4,7 +4,10 @@ #[path = "common/member_trust.rs"] mod trust; -use crowdb_monitor::{accept_member_trust_for_tests, DeploymentProfile, NodeServiceForTests, StatusStore}; +use crowdb_monitor::{ + accept_member_trust_for_tests, accept_node_policy, DeploymentProfile, NodeServiceForTests, StatusStore, +}; +use crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides; use std::{ os::unix::fs::PermissionsExt, sync::atomic::{AtomicBool, Ordering}, @@ -16,6 +19,15 @@ use std::{ async fn retained_quorum_startup_does_not_block_discovery_or_confuse_liveness_with_health() { let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("kv-resume"); let root = directory.path(); + accept_node_policy( + root, + &NodeDeploymentOverrides::default(), + &NodeDeploymentOverrides { + initial_voters: Some(3), + ..Default::default() + }, + ) + .unwrap(); let trust = trust::fixture(root); accept_member_trust_for_tests(root, &trust).unwrap(); let mut profile = DeploymentProfile::load( diff --git a/container/crowdb-monitor/tests/member_trust_test.rs b/container/crowdb-monitor/tests/member_trust_test.rs index d1ad5af84..bc21fc72b 100644 --- a/container/crowdb-monitor/tests/member_trust_test.rs +++ b/container/crowdb-monitor/tests/member_trust_test.rs @@ -375,3 +375,51 @@ fn a_new_admission_keeps_the_cancelled_operations_pins_and_fence() { "the cancelled operation remains fenced" ); } + +#[test] +fn recovered_contacts_preserve_member_specific_listener_ports() { + use crowdb_protocol::mgmt::node::recovery::{MemberEndpoint, MemberEndpoints}; + let dir = tempdir_in_test_data("monitor-offset-recovery-contacts"); + let trust = trust::fixture(dir.path()); + let operation = crowdb_console_shared::deployment::PreparedBootstrap::load( + &dir.path().join("prepared-bootstrap.json"), + ) + .unwrap(); + let mut config = operation.intent.to_config(); + config.servers[1].url = "http://192.0.2.2:10001".into(); + config.servers[1].rpc_url = Some("192.0.2.2:10101".into()); + let endpoints = MemberEndpoints { + version: 1, + bootstrap: trust.bootstrap, + observer_boot_id: uuid::Uuid::new_v4().to_string(), + observed_at_unix_ms: 1, + conflicts: vec![], + members: trust + .members + .into_iter() + .map(|member| MemberEndpoint { + discovery_id: member.discovery_id, + node_id: member.node_id, + boot_id: uuid::Uuid::new_v4().to_string(), + addresses: vec![if member.node_id == 3 { + "2001:db8::3" + } else { + "192.0.2.10" + } + .parse() + .unwrap()], + }) + .collect(), + }; + let seeds = crowdb_monitor::apply_member_endpoints_for_tests(&mut config, &endpoints).unwrap(); + assert_eq!( + seeds, + [ + "http://192.0.2.10:10000", + "http://192.0.2.10:10001", + "http://[2001:db8::3]:10000" + ] + ); + assert_eq!(config.servers[1].rpc_url.as_deref(), Some("192.0.2.10:10101")); + assert_eq!(config.servers[2].rpc_url.as_deref(), Some("[2001:db8::3]:10100")); +} diff --git a/container/crowdb-monitor/tests/node_cleanup_test.rs b/container/crowdb-monitor/tests/node_cleanup_test.rs index d878c5e7c..40c979cae 100644 --- a/container/crowdb-monitor/tests/node_cleanup_test.rs +++ b/container/crowdb-monitor/tests/node_cleanup_test.rs @@ -1,7 +1,8 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use crowdb_monitor::{cleanup_node_for_tests, DeploymentProfile, ProcessManager}; +use crowdb_monitor::{accept_node_policy, cleanup_node_for_tests, DeploymentProfile, ProcessManager}; +use crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides; use crowdb_protocol::mgmt::SystemBootstrapIdentity; use crowdb_test_harness::test_dirs::tempdir_in_test_data; use std::{os::unix::fs::PermissionsExt, path::Path}; @@ -11,6 +12,8 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; async fn retired_cleanup_restarts_its_local_worker_after_a_failed_deletion() { let directory = tempdir_in_test_data("monitor-retired-cleanup"); let root = directory.path(); + let defaults = NodeDeploymentOverrides::default(); + accept_node_policy(root, &defaults, &defaults).unwrap(); let bootstrap = SystemBootstrapIdentity { cluster_id: uuid::Uuid::new_v4().to_string(), operation_id: uuid::Uuid::new_v4().to_string(), diff --git a/container/crowdb-monitor/tests/node_policy_test.rs b/container/crowdb-monitor/tests/node_policy_test.rs index eeca7391d..0184deb97 100644 --- a/container/crowdb-monitor/tests/node_policy_test.rs +++ b/container/crowdb-monitor/tests/node_policy_test.rs @@ -10,6 +10,57 @@ use crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides; #[path = "common/run_root.rs"] mod run_root; +#[test] +fn explicit_file_listener_ports_are_shifted_once_across_repeated_acceptance() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-policy-file-offset"); + let omitted = NodeDeploymentOverrides::default(); + let mut file = NodeDeploymentOverrides { + services: Some( + crowdb_protocol::mgmt::node::deployment::NodeDeploymentPolicy::default() + .setup + .services, + ), + ..Default::default() + }; + let accepted = + crowdb_monitor::accept_node_policy_with_port_offset(root.path(), &file, &omitted, 1).unwrap(); + let bytes = fs::read(root.path().join("node-policy.json")).unwrap(); + assert_eq!( + crowdb_monitor::accept_node_policy_with_port_offset(root.path(), &file, &omitted, 1).unwrap(), + accepted + ); + file.services.as_mut().unwrap()[0].listeners.rpc_port = Some(10102); + assert!(crowdb_monitor::accept_node_policy_with_port_offset(root.path(), &file, &omitted, 1).is_err()); + assert_eq!(fs::read(root.path().join("node-policy.json")).unwrap(), bytes); +} + +#[test] +fn host_network_policy_shifts_every_diskdb_listener_once_and_keeps_instances_disjoint() { + let omitted = NodeDeploymentOverrides::default(); + let mut listeners = std::collections::BTreeSet::new(); + for offset in 0..3 { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-policy-offset"); + let policy = + crowdb_monitor::accept_node_policy_with_port_offset(root.path(), &omitted, &omitted, offset) + .unwrap(); + let diskdb = &policy.setup.services[1].listeners; + assert_eq!(diskdb.listen_port, Some(11000 + offset)); + assert_eq!(diskdb.http_port, Some(11100 + offset)); + assert_eq!(diskdb.rpc_port, Some(11200 + offset)); + for port in [diskdb.listen_port, diskdb.http_port, diskdb.rpc_port] + .into_iter() + .flatten() + { + assert!(listeners.insert(port)); + } + assert_eq!( + crowdb_monitor::accept_node_policy_with_port_offset(root.path(), &omitted, &omitted, offset,) + .unwrap(), + policy + ); + } +} + #[test] fn accepted_policy_survives_restart_without_defaults_or_identity_effects() { let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("accepted-node-policy"); diff --git a/container/crowdb-monitor/tests/node_run_inputs_test.rs b/container/crowdb-monitor/tests/node_run_inputs_test.rs index f7e0ba356..d53b66e50 100644 --- a/container/crowdb-monitor/tests/node_run_inputs_test.rs +++ b/container/crowdb-monitor/tests/node_run_inputs_test.rs @@ -36,6 +36,56 @@ fn command(profile: &Path) -> Command { command } +#[test] +fn container_preflight_reuses_explicit_file_ports_with_a_nonzero_offset() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-cli-file-offset"); + let profile = profile(root.path()); + let config = root.path().join("node.toml"); + let inputs = NodeDeploymentOverrides { + services: Some( + crowdb_protocol::mgmt::node::deployment::NodeDeploymentPolicy::default() + .setup + .services, + ), + ..Default::default() + }; + fs::write(&config, toml::to_string(&inputs).unwrap()).unwrap(); + let invoke = || { + let mut accept = Command::new(env!("CARGO_BIN_EXE_crowdb-monitor")); + for name in [ + "CROWDB_NODE_POLICY_FILE", + "CROWDB_DEPLOYMENT_MODE", + "CROWDB_CLUSTER", + "CROWDB_INITIAL_VOTERS", + "CROWDB_PHYSICAL_HOST_ID", + "CROWDB_S3_PUBLIC_URI", + "CROWDB_ICEBERG_PUBLIC_URI", + "CROWDB_DATASET_PUBLIC_URI", + "CROWDB_DISCOVERY_SEEDS", + ] { + accept.env_remove(name); + } + accept.args(["accept-policy", "--profile"]).arg(&profile); + accept + .args(["--deployment-config"]) + .arg(&config) + .env("CROWDB_PORT_OFFSET", "1"); + accept.output().unwrap() + }; + for _ in 0..2 { + let output = invoke(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + } + let accepted = crowdb_monitor::read_node_policy(&root.path().join("install/data")) + .unwrap() + .unwrap(); + assert_eq!(accepted.setup.services[0].listeners.rpc_port, Some(10101)); +} + #[test] fn public_uri_flags_override_environment_before_acceptance_and_changed_restarts_fail() { let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("public-origin"); diff --git a/container/crowdb-monitor/tests/node_service_dependencies_test.rs b/container/crowdb-monitor/tests/node_service_dependencies_test.rs index 7b2f6a8d3..a8184d96e 100644 --- a/container/crowdb-monitor/tests/node_service_dependencies_test.rs +++ b/container/crowdb-monitor/tests/node_service_dependencies_test.rs @@ -193,7 +193,7 @@ async fn access_readiness_follows_catalog_redirects_within_its_probe_deadline() }; let authority = node_setup::TestAuthority::start(root.path(), &request).await; let (rpc, requests) = redirecting_rpc(); - publish_catalog(&authority.kv, rpc.port()).await; + publish_catalog(&authority.kv, rpc.port(), None).await; let mut profile = DeploymentProfile::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml")) .unwrap(); @@ -217,9 +217,34 @@ async fn access_readiness_follows_catalog_redirects_within_its_probe_deadline() environment: BTreeMap::new(), }; let result = NodeServiceForTests::dependencies_ready_for_tests(&profile, &intent, &[]).await; - rpc.stop(); result.unwrap(); assert_eq!(requests.load(Ordering::Relaxed), 2); + let (second_rpc, second_requests) = redirecting_rpc(); + publish_catalog( + &authority.kv, + rpc.port(), + Some(u16::try_from(second_rpc.port()).unwrap()), + ) + .await; + NodeServiceForTests::dependencies_ready_for_tests(&profile, &intent, &[]) + .await + .unwrap(); + assert_eq!(second_requests.load(Ordering::Relaxed), 2); + let unavailable = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + publish_catalog( + &authority.kv, + rpc.port(), + Some(unavailable.local_addr().unwrap().port()), + ) + .await; + let result = NodeServiceForTests::dependencies_ready_for_tests(&profile, &intent, &[]).await; + rpc.stop(); + second_rpc.stop(); + assert!( + result.is_err(), + "a healthy first owner must not hide an unavailable second owner" + ); + assert!(!root.path().join("services/access-2/execution.json").exists()); } fn redirecting_rpc() -> (crowdb_rpc_ffi::RpcServer, Arc) { @@ -258,7 +283,7 @@ fn redirecting_rpc() -> (crowdb_rpc_ffi::RpcServer, Arc) { (rpc, requests) } -async fn publish_catalog(kv: &crowdb_kv_client::CrowdbKvClient, port: i32) { +async fn publish_catalog(kv: &crowdb_kv_client::CrowdbKvClient, port: i32, second: Option) { let mut page = ChunkKvRangeCatalogPage { generation: 1, page_index: 0, @@ -283,6 +308,20 @@ async fn publish_catalog(kv: &crowdb_kv_client::CrowdbKvClient, port: i32) { }, }], }; + if let Some(port) = second { + page.entries[0].range.end = Some(vec![128]); + let mut entry = page.entries[0].clone(); + entry.partition_id.low = 2; + entry.artifact.tree_id = 2; + entry.artifact.stream_name.low = 2; + entry.range = KeyRange { + start: vec![128], + end: None, + }; + entry.owner.instance_id = 3; + entry.owner.rpc_endpoint = format!("127.0.0.1:{port}"); + page.entries.push(entry); + } page.seal().unwrap(); let mut head = ChunkKvRangeCatalogHead { generation: 1, diff --git a/container/crowdb-monitor/tests/node_service_readiness_test.rs b/container/crowdb-monitor/tests/node_service_readiness_test.rs index dc3501c11..d67e92ed2 100644 --- a/container/crowdb-monitor/tests/node_service_readiness_test.rs +++ b/container/crowdb-monitor/tests/node_service_readiness_test.rs @@ -223,7 +223,7 @@ async fn recovery_waits_for_live_dependencies_without_consuming_launches() { #[tokio::test] #[allow(clippy::too_many_lines)] -async fn health_restarts_a_completed_service_after_consecutive_probe_failures() { +async fn completed_service_preserves_live_recovery_and_restarts_failed_health() { let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("service-health-restart"); let root = directory.path().join("data"); std::fs::create_dir_all(&root).unwrap(); @@ -275,7 +275,12 @@ health_file = sys.argv[2] class Handler(BaseHTTPRequestHandler): def do_GET(self): - status = 503 if health_file and os.path.exists(health_file) else 200 + status = 200 + if health_file and os.path.exists(health_file): + with open(health_file) as marker: + readiness_only = marker.read() == "readiness-only" + if self.path == "/ready" or not readiness_only: + status = 503 self.send_response(status) self.send_header("Content-Length", "0") self.end_headers() @@ -326,6 +331,26 @@ PY let mut worker = NodeServiceForTests::new_for_tests(profile).await.unwrap(); let first = worker.execute_for_tests(intent.clone()).await.unwrap(); let first_pid = first["pid"].as_u64().unwrap(); + std::fs::write(&unhealthy, b"readiness-only").unwrap(); + assert_eq!( + reqwest::get(format!("http://127.0.0.1:{port}/ready")) + .await + .unwrap() + .status(), + reqwest::StatusCode::SERVICE_UNAVAILABLE + ); + for _ in 0..3 { + worker.reconcile_for_tests().await.unwrap(); + } + let recovering: serde_json::Value = + serde_json::from_slice(&std::fs::read(root.join("services/chunkdb-2/execution.json")).unwrap()) + .unwrap(); + assert_eq!(recovering["pid"].as_u64().unwrap(), first_pid); + assert_eq!( + std::fs::read_to_string(&invocations).unwrap().trim(), + "1", + "readiness recovery must retain a responsive process" + ); std::fs::write(&unhealthy, b"probe failure").unwrap(); worker.reconcile_for_tests().await.unwrap(); assert_eq!( diff --git a/container/crowdb-monitor/tests/node_setup_test.rs b/container/crowdb-monitor/tests/node_setup_test.rs index 13f98e2e3..b6dcc339b 100644 --- a/container/crowdb-monitor/tests/node_setup_test.rs +++ b/container/crowdb-monitor/tests/node_setup_test.rs @@ -175,12 +175,19 @@ async fn unbound_setup_survives_a_worker_restart_without_creating_disks_or_servi #[tokio::test] async fn diskless_setup_requires_the_exact_confirmed_admission_and_does_not_start_services() { - let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-setup-diskless"); + for kv_enabled in [false, true] { + assert_diskless_setup(kv_enabled).await; + } +} + +async fn assert_diskless_setup(kv_enabled: bool) { + let directory = run_root::TestRunRoot::new("node-setup-diskless"); let root = directory.path(); let mut choices = crowdb_protocol::mgmt::node::deployment::NodeDeploymentPolicy::default().setup; choices.storage = SetupStorage::Diskless; for service in &mut choices.services { - service.enabled = false; + service.enabled = + kv_enabled && service.kind == crowdb_protocol::mgmt::node::setup::SetupServiceKind::PaxosKv; } let inputs = crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides { storage: Some(choices.storage.clone()), @@ -202,12 +209,17 @@ async fn diskless_setup_requires_the_exact_confirmed_admission_and_does_not_star let mut authority = node_setup::TestAuthority::start(root, &intent).await; let preparation_key = crowdb_console_shared::deployment::preparation::DEFAULT_PREPARATION_KEY; authority.kv.delete(0, 0, preparation_key, None).await.unwrap(); - let profile = profile(root); + let mut profile = profile(root); + profile.paths.run_root = root.join("run"); + let observe = kv_enabled.then(|| node_setup_control::processes_with_kv_once(&profile.paths.run_root)); NodeSetupForTests::advance_for_tests(profile.clone()) .await .unwrap(); let status = NodeSetupForTests::status_for_tests(root).unwrap().unwrap(); assert_eq!(status.phase, NodeSetupPhase::Ready); + if let Some(observe) = observe { + observe.await.unwrap(); + } assert!(matches!( authority .kv diff --git a/container/oci-image/Dockerfile b/container/oci-image/Dockerfile index 5299f7806..47a2e030a 100644 --- a/container/oci-image/Dockerfile +++ b/container/oci-image/Dockerfile @@ -45,6 +45,6 @@ ENV PATH="/opt/crowdb/bin:${PATH}" \ CROWDB_RUNTIME_ROOT="/opt/crowdb/run" USER root VOLUME ["/opt/crowdb/data"] -EXPOSE 9090 9091 9092 9093 9095 2222 +EXPOSE 9090 16000 17000 18000 9095 2222 STOPSIGNAL SIGTERM ENTRYPOINT ["/usr/bin/tini", "--", "/opt/crowdb/bin/entrypoint"] diff --git a/container/oci-image/README.md b/container/oci-image/README.md index 0a4fc3ae1..f752a568e 100644 --- a/container/oci-image/README.md +++ b/container/oci-image/README.md @@ -262,8 +262,11 @@ A direct single-container cluster needs a volume and four published ports: ```bash pixi run docker run -d --name crowdb-one --cpus 1 --memory 1536m \ - -p 127.0.0.1:9090:9090 -p 127.0.0.1:9091:9091 \ - -p 127.0.0.1:9092:9092 -p 127.0.0.1:9093:9093 \ + -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:9091 \ + -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:9092 \ + -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9093 \ + -p 127.0.0.1:9090:9090 -p 127.0.0.1:9091:16000 \ + -p 127.0.0.1:9092:17000 -p 127.0.0.1:9093:18000 \ --mount type=volume,source=crowdb-one-data,target=/opt/crowdb/data \ crowdb-node:dev ``` @@ -278,24 +281,24 @@ pixi run docker run -d --name crowdb-demo-1 --network crowdb-demo --cpus 1 --mem -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:9091 \ -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:9092 \ -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9093 \ - -p 127.0.0.1:9090:9090 -p 127.0.0.1:9091:9091 \ - -p 127.0.0.1:9092:9092 -p 127.0.0.1:9093:9093 \ + -p 127.0.0.1:9090:9090 -p 127.0.0.1:9091:16000 \ + -p 127.0.0.1:9092:17000 -p 127.0.0.1:9093:18000 \ --mount type=volume,source=crowdb-demo-1-data,target=/opt/crowdb/data crowdb-node:dev pixi run docker run -d --name crowdb-demo-2 --network crowdb-demo --cpus 1 --memory 1536m \ -e CROWDB_CLUSTER=demo -e CROWDB_INITIAL_VOTERS=3 \ -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:9101 \ -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:9102 \ -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9103 \ - -p 127.0.0.1:9100:9090 -p 127.0.0.1:9101:9091 \ - -p 127.0.0.1:9102:9092 -p 127.0.0.1:9103:9093 \ + -p 127.0.0.1:9100:9090 -p 127.0.0.1:9101:16000 \ + -p 127.0.0.1:9102:17000 -p 127.0.0.1:9103:18000 \ --mount type=volume,source=crowdb-demo-2-data,target=/opt/crowdb/data crowdb-node:dev pixi run docker run -d --name crowdb-demo-3 --network crowdb-demo --cpus 1 --memory 1536m \ -e CROWDB_CLUSTER=demo -e CROWDB_INITIAL_VOTERS=3 \ -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:9111 \ -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:9112 \ -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9113 \ - -p 127.0.0.1:9110:9090 -p 127.0.0.1:9111:9091 \ - -p 127.0.0.1:9112:9092 -p 127.0.0.1:9113:9093 \ + -p 127.0.0.1:9110:9090 -p 127.0.0.1:9111:16000 \ + -p 127.0.0.1:9112:17000 -p 127.0.0.1:9113:18000 \ --mount type=volume,source=crowdb-demo-3-data,target=/opt/crowdb/data crowdb-node:dev ``` @@ -309,6 +312,16 @@ pixi run docker compose -f container/oci-image/compose-three.yaml up -d pixi run docker compose -f container/oci-image/compose-three.yaml down ``` +Fresh nodes listen on container ports 9090 (Web), 16000 (S3), 17000 (Iceberg) +and 18000 (Dataset). The examples retain host ports 9090–9093. New local Docker +launch receipts use version 2; existing version-1 receipts retain their original +container ports and pinned image when resumed or recreated. + +For Compose volumes with accepted legacy listeners, set +`CROWDB_S3_CONTAINER_PORT=9091`, `CROWDB_ICEBERG_CONTAINER_PORT=9092` and +`CROWDB_DATASET_CONTAINER_PORT=9093` to match that policy. Public-origin variables +still describe the host-side ports. Retained policy is not rewritten on restart. + `down` removes the Compose containers and network, preserving named volumes. Set `CROWDB_IMAGE` to select the loaded image and `CROWDB_CLUSTER` to isolate another project. Change the port variables shown in each file before starting diff --git a/container/oci-image/compose-three.yaml b/container/oci-image/compose-three.yaml index af59582c7..01af56fff 100644 --- a/container/oci-image/compose-three.yaml +++ b/container/oci-image/compose-three.yaml @@ -17,9 +17,9 @@ services: CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE1_DATASET_PORT:-9093} ports: - "127.0.0.1:${CROWDB_NODE1_UI_PORT:-9090}:9090" - - "127.0.0.1:${CROWDB_NODE1_S3_PORT:-9091}:9091" - - "127.0.0.1:${CROWDB_NODE1_ICEBERG_PORT:-9092}:9092" - - "127.0.0.1:${CROWDB_NODE1_DATASET_PORT:-9093}:9093" + - "127.0.0.1:${CROWDB_NODE1_S3_PORT:-9091}:${CROWDB_S3_CONTAINER_PORT:-16000}" + - "127.0.0.1:${CROWDB_NODE1_ICEBERG_PORT:-9092}:${CROWDB_ICEBERG_CONTAINER_PORT:-17000}" + - "127.0.0.1:${CROWDB_NODE1_DATASET_PORT:-9093}:${CROWDB_DATASET_CONTAINER_PORT:-18000}" volumes: - node1-data:/opt/crowdb/data node2: @@ -32,9 +32,9 @@ services: CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE2_DATASET_PORT:-9103} ports: - "127.0.0.1:${CROWDB_NODE2_UI_PORT:-9100}:9090" - - "127.0.0.1:${CROWDB_NODE2_S3_PORT:-9101}:9091" - - "127.0.0.1:${CROWDB_NODE2_ICEBERG_PORT:-9102}:9092" - - "127.0.0.1:${CROWDB_NODE2_DATASET_PORT:-9103}:9093" + - "127.0.0.1:${CROWDB_NODE2_S3_PORT:-9101}:${CROWDB_S3_CONTAINER_PORT:-16000}" + - "127.0.0.1:${CROWDB_NODE2_ICEBERG_PORT:-9102}:${CROWDB_ICEBERG_CONTAINER_PORT:-17000}" + - "127.0.0.1:${CROWDB_NODE2_DATASET_PORT:-9103}:${CROWDB_DATASET_CONTAINER_PORT:-18000}" volumes: - node2-data:/opt/crowdb/data node3: @@ -47,9 +47,9 @@ services: CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE3_DATASET_PORT:-9113} ports: - "127.0.0.1:${CROWDB_NODE3_UI_PORT:-9110}:9090" - - "127.0.0.1:${CROWDB_NODE3_S3_PORT:-9111}:9091" - - "127.0.0.1:${CROWDB_NODE3_ICEBERG_PORT:-9112}:9092" - - "127.0.0.1:${CROWDB_NODE3_DATASET_PORT:-9113}:9093" + - "127.0.0.1:${CROWDB_NODE3_S3_PORT:-9111}:${CROWDB_S3_CONTAINER_PORT:-16000}" + - "127.0.0.1:${CROWDB_NODE3_ICEBERG_PORT:-9112}:${CROWDB_ICEBERG_CONTAINER_PORT:-17000}" + - "127.0.0.1:${CROWDB_NODE3_DATASET_PORT:-9113}:${CROWDB_DATASET_CONTAINER_PORT:-18000}" volumes: - node3-data:/opt/crowdb/data volumes: diff --git a/container/oci-image/compose.yaml b/container/oci-image/compose.yaml index 7ad711291..96e5e608f 100644 --- a/container/oci-image/compose.yaml +++ b/container/oci-image/compose.yaml @@ -15,9 +15,9 @@ services: CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_DATASET_PORT:-9093} ports: - "127.0.0.1:${CROWDB_UI_PORT:-9090}:9090" - - "127.0.0.1:${CROWDB_S3_PORT:-9091}:9091" - - "127.0.0.1:${CROWDB_ICEBERG_PORT:-9092}:9092" - - "127.0.0.1:${CROWDB_DATASET_PORT:-9093}:9093" + - "127.0.0.1:${CROWDB_S3_PORT:-9091}:${CROWDB_S3_CONTAINER_PORT:-16000}" + - "127.0.0.1:${CROWDB_ICEBERG_PORT:-9092}:${CROWDB_ICEBERG_CONTAINER_PORT:-17000}" + - "127.0.0.1:${CROWDB_DATASET_PORT:-9093}:${CROWDB_DATASET_CONTAINER_PORT:-18000}" volumes: - node-data:/opt/crowdb/data volumes: diff --git a/container/oci-image/docker_local_launch.py b/container/oci-image/docker_local_launch.py index a5242464f..99222ace3 100644 --- a/container/oci-image/docker_local_launch.py +++ b/container/oci-image/docker_local_launch.py @@ -8,6 +8,12 @@ from docker_local_state import CLUSTER, OWNER, check_daemon, load, require_owned, seal +CONTAINER_PORTS = (9090, 16000, 17000, 18000) + + +def container_ports(record): + return (9090, 9091, 9092, 9093) if record["version"] == 1 else CONTAINER_PORTS + def arguments(record): return ["--label", f"{CLUSTER}={record['cluster']}", "--label", f"{OWNER}={record['owner']}"] @@ -43,7 +49,7 @@ def accepted(docker, args): raise ValueError("local node image must be Linux amd64") if metadata.get("Config", {}).get("Labels", {}).get("org.opencontainers.image.source") != "https://github.com/buzzcrow/crowdb": raise ValueError("image is not a CROWDB node image") - record = {"version": 1, "cluster": args.cluster, "owner": str(uuid.uuid4()), "runtime": docker.identity(), + record = {"version": 2, "cluster": args.cluster, "owner": str(uuid.uuid4()), "runtime": docker.identity(), "nodes": args.nodes if args.nodes is not None else 1, "port_base": args.port_base if args.port_base is not None else 9090, "image": image, "image_id": metadata["Id"], "cpus": args.cpus if args.cpus is not None else 1, "memory_mib": args.memory_mib if args.memory_mib is not None else 1536} @@ -59,8 +65,8 @@ def verify_container(item, record, index, volume, network): require_owned(item, record, "container") host = item["HostConfig"] base = record["port_base"] + (index - 1) * 10 - ports = {f"{9090+offset}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(base+offset)}] - for offset in range(4)} + ports = {f"{port}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(base+offset)}] + for offset, port in enumerate(container_ports(record))} expected_environment = {"CROWDB_CLUSTER": record["cluster"], "CROWDB_INITIAL_VOTERS": str(record["nodes"])} for offset, protocol in enumerate((None, "S3", "ICEBERG", "DATASET")): if protocol: @@ -102,7 +108,7 @@ def run(docker, record): "--mount", f"type=volume,source={volume},target=/opt/crowdb/data", "-e", "CROWDB_CLUSTER=" + record["cluster"], "-e", f"CROWDB_INITIAL_VOTERS={record['nodes']}"] for offset, protocol in enumerate((None, "S3", "ICEBERG", "DATASET")): - command += ["-p", f"127.0.0.1:{base+offset}:{9090+offset}"] + command += ["-p", f"127.0.0.1:{base+offset}:{container_ports(record)[offset]}"] if protocol: command += ["-e", f"CROWDB_{protocol}_PUBLIC_URI=http://127.0.0.1:{base+offset}"] check_daemon(docker, record) diff --git a/container/oci-image/docker_local_state.py b/container/oci-image/docker_local_state.py index 0605f0357..960492c6f 100644 --- a/container/oci-image/docker_local_state.py +++ b/container/oci-image/docker_local_state.py @@ -71,7 +71,7 @@ def load(cluster, directory=RECORDS): if not stat.S_ISREG(metadata.st_mode) or metadata.st_mode & 0o077 or metadata.st_size > 1024 * 1024: raise ValueError("cluster receipt must be a bounded private regular file") record = json.loads(path.read_text()) - if record.get("version") != 1 or record.get("cluster") != cluster: + if record.get("version") not in (1, 2) or record.get("cluster") != cluster: raise ValueError("invalid cluster receipt") required = {"version", "cluster", "owner", "runtime", "nodes", "port_base", "image", "image_id", "cpus", "memory_mib"} if set(record) != required or not isinstance(record["runtime"], dict): diff --git a/container/oci-image/run-node.py b/container/oci-image/run-node.py index 5aa9e5947..fe5a94d39 100755 --- a/container/oci-image/run-node.py +++ b/container/oci-image/run-node.py @@ -21,9 +21,12 @@ def digest_reference(reference): # Base listeners used by one node. Host-network instances add -# ``CROWDB_PORT_OFFSET`` to every listener before the policy is sealed. -HOST_PORTS = (2222, 7100, 7101, 7102, 7200, 7300, 7301, 7400, 7401, - 9090, 9091, 9092, 9093, 9094, 9095, 10000, 10100, 10101) +# ``CROWDB_PORT_OFFSET`` to every listener before the policy is sealed. These +# values mirror the canonical service blocks in +# ``lib/crowdb-protocol/src/port/ports.rs``. +HOST_PORTS = (2222, 9090, 9095, 10000, 10100, 10101, 11000, 11100, 11200, + 12000, 12100, 12200, 13000, 14000, 15100, 15200, 16000, 17000, + 18000, 19000) def check_host_ports(ports=HOST_PORTS): diff --git a/container/oci-image/tests/docker_local_test.py b/container/oci-image/tests/docker_local_test.py index 1f4bb5eb7..f1ea14eb5 100644 --- a/container/oci-image/tests/docker_local_test.py +++ b/container/oci-image/tests/docker_local_test.py @@ -10,6 +10,7 @@ import unittest from unittest.mock import MagicMock, patch import uuid +import yaml sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from docker_local_cleanup import clean, image_garbage, selected @@ -19,7 +20,7 @@ class DockerLocalTest(unittest.TestCase): def setUp(self): - self.record = {"version": 1, "cluster": "first", "owner": str(uuid.uuid4()), + self.record = {"version": 2, "cluster": "first", "owner": str(uuid.uuid4()), "runtime": {"daemon": "owned", "context": "default", "endpoint": "unix:///owned"}, "image_id": "sha256:retained", "image": "crowdb-node:dev", "nodes": 1, "port_base": 9090, "cpus": 1, "memory_mib": 1536} @@ -109,23 +110,41 @@ def test_changed_container_inputs_are_rejected_before_resume(self): item = {"Name": "/crowdb-first-1", "Image": record["image_id"], "Config": {"Labels": self.labels, "Env": environment}, "HostConfig": {"NanoCpus": 1_000_000_000, "Memory": 1536 * 1024 * 1024, "NetworkMode": "crowdb-first", "PortBindings": { - f"{port}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(port)}] - for port in range(9090, 9094)}}, + f"{port}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(9090+offset)}] + for offset, port in enumerate((9090, 16000, 17000, 18000))}}, "Mounts": [{"Destination": "/opt/crowdb/data", "Name": "owned-volume", "Type": "volume", "RW": True}]} verify_container(item, record, 1, "owned-volume", "crowdb-first") + legacy = dict(item, HostConfig=dict(item["HostConfig"], PortBindings={ + f"{port}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(port)}] + for port in range(9090, 9094)})) + verify_container(legacy, dict(record, version=1), 1, "owned-volume", "crowdb-first") + with self.assertRaisesRegex(ValueError, "endpoints"): + verify_container(legacy, record, 1, "owned-volume", "crowdb-first") for field, value in (("Image", "foreign"), ("Mounts", []), ("HostConfig", {})): with self.subTest(field=field), self.assertRaises((ValueError, KeyError)): verify_container(dict(item, **{field: value}), record, 1, "owned-volume", "crowdb-first") + def test_compose_examples_publish_the_standard_node_listeners(self): + for filename in ("compose.yaml", "compose-three.yaml"): + document = yaml.safe_load((Path(__file__).resolve().parents[1] / filename).read_text()) + for service in document["services"].values(): + self.assertEqual([port.split("}:", 1)[1] for port in service["ports"]], + ["9090", "${CROWDB_S3_CONTAINER_PORT:-16000}", + "${CROWDB_ICEBERG_CONTAINER_PORT:-17000}", + "${CROWDB_DATASET_CONTAINER_PORT:-18000}"]) + def test_receipts_are_immutable_scoped_and_reject_symlinks(self): with tempfile.TemporaryDirectory() as temporary: directory = Path(temporary) seal(self.record, directory) self.assertEqual(load("first", directory), self.record) + legacy = dict(self.record, cluster="legacy", version=1) + seal(legacy, directory) + self.assertEqual(load("legacy", directory), legacy) self.assertEqual(selected("first", False, directory), [self.record]) other = dict(self.record, cluster="second", owner=str(uuid.uuid4())) seal(other, directory) - self.assertEqual(len(selected(None, True, directory)), 2) + self.assertEqual(len(selected(None, True, directory)), 3) with self.assertRaises(ValueError): seal(dict(self.record, owner=str(uuid.uuid4())), directory) (directory / "foreign.json").symlink_to(directory / "first.json") diff --git a/container/oci-image/tests/host_inputs_test.py b/container/oci-image/tests/host_inputs_test.py index 16c9df879..77205ebbd 100644 --- a/container/oci-image/tests/host_inputs_test.py +++ b/container/oci-image/tests/host_inputs_test.py @@ -17,10 +17,10 @@ class HostInputsTest(unittest.TestCase): def test_port_offset_shifts_every_host_listener_for_additional_instances(self): - shifted = run_node.shifted_ports(100) - self.assertEqual(shifted[0], 2322) - self.assertIn(10100, shifted) - self.assertIn(10200, shifted) + shifted = run_node.shifted_ports(1) + self.assertEqual(shifted[0], 2223) + self.assertIn(10001, shifted) + self.assertIn(10101, shifted) with self.assertRaisesRegex(ValueError, "beyond port 65535"): run_node.shifted_ports(65535) diff --git a/container/single-node-container/entrypoint.sh b/container/single-node-container/entrypoint.sh index 8fa74c3bd..6c65fa3fc 100644 --- a/container/single-node-container/entrypoint.sh +++ b/container/single-node-container/entrypoint.sh @@ -69,7 +69,7 @@ if test "$port_offset" -gt 0; then # The preview profile is intentionally self-contained. Shift every # declared listener and endpoint together so host-network containers can # coexist while retaining the same relative service layout. - for base in 9090 9091 9092 9093 9094 9095 10000 10100 10101 11000 11100 11200 12100 12200 13000 15100 15200; do + for base in 19000 18000 17000 16000 15200 15100 14000 13000 12200 12100 12000 11200 11100 11000 10101 10100 10000 9095 9094 9093 9092 9091 9090 7401 7400 7301 7300 7200 7102 7101 7100; do next=$((base + port_offset)) sed -i -E "s/(^|[^0-9])${base}([^0-9]|$)/\\1${next}\\2/g" "$offset_root/profile.toml" if test -d "$offset_templates"; then diff --git a/doc/doc_index.md b/doc/doc_index.md index 903bc2660..232ef5c87 100644 --- a/doc/doc_index.md +++ b/doc/doc_index.md @@ -56,12 +56,13 @@ Temporary plans live under `doc/working/`; flow analyses live under ## Component Acceptance -| Doc | When to read | -| ------------------------------------------- | ----------------------------------------------------- | -| `app/crowdb-access-server/tests/README.md` | Access gates, SDK commands and Dataset measurements. | -| `e2e-test/README.md` | Container domains, browser/SDK ownership and timings. | -| `container/single-node-container/README.md` | Container build, publication and tested client setup. | -| `e2e-test/clients/ecosystem/README.md` | Manual pinned client/engine matrix and exclusions. | +| Doc | When to read | +| ------------------------------------------- | ------------------------------------------------------- | +| `app/crowdb-access-server/tests/README.md` | Access gates, SDK commands and Dataset measurements. | +| `e2e-test/README.md` | Container domains, browser/SDK ownership and timings. | +| `container/single-node-container/README.md` | Container build, publication and tested client setup. | +| `container/oci-image/README.md` | Shared OCI builds, launch examples and retained inputs. | +| `e2e-test/clients/ecosystem/README.md` | Manual pinned client/engine matrix and exclusions. | ## Project Files (repo root) diff --git a/doc/working/plan-two-physical-nodes.md b/doc/working/plan-two-physical-nodes.md index cc8d4a4c1..b8c538477 100644 --- a/doc/working/plan-two-physical-nodes.md +++ b/doc/working/plan-two-physical-nodes.md @@ -5,160 +5,170 @@ Upstream: [deployment design](../design/deploy/design-crowdb-deploy.md), [OCI image guide](../design/deploy/design-crowdb-oci-image.md), -[KV quorum design](../design/kv/design-crowdb-kv-group0.md). - -Goal: validate the current CI reliability fixes on two real Linux hosts using -reproducible CROWDB clusters, with cross-host discovery, key-only admission, -host-network container composition, Group 0 bootstrap, service readiness -recovery, transient KV quorum loss during allocation, and owned cleanup -evidence. - -## Topology and limits - -The basic two-physical-node run uses one logical node per host and one initial -voter. It proves cross-host discovery, key-only admission and durable restart; -it does not claim two-voter quorum. The composition run uses three logical -voters on one host with listener offsets `0`, `200` and `400`; this validates -that a physical host can run multiple host-network containers. A later mixed -placement can put any of those logical nodes on the second host. - -- Physical host A: `u24-amd`, `192.168.31.129`, Linux 6.8, Docker 29.8.1. -- Physical host B: `cj7960`, `192.168.31.184`, Linux 6.11, Docker 28.3.3. -- Inventory evidence: `.crowdb-runtime/artifacts/two-physical-nodes-20261030T010205Z/run-manifest.json`. -- Bidirectional ICMP and SSH/22 reachability passed; CROWDB service ports are - currently closed on both hosts. -- Remote account: `cj`; credentials are used only through SSH/`sshpass` and - are never written to the plan, logs, environment artifacts, or reports. -- Required source/image identity: the current checkout plus a newly built - immutable OCI image; the existing images differ in source revisions and are - not acceptable for the final run. -- Network: direct host addresses on `192.168.31.0/24`; all advertised - management, KV, DiskDB, DiskIO, ChunkDB, Chunk-KV, Access, Web and SSH - endpoints must be reachable from both hosts. Host-network listener offsets - use a stride of at least `200`: offset `100` collides with the adjacent - default KV RPC range (`10000/10100`). - -## Safety gates - -- Only run containers, data directories, bridges and logs labeled with this - run's UUID. Never reuse an existing CROWDB cluster or data root. -- Capture `docker inspect`, image receipt, host inventory, endpoint map and - service logs before cleanup. Cleanup stops and removes only this run's - containers, volumes, networks and remote directories. -- Do not change system firewall, routes, Docker daemon settings, kernel - parameters, physical NIC settings or shared services. A missing port or - privilege is a blocked prerequisite, not a reason to weaken the test. -- Keep the first workload small and bounded. Do not add retries or lengthen - readiness deadlines to hide a failure; record the first failing operation, - logs and persisted status, then stop the affected phase. - -## Phase 1: inventory and owned fixture - -- [x] **Record host inventory**: capture hostnames, IP/interface selection, - kernel, Docker, CPU/memory, disk space, image presence, and bidirectional - SSH/TCP reachability. Files: this plan, run manifest and artifact directory. -- [x] **Add host-network port offsets**: extend the existing - `CROWDB_PORT_OFFSET` path to shift monitor, Web, Access, KV, DiskDB, - DiskIO, ChunkDB, Chunk-KV and SSH listeners; persist shifted setup choices - and use them for Group 0 endpoint publication. Verify the launcher offset - validator and shell syntax. Files: `container/single-node-container/entrypoint.sh`, - `container/crowdb-monitor/src/node_args.rs`, - `container/crowdb-monitor/src/node/runtime/kv.rs`, - `container/oci-image/run-node.py`. -- [x] **Build and distribute exact image**: build with `pixi run oci-build`, - record source tree and image digests, transfer/import the same archive on - both hosts, and verify the digest plus required runtime labels. The final - flexible-port image was also built locally for the three-container run; - transfer it to host B before a mixed-placement run. -- [x] **Prepare isolated roots**: create one run UUID, per-host data/log roots, - a unique Docker network or host-network port map, and a machine-readable - endpoint map. Verify no endpoint is already owned. -- [x] **Start two logical nodes**: launch one node per physical host with - `initial_voters = 1`, distinct data roots and explicit physical-host IDs. - Verify both candidates are visible before admission. -- [x] **Start three composed nodes**: launch three offset nodes on one - physical host with offsets `0/200/400`, verify all listener families are - unique, and verify all three candidates are visible before admission. - -## Phase 2: bootstrap and normal cross-host behavior - -- [x] **Admit and bootstrap the two-node run**: create one rack record, admit - both discovered nodes with `ssh_password = null`, initialize one-voter Group - 0, and record node IDs, discovery IDs and cluster ID. -- [x] **Admit and bootstrap the composed three-node run**: create one rack, - admit all three offset nodes with key-only SSH, initialize three-voter Group - 0, and verify management/RPC ports `10000/10100`, `10200/10300` and - `10400/10500`. -- [ ] **Verify dependency readiness**: wait for KV, DiskDB, DiskIO, ChunkDB, - Chunk-KV and Access readiness on all logical nodes. Confirm the monitor - status is `ready` and identities/PIDs are stable after one health interval. -- [ ] **Run bounded data corpus**: perform KV put/get/batch, S3 bucket/object - put/get, one Iceberg metadata round trip and one ChunkDB small allocation; - verify exact bytes and read through both physical-host endpoints. - -## Phase 3: failure and recovery cases - -- [x] **Remote logical-node restart**: stop and recreate host B's node using - the same data root without mounting a password file. Verify the original - discovery identity, cluster ID, active Group 0 status and key-only SSH. -- [ ] **Remote logical-node stop**: stop only host B's CROWDB node. Verify the - two host-A voters retain Group 0 quorum, linearizable KV writes continue, - existing S3 reads continue, and status identifies the stopped node. -- [x] **Key-only composed restart**: recreate each of the three composed - containers without `CROWDB_SSH_PASSWORD_FILE`; verify - `PasswordAuthentication no`, all three dynamic KV health endpoints and - active Group 0 status. -- [ ] **Managed listener failure**: stop or fault only the selected Chunk-KV - listener while leaving its process ownership observable. Verify consecutive - readiness failures trigger the monitor's bounded restart, the PID changes, - Access becomes ready again, and no caller timeout is modified. -- [ ] **Transient allocation quorum loss**: during a bounded small allocation, - pause one KV voter long enough to produce the known `leadership quorum - unavailable` response, restore it, and verify ChunkDB allocation recovers - through the allocation-only transient path. Verify outcome-unknown cases - remain terminal and no duplicate chunk metadata is published. -- [ ] **Expected two-voter host loss**: stop host A's two logical nodes only - after the preceding recovery cases pass. Record the expected Group 0 quorum - loss and the exact readiness/error surfaces, then restore both nodes and - verify durable state recovery. This is an observation case, not a pass claim - for availability while two of three voters are down. - -## Phase 4: evidence and cleanup - -- [ ] **Collect evidence**: save commands, source/image identity, endpoint map, - host inventory, cluster status snapshots, monitor events, service logs, - process identities, workload results and first-failure diagnostics. -- [ ] **Run final health/data gate**: after all recovery cases, require all - three logical nodes and their enabled services ready, repeat KV and object - reads from both physical hosts, and check no unexpected process/container - remains. -- [ ] **Cleanup owned resources**: stop/remove only this run's containers, - networks, volumes and remote directories; verify the endpoints are closed - and retain artifacts outside deleted data roots. -- [ ] **Report result**: mark each case pass, expected observation, failed or - blocked with evidence. Do not mark the run complete if the image, placement, - or cross-host path was not proven. - -## Consolidated files and commands - -- Plan/tracking: `doc/working/plan-two-physical-nodes.md`, - `doc/working/track-two-physical-nodes.md`. -- Image: `container/oci-image/build.py`, `container/oci-image/run-node.py`, - `container/oci-image/control.py`. -- Existing lifecycle references: `e2e-test/profiles/node-bootstrap.py`, - `e2e-test/profiles/containerd-cluster.py`, `e2e-test/console_native_fixture.py`. -- Local quality gates: `pixi run cargo fmt --all -- --check`, affected Rust - tests/clippy, and `pixi run oci-build`. -- Remote checks: `ssh`, `docker`, `docker inspect`, `docker logs`, and the - repository's CLI/HTTP cluster status commands, all recorded in the run - manifest. +[KV quorum design](../design/kv/design-crowdb-kv-group0.md), +[Chunk-KV recovery design](../design/chunkds/design-crowdb-chunk-kv-server.md). + +Goal: validate discovery, key-only admission, host-network composition, +service readiness, bounded data operations and recovery on two real hosts. +Results and receipts: [track-two-physical-nodes.md](track-two-physical-nodes.md). +The physical three-container matrix and requested CI selections pass. The plan +remains open for authorized publication and the five-container physical follow-up. + +## Fixture and constraints + +- Host A: `u24-amd`, `192.168.31.129`; host B: `cj7960`, `192.168.31.184`. +- Final placement: two logical voters on A, one on B; host-network offsets + `0/1` on A and `0` on B. Each node owns a separate labeled volume. +- All remote work now uses `cj` and `id_ed25519`, key-only SSH. Existing + authorized keys are preserved; passwords are absent from run artifacts. +- Build and execute through `pixi run`. Record immutable OCI receipts and + compare execution configuration plus ordered rootfs layers on both hosts. +- Never alter shared firewall, Docker, NIC, routes or kernel settings. + Stop/remove only resources verified to carry the exact run label. +- Preserve the first failing operation and complete logs. Do not lengthen + deadlines or add caller retries to turn a failure into a pass. + +## Completed prerequisites and baseline + +- [x] **Inventory and reachability**: capture hosts, interfaces, Docker versions, + resources, endpoint ownership and cross-host SSH/TCP reachability. +- [x] **Repair host SSH**: install the existing public key on B, preserve other + keys, verify permissions and key-only `BatchMode` login. +- [x] **Compose offset nodes**: verify all canonical listener blocks and + offset-resolved setup choices. Local offsets `0/1/2` reached active Group 0. + Files: `container/oci-image/run-node.py`, + `container/single-node-container/entrypoint.sh`, + `lib/crowdb-protocol/src/mgmt/node/deployment.rs`, + `container/crowdb-monitor/src/node/startup/policy.rs`. +- [x] **Persist accepted admission choices**: use the target-resolved choices + for endpoint publication when the request omits setup. Files: + `app/crowdb-web/src/node/{admission,setup}.rs`. +- [x] **Separate DiskDB listener blocks**: support explicit main/HTTP/RPC ports, + shift each once, retain custom legacy base behavior, and prove adjacent + instances disjoint. Files: protocol node setup/validation/deployment, + Web DiskDB lifecycle/configuration and planned deployment, monitor policy. +- [x] **Exact mixed-placement image and bootstrap**: import the verified image + on both hosts, admit all three nodes without passwords, initialize + three-voter Group 0, provision production disk media, and reach seven + healthy services per node. Files: attempt-12 receipts and snapshots. +- [x] **Bounded baseline corpus**: KV put/get/batch, 96 KiB S3 object, + Iceberg metadata, and a small mirrored ChunkDB allocation; cross-host exact + reads. Files: attempt-12 corpus and native-client evidence. +- [x] **Retained-volume recreation**: recreate all three nodes with the + corrected image and original identities, without a password-file mount. + +## Recovery verification and remaining work + +- [x] **Managed listener recovery**: fault one owned Chunk-KV child, require a + changed PID within the existing bound, all services ready and cross-host + corpus reads. Strengthen Access startup to probe every catalog owner before + another launch. Files: monitor service dependencies and regression tests. +- [x] **Allocation retry contract regression**: use real RPC responses to prove + known `Unavailable` retries the same chunk ID and a lost reply submits once. + File: `app/crowdb-chunkdb/tests/allocation_retry_test.rs`. +- [x] **Observe remote loss**: stop B, verify KV quorum write/read, retain the + S3 `ServiceUnavailable` result and failed 180-second catalog failover wait, + then restore B. This completed observation is a failed availability case. +- [x] **Diagnose recovery defects**: reproduce the unfinished owner-2-to-owner-1 + `TargetPreparing` transfer and explain why it cannot finish; inspect the + planner's busy-owner exclusion and the target preparation dependency. + DiskDB stale-owner discovery and released-source WAL/grant fixes pass focused + regressions. Attempt 15 also reproduced empty sealed-WAL recovery rejection; + the production adapter regression passes. The upgrade then exposed reclaimed-journal recovery losing live DiskDB occupancy; KV and DiskDB restart regressions now pass. Preserve the corrupted attempt-15 evidence and run a fresh dynamic-ownership matrix after rebuilding. Files: Chunk-KV transition worker/monitor, DiskDB ownership lifecycle, + and focused regressions selected from first divergence evidence. +- [x] **Repeat physical allocation retry**: attempt 15 retained three voters, + injected Group-1-only leadership unavailability, observed the explicit + allocation-level `Unavailable` retry, and completed same-ID allocation/query + within the original five-retry/10-second limit. The KV processes and Group 0 + stayed live. Earlier process-pause attempts are separate failed observations. +- [x] **Repeat remote storage recovery**: after repairing the defects, stop B, + require bounded range recovery and one-shot S3/Iceberg reads, recreate B with + its volume/key-only admission, and recheck identities and exact data. +- [x] **Repeat expected two-voter host loss**: after preceding recovery passes, + stop both A nodes, record expected Group-0 unavailability, restore them and + verify all services and durable data. Earlier-image evidence is historical. +- [x] **Pass final health/data/allocation gate**: require stable healthy PIDs, + existing KV/S3/Iceberg reads, a fresh batch and fresh allocation/query. + Attempt 12 passed restored health, existing reads and batch, but failed fresh + allocation with `disk-group not owned`. + +## Evidence and cleanup + +- [x] **Freeze diagnostics**: retain status, inspect, policies, receipts, + transitions, workload errors and complete logs. Capture final log archives + after container stop to avoid concurrent-write tar failures. +- [x] **Clean exact owned resources**: remove attempt-12 through attempt-16 containers and volumes, + delete its remote transfer archives, verify all owned endpoints closed, + retain artifacts outside deleted volumes and retain host-B SSH access. +- [x] **Report actual coverage**: distinguish passes, failed recovery cases, + earlier-image observations and the physical retry coverage gap. ## Verification groups -- Unit: current monitor and ChunkDB regression tests already added for the CI - failures. -- Integration: image/launcher validation, endpoint ownership, discovery and - SSH admission. -- Physical E2E: three logical voters over two physical hosts, normal writes, - one-voter loss/restart, managed listener recovery, transient allocation - recovery, expected two-voter quorum loss and durable recovery. +- Protocol: node deployment/setup contract tests and full crate tests. +- Web: DiskDB deployment configuration tests and full crate tests. +- Monitor: node policy, service dependencies and service readiness tests. +- ChunkDB: allocation retry tests and dynamic retry tests. +- Quality: Rust fmt, affected crates' clippy with warnings denied, launcher + Python tests and shell syntax; immutable OCI build for production changes. +- Physical: repeat only the affected matrix after the first failures are + repaired, preserving successful baseline evidence and default limits. + +## After physical validation: requested CI jobs + +- [x] **Fetch requested CI evidence**: retain all eight job logs and map the + exact failed cases in run 41003611925 after completing physical attempt 16. +- [x] **Repair retained Monitor fixtures**: accept valid startup policy before + identity, resume and cleanup operations. The exact two failures pass; the + earlier passing Monitor targets and the remaining targets are retained. + Files: `container/crowdb-monitor/tests/{kv_resume,node_cleanup}_test.rs`. +- [x] **Repair common Console/bootstrap policy**: the exact inspection case + reproduces container exit due to disabled Paxos KV, followed by no route to + host. Keep mandatory KV listeners enabled while skipping storage setup; + verify the original case and affected Console/deployment selections. + Files: `e2e-test/profiles/setup_choices.py`, native policy, bridge-node and + reference-recovery fixtures. +- [x] **Preserve live services during readiness recovery**: native inspection + passed serially after bootstrap repair but a concurrent-load run exposed a + monitor restart during normal transfer readiness withdrawal. Recorded HTTP + evidence distinguishes `/ready` 503 from `/health` 200. Keep readiness for + admission/status and liveness for restart decisions; preserve dead-listener + restart assertions and verify the new image with the affected case. + Files: monitor runtime health/probes and service readiness regression. +- [x] **Verify Go SDK failure cleanup**: focused and complete S3 selections + pass. Retain safe operation/error diagnostics; the original opaque CI + cleanup cause cannot be established from its artifact. +- [x] **Skip ordinary metadata for diskless KV-only setup**: the mandatory + Group-0 prerequisite must not reserve a default Group 1. Reproduce the + metadata race, retain both disabled/KV-only unit coverage, repair the + preparation predicate, rebuild and resume the stopped native-data group. +- [x] **Resolve remote wildcard topology addresses**: HTTP Console topology + reports must inherit the reporting server management host before entering + the shared cache. Reproduce false loopback Group-0 authority loss after + stopping a local follower, reuse the existing endpoint resolver, add an + HTTP/cache regression and rerun the affected lifecycle case. +- [x] **Publish canonical protocol listeners with retained-input compatibility**: + repair Pixi/CLI/Compose mappings and image metadata, retain version-1 launch + ports while sealing fresh version-2 inputs, verify mapping/receipt regressions + and CLI quality gates; the image rebuild is complete before reference verification. +- [x] **Complete requested Console partitions**: browser, native-data, + native-operations, transition and inspection, with original coverage and + fail-fast progression. Keep browser fixture networks isolated from other jobs. +- [x] **Complete deployment reference**: rerun its original case set after the + shared bootstrap repair, then diagnose any remaining failure. +- [x] **Close AccessTests and quality gates**: run the requested Access component + and SDK commands once, fmt, affected clippy and required C++ lint before the + first coherent commit/push. +- Job IDs: `123893403740`, `123899615935`, `123899616023`, `123899615992`, + `123899616153`, `123899616076`, `123899616075`, `123899615882`. +- Run: https://github.com/buzzcrow/crowdb/actions/runs/41003611925 + +## Authorized publication and five-container follow-up + +- [~] **First publication**: user authorized a single coherent commit and push + after the physical matrix, requested CI repairs and relevant gates pass. +- [ ] **Five-container physical matrix**: after that push, run a fresh cluster on the same two physical + hosts, retaining immutable image receipts, cross-host data witnesses, fault + and recovery results, and exact owned-resource cleanup. +- [ ] **Final publication**: repair failures and verify affected groups, then commit and push the + resulting five-node validation changes and evidence summary a second time. diff --git a/doc/working/track-two-physical-nodes.md b/doc/working/track-two-physical-nodes.md index b92111dc7..f6b20a2e3 100644 --- a/doc/working/track-two-physical-nodes.md +++ b/doc/working/track-two-physical-nodes.md @@ -5,45 +5,436 @@ Plan: [plan-two-physical-nodes.md](plan-two-physical-nodes.md) -Run ID: `two-physical-nodes-20261030T010205Z` - -## Preconditions - -- [x] Local host inventory captured. -- [x] Remote host inventory captured through SSH. -- [x] Direct peer reachability and SSH reachability verified. -- [x] Host-network port-offset support verified by focused test, clippy and - shell syntax checks. -- [x] Current source/image identity recorded and identical on both hosts. -- [x] Final flexible-port image built locally: digest - `sha256:7f4c32b59e3ea9315a7726f6667ae6d5f4ea795f70c29c920371d045a80cf42d`. -- [ ] Final flexible-port image transferred to host B for mixed placement. -- [x] Isolated run UUID, roots, network and endpoint map created. - -## Bootstrap - -- [x] Two logical nodes launched: host A ×1, host B ×1. -- [x] Two candidates discovered and admitted with key-only SSH. -- [x] One-voter Group 0 initialized and remote node restarted without a password file. -- [x] Three logical nodes launched on one host with offsets 0/200/400. -- [x] Three candidates discovered and admitted with key-only SSH. -- [x] Group 0 initialized with three voters and all dynamic KV listeners healthy. -- [ ] All required services ready on all logical nodes. -- [ ] Baseline KV/S3/Iceberg/ChunkDB corpus passed. - -## Recovery matrix - -- [ ] Stop host B logical node: two-voter quorum remains available. -- [ ] Restart host B logical node: identity and data retained. -- [ ] Managed Chunk-KV listener failure: monitor restarts unhealthy service. -- [ ] Transient allocation quorum loss: allocation-only recovery succeeds. -- [ ] Stop host A's two logical nodes: expected quorum loss recorded. -- [x] Restore the two-node run and all three composed nodes with key-only SSH. - -## Evidence and exit - -- [x] Status snapshots, process identities and monitor events saved. -- [x] Service/container logs saved for every phase. -- [ ] Final cross-host reads and health gate passed. -- [ ] Owned resources cleaned; endpoints closed. -- [ ] Result report completed with pass/expected/failed/blocked classification. +Run ID: `two-physical-nodes-20261030T010205Z`. +Latest fixture: `physical-20261030-r16`; result: **physical recovery matrix passed**. +SSH access is repaired. All owned physical fixtures are cleaned up. Attempt 15 +forensic metadata/WAL, raw mirror reads and complete logs remain archived; +attempt 16 passed using fresh test data and the retained preflight policy +volumes. Historical failures remain recorded below. + +## Attempt 12 placement and image (historical) + +- Host A: `u24-amd`, `192.168.31.129`, two logical voters, offsets `0/1`. +- Host B: `cj7960`, `192.168.31.184`, one logical voter, offset `0`. +- Cluster ID: `475d2a73-19f4-421e-8ccf-39693ef368c2`. +- Host networking, separate retained volumes, one CPU and 2 GiB per node. +- Final OCI digest: + `sha256:105cd3d3fccad16b1e907e3ba7a7f8cad327319a9f678a4815cdd09921a9b080`. +- Build revision: `5d03757b75a9b1867eebbcab9346e84d153bdcf2`; source snapshot: + `b1a845605cd9f41136a67e81a91f51119912639b`. +- Both hosts imported the same archive. Execution configuration and ordered + rootfs layers were verified, including across Docker 29/28 import formats. + The receipt records the production sources at build time; the allocation + retry tests and this report were added afterward. + +## Attempt 12 verified observations (historical) + +- [x] Host inventory, direct reachability, and distinct host-network listeners. +- [x] Host-B `cj` key-only SSH using `/home/cj/.ssh/id_ed25519` with + `BatchMode=yes` and `IdentitiesOnly=yes`; public key installed without + removing existing authorized keys. SSH directory/file modes verified. +- [x] All discovery candidates admitted with `ssh_password = null`. +- [x] Three-voter Group 0 active across the two physical hosts. +- [x] All seven services healthy on every node after production disk setup. +- [x] KV put/get/batch, 96 KiB S3 object exact-byte reads, Iceberg namespace, + table and property round trip, and initial small mirrored allocation. +- [x] All three containers recreated from retained volumes with the corrected + image, original identities retained, and no password-file environment. +- [x] Managed Chunk-KV listener fault: `SIGSTOP` on the owned child, bounded + monitor replacement with a different PID, all services healthy, then + KV/S3/Iceberg reads through all three endpoints. +- [x] With host B stopped, a Group-3 KV write and read succeeded through host A. +- [x] After host B returned, existing KV/S3/Iceberg data read correctly from + every endpoint. All seven services had stable PIDs across the next health + pass; a final native KV batch also passed. +- [x] Focused RPC tests prove a known allocation `Unavailable` retries with the + same chunk ID, while a lost allocation reply returns `OutcomeUnknown` and + submits exactly once. These are separate from physical fault coverage. +- [x] Full final service logs frozen after container stop, inspect/status and + policy snapshots retained, and exact owned containers/volumes removed. + All owned endpoints closed; the host SSH key remains installed. + +## Attempt 12 failures and coverage limits (historical) + +These failures are retained as first-divergence evidence. Attempt 16 below +passed the repaired storage failover, fresh allocation, physical safe-retry +and expected majority-host-loss cases. + +- [ ] **Host-B storage availability:** the first S3 read while B was stopped + returned `ServiceUnavailable` with caller retries disabled. A subsequent + observation-only wait also failed: after the existing 180-second bound, + catalog generation 14 still assigned B's range to owner 3. A persisted + `TargetPreparing` transfer from owner 2 to owner 1 reserved both surviving + owners. That explains why the current planner had no eligible target; the + reason preparation remained unfinished required diagnosis. Returning B + restored corpus reads. Availability during B's absence did not pass. +- [ ] **Final allocation:** a fresh native allocation after restored health + failed with `diskdb allocate failed for disk_group 2: diskdb server does not + own the requested resource: disk-group not owned`. Listener health and old + data reads therefore do not establish restored allocation readiness. +- [ ] **Physical allocation-only retry:** a bounded pause of the two follower + KV processes was followed by successful allocation and query, but the + instrumented client observed no allocation-level safe retry. The physical + case does not prove that specific path. Retry limits and caller deadlines + were unchanged. Pausing only one of three voters cannot remove quorum. +- [ ] **Final-image two-voter host-loss repeat:** not run after the preceding + recovery failure. The expected quorum-loss observation passed in attempt 11 + on its earlier image and is retained as historical evidence only. + +## Repairs validated during continuation + +- Admission now publishes the target's accepted, offset-resolved setup choices; + omitted choices previously published identical KV endpoints for every node. +- Standard DiskDB policy uses independent main/HTTP/RPC blocks + `11000/11100/11200`, each shifted once. Adjacent offset nodes previously + collided because the RPC field was interpreted as a legacy three-port base. + Explicit listeners preserve legacy custom-base deployment behavior. +- Access startup probes every catalog owner before another process launch; + a healthy owner previously masked another owner replaying its WAL and + consumed the restart budget. Tests cover redirects, multiple healthy owners + and an unavailable second owner. The physical listener fault then passed. + +## Evidence + +Root: `.crowdb-runtime/artifacts/two-physical-nodes-20261030T010205Z/`. + +- `attempt10-local-fixed/`: local offsets `0/1/2`, active Group 0; no storage + readiness claim for that diskless fixture. +- `attempt11-final-physical/`: earlier mixed-placement KV/quorum/recreation + results, expected two-voter loss, and cleanup. Its storage phase failed. +- `attempt12/result.json`: historical case classification and next unrun work. +- `attempt12/catalog-all-owners-fix-receipt.json`: final image receipt. +- `attempt12/baseline-catalog-all-owners-fix/`, `listener-fixed/`: + corrected-image baseline and successful managed recovery evidence. +- `attempt12/remote-stop-{catalog,transfer,split}.json`, + `matrix-failure-remote/`: failover diagnostics and durable transitions. +- `attempt12/native-final-allocation.json`, `final-allocation-failure/`: + final allocation error and complete frozen service logs. +- `attempt12/corpus-read.json`, `final-health-stable.json`, `cleanup.json`: + restored reads, stable process observations, and verified cleanup. +- Reproduction scripts are retained in `attempt12/`; client-only retry logging + lived under `/tmp` and did not change production client sources. + +## Local verification + +- Passed: full `crowdb-protocol` and `crowdb-web` Rust tests. +- Passed: monitor policy/dependencies/readiness, 15 tests; ChunkDB allocation + and dynamic retry, five tests. +- Passed: clippy for protocol, monitor, Web and ChunkDB with tests and + `-D warnings`; Rust fmt and `git diff --check`. +- Earlier unchanged launcher verification: 134 OCI Python tests and shell + syntax passed. The final production image build/import passed. +- Default `ssh -o BatchMode=yes cj@192.168.31.184` also succeeded without an + explicit identity flag after fixture cleanup. + +## Attempt 13 diagnostics (historical) + +- User requested continued repair and full physical validation before inspecting + the eight linked CI jobs. +- Corrected stale DiskDB ownership rejection routing and released-source + Chunk-KV grant/WAL recovery; focused regressions and affected clippy pass. +- OCI digest: `sha256:3b21576d0bb18aec223bdb4cd4916b6400c76d3b1cf74a883afa90eb588d78f7`; + source snapshot `8d0e21fb044c27b8dd0e9bcb98cf4de1f808b7fd`. +- New owned fixture `physical-20261030-r13` reached active three-voter Group 0 + and seven healthy services on every node. Baseline KV/S3/Iceberg reads pass. + Retained-volume recreation and the remaining matrix are in progress. +- The corpus harness now waits for a newly created data group's elected leader + before its first write; the initial immediate write encountered election + startup. This adds a setup assertion, not a retry of a data operation. +- Evidence is retained in `attempt13/`; attempt-12 failures remain preserved. + +- Attempt 13 retained-volume recreation and listener recovery passed. The + isolated quorum case exposed a stale KV route to a removed replica: the + missing-local-group response did not trigger discovery. Regression first + failed, then passed for point reads/writes, conditional writes and scans; + the 65-test KV store suite also passed. Missing-group responses now carry + the existing typed NotLeader status, with legacy point-response fallback. +- The temporary two-voter metadata fixture retired its removed replica's local + creation record; it is not reused for the later three-voter recovery cases. + Its logs, membership errors and cleanup evidence remain in `attempt13/`. + +## Attempt 14 diagnostics + +- The isolated Group-1 outage also affected DiskDB metadata and therefore + produced terminal disk-allocation errors before an allocation-only retry. + These are failed observations, not retry coverage. The removed replica is + retired and the invalid two-voter production fixture is not reused. +- Recovery publication incorrectly applied the observer's local KV management + and RPC ports to every member. Binding and service receipts showed duplicated + Host-A seeds and an incorrect offset on Host B. A mixed-port/same-host/IPv6 + regression reproduced this, then passed after retaining each member's ports. +- All 13 monitor member-trust tests and monitor clippy pass. Attempt-14 owned + containers, volumes and transfer archives were removed; frozen logs remain. +- User authorized the full physical and CI repair commit/push, followed by a + separate five-container test on these two hosts and another commit/push. + +## Attempt 15 continuation + +- OCI `sha256:5a9495fc919858e5c4aebc707c35115dce769aeb3646dba88d83c489660a7e43`, + source snapshot `41c9b797a28641a2408d3fa3562805d754f25a7e`, passed baseline, + all-node retained-volume recreation, and managed listener recovery. +- Every binding retained the same correct three mixed-port management seeds. +- Allocation-only retry passed with the original five-retry/10-second client + limits: concurrent Group-1-only step-downs left all three voters and Group 0 + live, produced a known `Unavailable` response, and then allocation/query + succeeded for the same chunk identity. This validates transient leadership + unavailability; it is not a claim of physically pausing two KV processes. +- Host-B-stop catalog recovery failed within the existing 180-second bound. + A target rejected a sealed, physically empty WAL chunk whose strips had + legitimately been returned by `trim_sealed_strips`. The authoritative chunk + had zero capacity, sealed length and acknowledged cursor. Recovery incorrectly + required at least one mirror strip. B was restored in the fixture's finally + block, and complete diagnostics remain under `attempt15/matrix-failure-remote`. +- A regression reproduces the empty sealed tail and checks active, nonempty, + wrong-purpose and stale-snapshot rejection. The 47-test production chunk + adapter suite passes after handling only this canonical empty sealed state. + The existing physical fixture will be upgraded with its volumes preserved. + +- Upgrading the retained fixture to OCI + `sha256:0f9326c2f4b08b8e069983b9348efad7dea88a2e275392dde0c6c7d189828ae9` + resolved the previously stuck empty-tail transfer; all services and corpus + reads recovered, and listener recovery passed again. +- The next Host-B-stop run exposed missing repair admission on a failed old-WAL + drain. The referenced active chunk remained at writer epoch 4, with no new + epoch-5 allocation: failure occurred before successor allocation. The + recovery path returned on an offline mirror without feeding the existing + durable repair queue. The fix drains every mirror, conditionally marks failed + segments unavailable only when a drained survivor exists, and keeps recovery + unavailable until the replacement layout itself drains successfully. It does + not mark transport failures as corruption or omit failed drains from success. +- The ChunkDB crash-gap regression confirms unavailable layout records are + admitted by its repair scanner. Focused recovery tests cover lost metadata + replies and the absence of any drained survivor. + +- All 49 production adapter tests pass, including durable failure-report reply + loss and all-mirror drain failure. The existing recovery IO deadline remains + unchanged. The repaired physical matrix remains the next required gate. + +## Attempt 15: reclaimed journal exposed duplicate physical allocation + +- A1 Chunk-KV recovery rejected tree `4789280886995677209`: both mirrors of + chunk `216631886962935262:3939770405816213384` contained zero bytes in the + first 262284 bytes, including the frame at offset 65606. +- Its sealed tree metadata still referenced A2 disk unit 318 and A1 disk unit + 321. Current DiskDB busy records assigned both units to newer WAL chunks. + Allocation timestamps and chunk records are retained in + `attempt15/allocation-records.log` and `overlapping-chunks.log`; both raw + mirror reads and complete service logs are retained with the fixture. +- A deterministic restart regression reproduced the upstream fault: a durable + KV engine retained a busy record while earlier WAL segments were reclaimed; + the restored acceptor still had trim watermark zero before maintenance. + `JournalScan` returned success with empty history, so DiskDB recovered an + incomplete occupancy bitmap and could reuse a live extent. +- `JournalScan` now requires every slot in the requested applied interval, + including NoOps, before filtering by key. Missing history returns the existing + typed GC-gap error, selecting DiskDB's full-scan fallback. The regression was + observed failing before the fix and passing afterward. The full KV store + suite passed 66 tests. A separate DiskDB integration regression verifies the + old busy unit remains reserved and is excluded from subsequent allocations. +- This does not repair overwritten mirror contents. Attempt 15 remains a + failed durability case; preserve its evidence and repeat the matrix with + fresh owned volumes after rebuilding. No data-loss case is counted as a pass. +- The next fixture explicitly accepts ChunkDB dynamic ownership policy. + Attempt 15's fixed ownership policy cannot establish availability while a + ChunkDB owner is stopped; changing ownership mode requires a fresh accepted + deployment rather than silently changing the existing cluster. + +- The existing DiskDB recovery suite passed all eight tests, and KV/DiskDB + clippy with tests and denied warnings passed. The failed fixture is stopped; + its volumes remain intact, with frozen metadata/WAL and service-log archives + for all three nodes under `attempt15/allocation-overlap-frozen/`. + +## Attempt 16: explicit policy file and nonzero port offset + +- The fresh dynamic-ownership fixture stopped at A2 startup: `deployment inputs + differ from accepted node policy`. The preflight had sealed shifted service + ports, but the following monitor invocation compared the unshifted file + inputs against that accepted record. No Group-0 bootstrap or data corpus had + started. A1/A2 volumes are retained for retry with the same accepted inputs. +- Port resolution now compares explicit file/argument listeners in the accepted + port space and hands the complete resolved policy to preflight/runtime once. + Repeated acceptance preserves policy bytes; a changed listener is rejected. +- The new repeated-file-acceptance regression failed before repair. All 20 + policy, CLI-input and management-input tests pass, as does monitor clippy + with tests and denied warnings. `pixi run tree-lint` also completed with exit + zero; its complete diagnostic output is retained with verification logs. +- First-start failure logs and the prior image receipt remain under + `attempt16/before-policy-file-offset-fix/`; subsequent bootstrap has not yet + been verified on the corrected image. + +- Corrected attempt-16 image: OCI + `sha256:faab2bd15b6abe37598caa9a78e8e6d75d38490559f2ae97144d05e1846aa8fc`, + source snapshot `59bca7e66bddd0454c2a3121b57dd6219264e3c2`. Both hosts + imported matching configuration/rootfs layers. All three execution records + explicitly contain `CROWDB_CHUNKDB_OWNERSHIP_POLICY=dynamic`. +- All three nodes initialized and reached seven healthy services. KV batch/data, + S3 exact-byte payload and Iceberg metadata reads passed through every node. +- All containers recreated with their retained volumes and identities; all + services and three-endpoint data reads passed again. Native scans found no + cross-chunk physical overlap among 134 live mirrored segments before + recreation or 238 afterward. +- Managed Chunk-KV `SIGSTOP` recovery completed inside the existing 60-second + bound with a changed PID, followed by all-service health and exact corpus + reads on every endpoint. Host-B stop/recovery is the next active case. + +- Host B stopped: Group-3 KV quorum write/read passed, all catalog entries + returned to surviving owners within the original bound, and one-shot S3 and + Iceberg corpus reads passed through both A endpoints while B stayed stopped. + B then returned and was recreated from its volume: all services and exact + reads passed on all three endpoints. This supersedes the earlier failed + remote-availability observations for this corrected dynamic-policy image. + +- The first host-A loss case confirmed quorum loss and rejection of a write. + Its recovery assertion then failed with S3 connection refused because the + harness read a retained pre-stop `monitor.json` as current readiness. + Failure capture showed A1/A2's new snapshots had no Access entry yet, while + the earlier readiness observations had already accepted the old seven-service + snapshot. The same processes later became ready without another intervention. +- The physical harness now requires status timestamps to follow the current + Docker container start and be no older than the existing three-second status + freshness contract. Failure capture still retains raw stale evidence. The + original 180-second recovery bound and one-shot corpus reads are unchanged; + repeat this host-loss case with the corrected assertion before advancing. + +- Repeating the host-A two-voter loss with current-status checks passed: + Group 0 became unavailable, a write could not complete during quorum loss, + and all seven services plus exact KV/S3/Iceberg reads recovered at all three + endpoints. The timed-out write has an unknown eventual outcome; this case + asserts unavailable write completion during the outage, not nonexistence of + its key after quorum returns. +- Group-1-only rapid leadership step-downs reproduced the explicit native + allocation `Unavailable` retry (`retries exhausted after 11 attempts, last + error: not leader`). Releasing the fault completed same-ID allocation/query + within the unchanged client bounds, with all three metadata voters retained. + All service checks and three-endpoint corpus reads passed afterward. + +- Final attempt-16 gate passed: every catalog partition Serving, all seven + services per node healthy with stable PIDs across the next health pass, a + fresh native KV batch and fresh allocation/query, plus exact corpus reads + through all three endpoints. A final native metadata scan found no + cross-chunk overlap among 2432 live mirrored segments. +- Attempt-16 and the quarantined attempt-15 owned containers/volumes were then + removed after freezing final service logs. Exact run labels were checked; + all owned endpoints closed, transfer/policy files were removed, and the host + SSH key was retained. Cleanup receipts are stored in each attempt directory. + +## Requested CI continuation + +- All eight requested job logs are retained under `ci/run-41003611925/`. + The original run used commit `39440d023d7fb162dd61c377e8eb17eefa8c4c9a`. +- AccessTests: retained Monitor startup/resume/cleanup fixtures now seal accepted + policy before identity effects. Exact failures, full Monitor tests, component + Access tests, official SDK tests, Dataset SDK boundary/lint/tests and Python + installed-consumer checks pass. Python coverage includes 21 consumer cases + and the real HTTP Split SDK round trip. Records: + `ci/ci-dataset-sdk-results.json`, `ci/ci-access-tests.log`, + `ci/ci-access-sdk.log`. +- Console/deployment bootstrap: diskless fixtures incorrectly disabled mandatory + Paxos KV, causing node exit followed by `No route to host`. Shared fixtures + now retain explicit Group-0 KV listeners and disable ordinary storage setup. +- Monitor restart decisions now use liveness. Normal ChunkKV transfer readiness + withdrawal previously caused responsive children to be replaced. The + regression preserves responsive PIDs beyond the failure threshold and still + requires replacement for a genuinely unhealthy listener. Published status, + startup dependencies and admission retain readiness checks. +- KV-only diskless setup must not reserve default Group 1. Dataset browse + reproduced the race (`200` instead of absent-group `404`), and the setup + regression reproduced `Running` instead of `Ready`. Preparation now excludes + this mandatory prerequisite when storage is diskless. All 12 setup tests, + full Monitor tests and clippy pass. +- S3: all 45 original cases pass with two slots and the original resource and + deadline budgets, including official SDK injected cleanup, storage outages, + process recovery and protocol parity. Artifact: + `container-e2e/run-76afe2e0dfe24ac6a583002643fddd3d`; log: `ci/s3-final.log`. + The original Go cleanup failure is unclassified: its artifact omitted the + underlying error. Focused and full executions pass; safe operation/error-code + diagnostics now preserve evidence if it recurs. +- Console browser: all 27 cases pass, including original failed replica + navigation and logical creation. Artifact: + `container-e2e/run-810e73b204924f97bd774df1c148dbc0`; + log: `ci/console-browser-final.log`. +- Native data: all 14 original cases pass across the five Dataset cases, + six-node round trips and ordered remainder. Original data assertions, + balancing, process-preservation checks and deadlines remain. Records: + `ci/remaining-sequence.json`, `ci/native-data-remainder-cases.json`. +- The candidate review helper now expects mandatory Paxos KV checked and the + other five services unchecked; all fixed choices remain read-only. Six-node + verification passes. Bridge default-policy assertions now use the accepted + DiskDB RPC port 11200 and one 16-GiB zone. No visible UI behavior changed. +- Complete bridge admission passes SSH proof, candidate review, foreign-cluster + rejection, restart, service lifecycle, minority authority gating and default + setup. It verifies retained 16-GiB disk identity after Monitor restart, + bitmap/capacity properties and no OOM/PID exhaustion. Artifact: + `container-e2e/run-29a8a32454d945c795c825e79084525b`; + log: `ci/bridge-admission-contract-final.log`. +- Inspection passes the complete original production balancing/browser case + in 329 seconds. Artifact: + `container-e2e/run-4380b4daaa054bd1a528ac73ffafbb01`; + log: `ci/console-inspection-final.log`. +- Native operations exposed a remote-leader routing defect. Stopping a local + follower left the remote leader healthy with a valid lease, but restart + returned 502 after requests to `127.0.0.1:10100`. HTTP topology ingestion now + resolves wildcard store listeners against the reporting management origin + before caching. Explicit peer endpoints remain intact. The regression fails + before the fix and passes afterward, covering IPv4/IPv6 and the cached + Group-0 leader. Full Console shared, Web and Monitor tests/clippy pass. + The exact lifecycle case passes in 8.8 seconds with original assertions: + `container-e2e/run-4315916e556445948d33173a097cfede`; + log: `ci/ci-diskdb-kv-origin-fixed.log`. +- Reference launch preflight exposed stale container protocol mappings. Fresh + Docker/Pixi/CLI/Compose launches now publish S3 16000, Iceberg 17000 and + Dataset 18000 while retaining host-side port inputs. Version-2 launch + receipts select canonical targets; version-1 receipts keep their immutable + ports/image through resume and retained-volume recreation. Compose provides + explicit legacy target overrides. Image exposed ports and examples agree. + All 136 OCI tests, 14 reference-fixture tests, six CLI lifecycle tests and + full CLI tests/clippy pass. The rebuilt image receipt is retained as + `ci/protocol-ports-image-receipt.json`. +- On the rebuilt image, complete native operations passes all 19 original + cases in 530 seconds, including typed restarts, retained bytes under load, + catalog navigation, ownership, multipart and explicit binding repair. + Artifact: `container-e2e/run-912e419cf5774ea197ec626fce31dbee`; + log: `ci/ci-console-native-operations-closing.log`. +- Complete transition passes production split, journal inspection and final + balanced Serving placement across three owners in 281 seconds. Artifact: + `container-e2e/run-a06b30aecc5e4ba8953d59df60b463a5`; + log: `ci/ci-console-transition-closing.log`. +- All six deployment-reference cases pass on the final image: Docker, Compose + and editable UI for one/three nodes, plus real Pixi shortcuts and confirmed + nonvoter cold recovery on new names/IPs. Original voter IDs, retained data, + credentials and identities are preserved. Artifact: + `container-e2e/run-554eaff75f0b4beb83d0dfdb59e3bb45`; + log: `ci/ci-deployment-reference-closing.log`. +- Final-image Console browser again passes all 27 original cases in 291 + seconds. Artifact: `container-e2e/run-5d6bcf681bda4e88998e34dd06bb9e47`; + log: `ci/ci-console-browser-closing.log`. +- Final-image native data passes all 14 original cases in one complete run, + including all five Dataset cases, six-node protocol round trips, topology, + retry, geometry, hardware and maintenance. Artifact: + `container-e2e/run-c831fca5df334574a14035c11c89ae54`; + log: `ci/ci-console-native-data-closing.log`. +- Final-image inspection passes the complete original case in 317 seconds, + including production balancing, real page layout and Iceberg inspection. + Artifact: `container-e2e/run-eac3d6766dab43809a0eee22414435e0`; + log: `ci/ci-console-inspection-closing.log`. +- All eight requested CI job selections now pass locally. Ordered closing + commands, image identity and complete results are in `ci/closing-sequence.json`; + per-job classification is in `ci/requested-job-results.json`. Remote CI for + the new commits is observed after publication. +- Quality: affected all-target clippy with warnings denied, workspace Rust + formatting, launcher shell syntax and C++ tree-lint pass. No C++ source + changed. Both bridge fixtures completed resource cleanup; their six owned + 16-GiB temporary payload files were retired after matching inode/size receipts. + Logs, metadata, disk headers and exact retirement records remain under + `ci/retired-bridge-disks-*`. Attempt-15 forensic archives remain intact. + +## Authorized publication and five-container follow-up + +- The first coherent commit/push includes the passed physical matrix, all eight + requested local CI selections and relevant quality gates. +- After that push, a fresh five-container cluster will span the same two + physical hosts (three containers on A, two on B). Preserve immutable image + identity, key-only admission, exact data witnesses, retained-volume recovery, + both host-loss cases and owned-resource cleanup. +- Verify repairs and commit/push the five-node changes and evidence summary + a second time. The execution plan remains active until then. diff --git a/e2e-test/clients/console-ui/e2e/fixtures/candidateSetup.ts b/e2e-test/clients/console-ui/e2e/fixtures/candidateSetup.ts index c93ec1036..40ed9dd19 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/candidateSetup.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/candidateSetup.ts @@ -18,8 +18,10 @@ export async function reviewDisklessCandidate(dialog: Locator, request?: APIRequ await expect(dialog.getByLabel('Managed file capacity (GiB)', { exact: true })).toHaveCount(0); for (const name of ['crowdb-paxos-kv', 'crowdb-disk-db', 'crowdb-disk-io', 'crowdb-chunk-db', 'crowdb-chunk-kv', 'crowdb-access-server']) { const service = dialog.getByRole('checkbox', { name, exact: true }); - await expect(service).not.toBeChecked(); + if (name === 'crowdb-paxos-kv') await expect(service).toBeChecked(); + else await expect(service).not.toBeChecked(); await expect(service).toBeDisabled(); } + await expect(dialog.getByText("This node's startup settings fix these choices.", { exact: true })).toBeVisible(); await dialog.getByRole('button', { name: 'Join and start', exact: true }).click(); } diff --git a/e2e-test/clients/console-ui/e2e/flows/12-bridge-discovery.spec.ts b/e2e-test/clients/console-ui/e2e/flows/12-bridge-discovery.spec.ts index a7b3d91d4..8088015b6 100644 --- a/e2e-test/clients/console-ui/e2e/flows/12-bridge-discovery.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/12-bridge-discovery.spec.ts @@ -26,7 +26,7 @@ test('Docker default setup: submit storage and services then close the initiatin for (const checkbox of await dialog.getByRole('region', { name: 'Node service choices' }).getByRole('checkbox').all()) await expect(checkbox).toBeChecked(); await expect(dialog.getByLabel('Managed file capacity (GiB)', { exact: true })).toHaveValue('16'); await expect(dialog.getByLabel('Managed file capacity (GiB)', { exact: true })).toBeDisabled(); - await expect(dialog.getByLabel('crowdb-disk-db RPC port', { exact: true })).toHaveValue('7100'); + await expect(dialog.getByLabel('crowdb-disk-db RPC port', { exact: true })).toHaveValue('11200'); await expect(dialog.getByLabel('crowdb-disk-db RPC port', { exact: true })).toBeDisabled(); await dialog.getByRole('button', { name: 'Join and start', exact: true }).click(); await expect(dialog).toHaveCount(0, { timeout: 10_000 }); @@ -58,8 +58,8 @@ test('Docker default setup: progress is visible in the node properties', async ( const groups = (await usage.json()).disk_groups; expect(groups.map((group: { disk_group_id: number }) => group.disk_group_id)).toContain(1); const disk = groups.find((group: { disk_group_id: number }) => group.disk_group_id === 1).disks[0]; - expect(disk.zone_count).toBe(64); - expect(disk.zone_size_units * disk.unit_size_bytes).toBe(256 * 1024 * 1024); + expect(disk.zone_count).toBe(1); + expect(disk.zone_size_units * disk.unit_size_bytes).toBe(16 * 1024 ** 3); const bitmap = await page.request.get(`${process.env.CROWDB_NODE_UI_ORIGIN}/api/diskdb/usage?dg=1&disk=${disk.disk_id}&zone=0`); expect(bitmap.ok(), await bitmap.text()).toBe(true); expect((await bitmap.json()).disk_groups[0].disks[0].zone_usages[0].usage_bitmap).toMatch(/^[0-9a-f]+$/); diff --git a/e2e-test/clients/s3-go/main.go b/e2e-test/clients/s3-go/main.go index 000e0e4eb..eb234210b 100644 --- a/e2e-test/clients/s3-go/main.go +++ b/e2e-test/clients/s3-go/main.go @@ -129,6 +129,24 @@ func capture(action func()) (failure any) { return nil } +func failureCode(failure any) string { + code := fmt.Sprintf("%T", failure) + if assertion, ok := failure.(assertionFailure); ok { + return "assertion: " + string(assertion) + } + if err, ok := failure.(error); ok { + var api smithy.APIError + if errors.As(err, &api) { + code = api.ErrorCode() + } + var operation *smithy.OperationError + if errors.As(err, &operation) { + code = operation.Operation() + ": " + code + } + } + return code +} + func run() int { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() @@ -149,20 +167,10 @@ func run() int { exit = 42 } // SDK errors can contain request URLs; report only the code/type and stage. - code := fmt.Sprintf("%T", failure) - if assertion, ok := failure.(assertionFailure); ok { - code = "assertion: " + string(assertion) - } - if err, ok := failure.(error); ok { - var api smithy.APIError - if errors.As(err, &api) { - code = api.ErrorCode() - } - } - fmt.Fprintf(os.Stderr, "go: FAILED step=%s code=%s\n", s.current, code) + fmt.Fprintf(os.Stderr, "go: FAILED step=%s code=%s\n", s.current, failureCode(failure)) } - if capture(s.cleanup) != nil { - fmt.Fprintln(os.Stderr, "go: cleanup FAILED") + if failure := capture(s.cleanup); failure != nil { + fmt.Fprintf(os.Stderr, "go: cleanup FAILED code=%s\n", failureCode(failure)) exit = 1 } if exit == 0 { diff --git a/e2e-test/console_native_policy.py b/e2e-test/console_native_policy.py index d0ed9e1ac..d9f2320d2 100644 --- a/e2e-test/console_native_policy.py +++ b/e2e-test/console_native_policy.py @@ -3,13 +3,12 @@ """Accept the owned diskless policy before attaching unadmitted fixture media.""" from allocation import LABEL +from profiles.setup_choices import diskless_policy def accept(cluster, index, data, physical_host): policy = cluster.artifacts / f"node-policy-{index}.toml" - kinds = ("paxos-kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access-server") - services = ", ".join(f'{{ kind = "{kind}", enabled = false }}' for kind in kinds) - policy.write_text(f'services = [{services}]\n[storage]\nmode = "diskless"\n') + policy.write_text(diskless_policy()) environment = ("CROWDB_DEPLOYMENT_MODE=production", "CROWDB_INITIAL_VOTERS=3", f"CROWDB_CLUSTER={cluster.run_id}", "CROWDB_SSH_PASSWORD=crowdb", f"CROWDB_PHYSICAL_HOST_ID={physical_host}", diff --git a/e2e-test/oci_reference_recovery.py b/e2e-test/oci_reference_recovery.py index 604ad5a8d..715685956 100644 --- a/e2e-test/oci_reference_recovery.py +++ b/e2e-test/oci_reference_recovery.py @@ -12,6 +12,7 @@ from fixture import HTTP, LABEL, request, wait from profiles.node import request as admission_request +from profiles.setup_choices import diskless_policy import oci_reference_runtime as runtime import oci_reference_protocols as protocols @@ -24,11 +25,8 @@ def create_nonvoter(cluster): name = cluster.expected_names[3] volume = runtime.node_volume(cluster, 3) policy = cluster.artifacts / 'later-member.toml' - lines = ['version = 1', 'mode = "local"', f'cluster = "{cluster.run_id}"', - 'initial_voters = 3', '[storage]', 'mode = "diskless"'] - for kind in ('paxos-kv', 'diskdb', 'diskio', 'chunkdb', 'chunk-kv', 'access-server'): - lines += ['[[services]]', f'kind = "{kind}"', 'enabled = false'] - content = '\n'.join(lines) + '\n' + content = ('version = 1\nmode = "local"\n' + f'cluster = "{cluster.run_id}"\ninitial_voters = 3\n' + diskless_policy()) if policy.exists(): assert policy.read_text() == content else: diff --git a/e2e-test/oci_reference_runtime.py b/e2e-test/oci_reference_runtime.py index 1c024232b..5030c7bc4 100644 --- a/e2e-test/oci_reference_runtime.py +++ b/e2e-test/oci_reference_runtime.py @@ -90,7 +90,7 @@ def create_docker_nodes(cluster, indexes): "--env", f"CROWDB_CLUSTER={cluster.run_id}", "--env", f"CROWDB_INITIAL_VOTERS={cluster.node_count}"] for protocol, port in ports(cluster.port_base + index * 10).items(): - internal = 9090 + (port - cluster.port_base - index * 10) + internal = {"UI": 9090, "S3": 16000, "ICEBERG": 17000, "DATASET": 18000}[protocol] arguments += ["--publish", f"127.0.0.1:{port}:{internal}"] if protocol != "UI": arguments += ["--env", f"CROWDB_{protocol}_PUBLIC_URI=http://127.0.0.1:{port}"] diff --git a/e2e-test/profiles/bridge_node.py b/e2e-test/profiles/bridge_node.py index 7d1a6f317..e767b250e 100644 --- a/e2e-test/profiles/bridge_node.py +++ b/e2e-test/profiles/bridge_node.py @@ -11,6 +11,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from admission_profile import cleanup as cleanup_runtime +from profiles.setup_choices import diskless_policy IMAGE = os.environ["CROWDB_CONTAINER_IMAGE"] PREFIX = os.environ["CROWDB_E2E_PROFILE_NAME"] @@ -32,12 +33,10 @@ def start_node(name, *, default_setup=False): data = DATA / name.removeprefix(PREFIX + "-") data.mkdir(mode=0o755) policy = DATA / (name.removeprefix(PREFIX + "-") + ("-default-policy.toml" if default_setup else "-policy.toml")) - kinds = ("paxos-kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access-server") - services = ", ".join(f'{{ kind = "{kind}", enabled = false }}' for kind in kinds) if default_setup: policy.write_text('[storage]\nmode = "automatic"\ncapacity_bytes = 17179869184\nreserve_bytes = 1073741824\nsparse = false\n') else: - policy.write_text(f'services = [{services}]\n[storage]\nmode = "diskless"\n') + policy.write_text(diskless_policy()) docker("run", "--rm", "--name", PREFIX + "-prepare", "--network", "none", "--label", f"org.crowdb.e2e.run={PREFIX}", "--user", "0", "--cpu-shares", "1024", "--env", "TOKIO_WORKER_THREADS=1", "--memory", "32m", "--entrypoint", "/bin/chown", diff --git a/e2e-test/profiles/setup_choices.py b/e2e-test/profiles/setup_choices.py index 96eac2aa5..eaa7f504f 100644 --- a/e2e-test/profiles/setup_choices.py +++ b/e2e-test/profiles/setup_choices.py @@ -4,6 +4,21 @@ def diskless_setup(): - return {"storage": {"mode": "diskless"}, "services": [ + services = [ + {"kind": "paxos-kv", "enabled": True, + "listeners": {"http_port": 10000, "rpc_port": 10100}}] + services += [ {"kind": kind, "enabled": False, "listeners": {}} - for kind in ("paxos-kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access-server")]} + for kind in ("diskdb", "diskio", "chunkdb", "chunk-kv", "access-server")] + return {"storage": {"mode": "diskless"}, "services": services} + + +def diskless_policy(): + lines = ['[storage]', 'mode = "diskless"'] + for service in diskless_setup()["services"]: + lines += ['[[services]]', f'kind = "{service["kind"]}"', + f'enabled = {str(service["enabled"]).lower()}'] + if service["listeners"]: + lines += ['[services.listeners]', + *(f'{key} = {value}' for key, value in service["listeners"].items())] + return '\n'.join(lines) + '\n' diff --git a/e2e-test/tests/admission_profile_test.py b/e2e-test/tests/admission_profile_test.py index 7967587f9..8eab5714f 100644 --- a/e2e-test/tests/admission_profile_test.py +++ b/e2e-test/tests/admission_profile_test.py @@ -9,6 +9,7 @@ from pathlib import Path import sys import tempfile +import tomllib import unittest from unittest.mock import patch import run @@ -92,3 +93,8 @@ def reply(*arguments): self.assertNotEqual(policies[0], policies[1]) self.assertTrue(all(policy.parent == Path(directory) for policy in policies)) self.assertEqual(policies[0].read_text(), policies[1].read_text()) + accepted = tomllib.loads(policies[0].read_text()) + enabled = [service for service in accepted["services"] if service["enabled"]] + self.assertEqual(accepted["storage"], {"mode": "diskless"}) + self.assertEqual(enabled, [{"kind": "paxos-kv", "enabled": True, + "listeners": {"http_port": 10000, "rpc_port": 10100}}]) diff --git a/e2e-test/tests/oci_reference_test.py b/e2e-test/tests/oci_reference_test.py index c567a314b..270fda706 100644 --- a/e2e-test/tests/oci_reference_test.py +++ b/e2e-test/tests/oci_reference_test.py @@ -126,6 +126,9 @@ def test_recreated_names_reuse_exact_retained_volumes_on_the_accepted_new_networ self.assertIn('type=volume,source=owned-node1-data,target=/opt/crowdb/data', create) self.assertEqual(create[create.index('--network') + 1], 'owned-recovered') self.assertNotIn('owned-node1-cold-data', create) + published = [create[index+1] for index, arg in enumerate(create) if arg == '--publish'] + self.assertEqual(published, ['127.0.0.1:20000:9090', '127.0.0.1:20001:16000', + '127.0.0.1:20002:17000', '127.0.0.1:20003:18000']) def test_cold_network_scope_still_rejects_foreign_names_and_attachments(self): cluster = self.cluster() diff --git a/lib/crowdb-chunk-stream/src/production_chunk/recovery.rs b/lib/crowdb-chunk-stream/src/production_chunk/recovery.rs index 49d60f15d..8d9b2b2e1 100644 --- a/lib/crowdb-chunk-stream/src/production_chunk/recovery.rs +++ b/lib/crowdb-chunk-stream/src/production_chunk/recovery.rs @@ -7,10 +7,12 @@ use super::{io_error, update_state, Ordering, ProductionStreamChunkStore}; use crate::{ActiveChunkDescriptor, RecoveredStreamTail, Result, StreamError, StreamName}; use async_trait::async_trait; use crowdb_chunk_client::{replay_frame_prefix, select_frame_prefix, FrameExtentSink, MirroredFrameReaders}; -use crowdb_protocol::chunkdb::rpc::{ChunkState, QueryChunkRequest}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, QueryChunkRequest}; use crowdb_protocol::frame::{ScannedFrameExtent, MAX_FRAME_SCAN_INPUT_BYTES}; use std::{sync::Arc, time::Duration}; +mod drain; + struct RecoveredExtents(Vec); #[async_trait] @@ -46,20 +48,17 @@ impl ProductionStreamChunkStore { return Err(StreamError::StaleWriter); } let state = self.install(chunk.clone())?; + if chunk.strips.is_empty() { + let recovered = empty_sealed_tail(&chunk, active)?; + self.verify_recovery_snapshot(&chunk).await?; + context.check_io()?; + update_state(&state, chunk); + return Ok(recovered); + } let deadline = tokio::time::Instant::from_std(context.deadline()) .min(tokio::time::Instant::now() + Duration::from_secs(60)); - for strip in &chunk.strips { - let Some(crowdb_protocol::chunkdb::rpc::Strip::MirrorStrip(mirror)) = &strip.strip else { - return Err(StreamError::Corruption("recovery chunk is not mirrored".into())); - }; - for segment in &mirror.segments { - context.check_io()?; - self.disk_writer - .drain_segment(segment, u64::from(strip.unit_kb) * 1024, deadline.into_std()) - .await - .map_err(io_error)?; - } - } + self.drain_recovery_layout(&chunk, &context, deadline.into_std()) + .await?; let readers = MirroredFrameReaders::new(Arc::new(chunk.clone()), name, self.disk_writer.clone(), deadline) .map_err(|error| observation_error(&error))?; @@ -91,17 +90,7 @@ impl ProductionStreamChunkStore { ) .await .map_err(|error| observation_error(&error))?; - let refreshed = self - .allocator - .query_chunk(QueryChunkRequest { - chunk_id: Some(active.chunk_id), - }) - .await - .map_err(io_error)? - .chunk; - if refreshed.as_ref() != Some(&chunk) { - return Err(StreamError::StaleWriter); - } + self.verify_recovery_snapshot(&chunk).await?; context.check_io()?; update_state(&state, chunk); state @@ -113,6 +102,43 @@ impl ProductionStreamChunkStore { extents: extents.0, }) } + async fn verify_recovery_snapshot(&self, chunk: &Chunk) -> Result<()> { + let refreshed = self + .allocator + .query_chunk(QueryChunkRequest { chunk_id: chunk.id }) + .await + .map_err(io_error)? + .chunk; + if refreshed.as_ref() != Some(chunk) { + return Err(StreamError::StaleWriter); + } + Ok(()) + } +} + +fn empty_sealed_tail(chunk: &Chunk, active: &ActiveChunkDescriptor) -> Result { + use crowdb_protocol::chunk_stream::StreamPurpose; + let valid_purpose = [StreamPurpose::Wal, StreamPurpose::Stream] + .into_iter() + .any(|purpose| purpose.matches(active.chunk_id) && chunk.chunk_type == purpose.chunk_type() as i32); + // Sealing an empty tail returns every strip. Its durable zero boundary has + // no physical bytes to scan, but an active or nonempty layout is corrupt. + if !valid_purpose + || chunk.writer_epoch == 0 + || chunk.state != ChunkState::Sealed as i32 + || chunk.capacity != 0 + || chunk.sealed_length != 0 + || chunk.acknowledged_cursor != 0 + || active.acknowledged_cursor != 0 + || active.physical_start != 0 + { + return Err(StreamError::Corruption("invalid empty sealed tail".into())); + } + Ok(RecoveredStreamTail { + physical_end: 0, + logical_end: active.logical_start, + extents: Vec::new(), + }) } fn observation_error(error: &crowdb_chunk_client::FrameObservationError) -> StreamError { diff --git a/lib/crowdb-chunk-stream/src/production_chunk/recovery/drain.rs b/lib/crowdb-chunk-stream/src/production_chunk/recovery/drain.rs new file mode 100644 index 000000000..e7901ccee --- /dev/null +++ b/lib/crowdb-chunk-stream/src/production_chunk/recovery/drain.rs @@ -0,0 +1,102 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Failed recovery drains feed the existing durable mirror repair queue. + +use super::{io_error, ProductionStreamChunkStore}; +use crate::{Result, StreamError, StreamWriteContext}; +use crowdb_protocol::{ + chunkdb::rpc::{Chunk, ChunkStrip, ReplaceChunkStripRangeRequest, Strip}, + common::ChunkId, + diskdb::rpc::Segment, +}; +use futures::{stream, StreamExt}; + +impl ProductionStreamChunkStore { + pub(super) async fn drain_recovery_layout( + &self, + chunk: &Chunk, + context: &StreamWriteContext, + deadline: std::time::Instant, + ) -> Result<()> { + for (index, strip) in chunk.strips.iter().enumerate() { + let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { + return Err(StreamError::Corruption("recovery chunk is not mirrored".into())); + }; + if !strip.unavailable_segments.is_empty() { + return Err(StreamError::ReadUnavailable( + "recovery mirror repair is pending".into(), + )); + } + let results = stream::iter(mirror.segments.iter().copied()) + .map(|segment| async move { + let result = async { + context.check_io()?; + self.disk_writer + .drain_segment(&segment, u64::from(strip.unit_kb) * 1024, deadline) + .await + .map_err(io_error) + } + .await; + (segment, result) + }) + .buffer_unordered(8) + .collect::>() + .await; + let failed: Vec<_> = mirror + .segments + .iter() + .copied() + .filter(|segment| { + results + .iter() + .any(|(observed, result)| observed == segment && result.is_err()) + }) + .collect(); + let Some(error) = results.into_iter().find_map(|(_, result)| result.err()) else { + continue; + }; + context.check_io()?; + // Keep the failed operation unavailable. A drained surviving mirror + // gives the durable repair executor a stable source; its published + // replacement must drain successfully on a later recovery attempt. + if failed.len() < mirror.segments.len() { + tracing::warn!(chunk_id = ?chunk.id, strip_sequence = strip.strip_sequence, + %error, "recovery drain failed; requesting mirror repair"); + self.mark_recovery_mirrors(chunk, index, strip, failed).await?; + } + return Err(error); + } + Ok(()) + } + + async fn mark_recovery_mirrors( + &self, + chunk: &Chunk, + index: usize, + strip: &ChunkStrip, + failed: Vec, + ) -> Result<()> { + let id = chunk + .id + .ok_or_else(|| StreamError::Corruption("recovery chunk has no ID".into()))?; + let mut replacement = strip.clone(); + replacement.unavailable_segments = failed; + self.allocator + .replace_chunk_strip_range(ReplaceChunkStripRangeRequest { + chunk_id: Some(id), + expected_modify_ts: chunk.modify_ts, + start_index: u32::try_from(index) + .map_err(|_| StreamError::Corruption("strip index overflow".into()))?, + old_strips: vec![strip.clone()], + replacement_strips: vec![replacement], + operation_id: Some(ChunkId { + high: id.high ^ chunk.modify_ts ^ 0xd7a1_0000_0000_0001, + low: id.low ^ u64::from(strip.strip_sequence), + }), + }) + .await + .map_err(io_error)?; + Ok(()) + } +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test.rs index 89ebc171a..cc25d1e0c 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test.rs @@ -28,6 +28,9 @@ use crowdb_protocol::diskdb::rpc::Segment; #[path = "production_chunk_test/recovery.rs"] mod recovery; +#[path = "production_chunk_test/recovery_drain.rs"] +mod recovery_drain; + #[path = "production_chunk_test/local_progress.rs"] mod local_progress; @@ -441,6 +444,7 @@ struct Disks { initialization_started: tokio::sync::Notify, initialization_resume: tokio::sync::Notify, drain_delay_ms: AtomicU64, + fail_all_drains: AtomicBool, active_drains: AtomicUsize, max_active_drains: AtomicUsize, completed_drains: AtomicUsize, @@ -482,7 +486,11 @@ impl DiskWriter for Disks { if delay != 0 { tokio::time::sleep(std::time::Duration::from_millis(delay)).await; } - let result = self.fsync(segment).await; + let result = if self.fail_all_drains.load(Ordering::Acquire) { + Err(IoError::WriteFailed("all mirror drains unavailable".into())) + } else { + self.fsync(segment).await + }; self.active_drains.fetch_sub(1, Ordering::AcqRel); self.completed_drains.fetch_add(1, Ordering::AcqRel); result diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery.rs index d3bb7085e..5ad1e196d 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery.rs @@ -86,3 +86,69 @@ async fn cold_adapter_uses_the_shortest_healthy_complete_prefix() { async fn cold_adapter_does_not_turn_a_corrupt_replica_into_an_empty_tail() { recover_case(false, true).await; } + +#[tokio::test] +async fn cold_adapter_recovers_an_empty_sealed_tail_after_its_strips_are_reclaimed() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let make = || { + ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap() + }; + let writer = make(); + let name = StreamName { high: 803, low: 93 }; + let mut active = writer + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Wal) + .await + .unwrap(); + active.logical_start = 42; + // DiskDB returns all strips when ChunkDB seals a physically empty chunk. + let sealed = { + let mut chunk = allocator.chunk.lock().unwrap(); + let chunk = chunk.as_mut().unwrap(); + chunk.state = ChunkState::Sealed as i32; + chunk.modify_ts += 1; + chunk.capacity = 0; + chunk.strips.clear(); + chunk.clone() + }; + drop(writer); + let cold = make(); + let memory = MemoryStreamStore::new(1024); + cold.bind_writer(name, 10, memory.claim_writer(name, 10).await.unwrap()); + let recovered = cold.recover_tail(name, 10, &active).await.unwrap(); + assert_eq!((recovered.physical_end, recovered.logical_end), (0, 42)); + assert!(recovered.extents.is_empty()); + for case in 0..8 { + let mut invalid = sealed.clone(); + let mut descriptor = active.clone(); + match case { + 0 => invalid.state = ChunkState::Active as i32, + 1 => invalid.capacity = 1, + 2 => invalid.sealed_length = 1, + 3 => invalid.acknowledged_cursor = 1, + 4 => descriptor.acknowledged_cursor = 1, + 5 => descriptor.physical_start = 1, + 6 => invalid.writer_epoch = 0, + _ => invalid.chunk_type = crowdb_protocol::chunkdb::rpc::ChunkType::S3 as i32, + } + *allocator.chunk.lock().unwrap() = Some(invalid); + assert!(matches!( + cold.recover_tail(name, 10, &descriptor).await, + Err(crowdb_chunk_stream::StreamError::Corruption(_)) + )); + } + let mut changed = sealed.clone(); + changed.modify_ts += 1; + *allocator.chunk.lock().unwrap() = Some(changed); + *allocator.stale_query_once.lock().unwrap() = Some(sealed); + assert!(matches!( + cold.recover_tail(name, 10, &active).await, + Err(crowdb_chunk_stream::StreamError::StaleWriter) + )); +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery_drain.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery_drain.rs new file mode 100644 index 000000000..8110b00fd --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery_drain.rs @@ -0,0 +1,100 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test] +async fn a_failed_recovery_drain_requests_layout_repair_before_tail_observation() { + for lost_reply in [false, true] { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let make = || { + ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap() + }; + let writer = make(); + let name = StreamName { high: 811, low: 97 }; + let active = writer + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Wal) + .await + .unwrap(); + let mut encoded = TestGroupedStrip::default(); + let end = encoded + .append(&writer, name, 9, active.chunk_id, b"retained") + .await; + drop(writer); + let cold = make(); + let memory = MemoryStreamStore::new(1024); + cold.bind_writer(name, 10, memory.claim_writer(name, 10).await.unwrap()); + disks.fail_fsync_once.store(2, Ordering::Release); + allocator + .fail_replacement_response_once + .store(lost_reply, Ordering::Release); + assert!(cold.recover_tail(name, 10, &active).await.is_err()); + assert_eq!(disks.completed_drains.load(Ordering::Acquire), 2); + assert_eq!(allocator.replacement_calls.load(Ordering::Acquire), 1); + let pending = allocator.chunk.lock().unwrap().clone().unwrap(); + assert_eq!(pending.state, ChunkState::Active as i32); + let Some(Strip::MirrorStrip(mirror)) = &pending.strips[0].strip else { + panic!("mirror") + }; + assert_eq!(pending.strips[0].unavailable_segments, vec![mirror.segments[1]]); + // Even if a transport recovers, the published unavailable layout still + // needs its durable repair before it can authorize recovery. + assert!(cold.recover_tail(name, 10, &active).await.is_err()); + assert_eq!(allocator.replacement_calls.load(Ordering::Acquire), 1); + // Model the existing repair executor copying the drained surviving mirror + // into a replacement and publishing its new allocation identity. + { + let mut bytes = disks.bytes.lock().unwrap(); + let retained = bytes.get(&1).unwrap().clone(); + bytes.insert(101, retained); + } + { + let mut chunk = allocator.chunk.lock().unwrap(); + let chunk = chunk.as_mut().unwrap(); + let strip = &mut chunk.strips[0]; + let Some(Strip::MirrorStrip(mirror)) = &mut strip.strip else { + panic!("mirror") + }; + mirror.segments[1].disk_id.as_mut().unwrap().high = 101; + mirror.segments[1].allocation_ts += 1; + strip.unavailable_segments.clear(); + chunk.modify_ts += 1; + } + let recovered = cold.recover_tail(name, 10, &active).await.unwrap(); + assert_eq!((recovered.physical_end, recovered.logical_end), (end, 8)); + assert_eq!(recovered.extents.len(), 1); + } +} + +#[tokio::test] +async fn failed_drains_on_every_mirror_cannot_admit_repair_or_a_tail() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(); + let memory = MemoryStreamStore::new(1024); + let name = StreamName::generate(); + store.bind_writer(name, 10, memory.claim_writer(name, 10).await.unwrap()); + let active = store + .allocate_mirrored(name, 10, crowdb_chunk_stream::StreamPurpose::Wal) + .await + .unwrap(); + let before = allocator.chunk.lock().unwrap().clone(); + disks.fail_all_drains.store(true, Ordering::Release); + assert!(store.recover_tail(name, 10, &active).await.is_err()); + assert_eq!(disks.completed_drains.load(Ordering::Acquire), 2); + assert_eq!(allocator.replacement_calls.load(Ordering::Acquire), 0); + assert_eq!(*allocator.chunk.lock().unwrap(), before); +} diff --git a/lib/crowdb-console-shared/src/clients/http.rs b/lib/crowdb-console-shared/src/clients/http.rs index 717b76810..afdf1cc2c 100644 --- a/lib/crowdb-console-shared/src/clients/http.rs +++ b/lib/crowdb-console-shared/src/clients/http.rs @@ -87,12 +87,19 @@ impl ServerClient { self.get_json("/health").await } - /// `GET /topology`. + /// `GET /topology`. Resolve wildcard store listeners against the + /// reporting management origin before callers cache them. Explicit peer + /// endpoints retain their advertised addresses. /// /// # Errors /// Returns `Error::UpstreamRpc` for transport / decode failures. pub async fn topology(&self) -> Result> { - let resp: TopologyResponse = self.get_json("/topology").await?; + let mut resp: TopologyResponse = self.get_json("/topology").await?; + for store in &mut resp.stores { + if let Some(endpoint) = &store.listen_addr { + store.listen_addr = Some(self.resolve_rpc_endpoint(endpoint)?); + } + } Ok(resp.stores) } diff --git a/lib/crowdb-console-shared/tests/topology_endpoint_test.rs b/lib/crowdb-console-shared/tests/topology_endpoint_test.rs new file mode 100644 index 000000000..51592c1de --- /dev/null +++ b/lib/crowdb-console-shared/tests/topology_endpoint_test.rs @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{routing::get, Json, Router}; +use crowdb_console_shared::{ + clients::http::ServerClient, + cluster::NodeHealth, + monitor::{legacy_topology_to_node_stores, MonitorCache, NodeRecord}, +}; +use crowdb_protocol::mgmt::{GroupStatus, RemoteStatus, ReplicaStatus, StoreStatus, TopologyResponse}; + +#[tokio::test] +async fn wildcard_topology_listeners_use_the_reporting_origin_before_seeding_a_leader() { + let response = TopologyResponse { + stores: vec![ + StoreStatus { + store_id: 0, + listen_addr: Some("0.0.0.0:10100".into()), + groups: vec![GroupStatus { + group_id: 0, + leader_id: 1, + local_replica: ReplicaStatus { + id: 1, + role: "Leader".into(), + ..Default::default() + }, + remotes: vec![RemoteStatus { + id: 2, + endpoint: "192.0.2.20:10101".into(), + ..Default::default() + }], + ..Default::default() + }], + ..Default::default() + }, + StoreStatus { + store_id: 1, + listen_addr: Some("[::]:11234".into()), + ..Default::default() + }, + StoreStatus { + store_id: 2, + listen_addr: Some("192.0.2.30:12345".into()), + ..Default::default() + }, + ], + }; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://localhost:{}", listener.local_addr().unwrap().port()); + let app = Router::new().route( + "/topology", + get(move || { + let response = response.clone(); + async move { Json(response) } + }), + ); + let task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let stores = ServerClient::new(origin).unwrap().topology().await.unwrap(); + task.abort(); + assert_eq!(stores[0].listen_addr.as_deref(), Some("localhost:10100")); + assert_eq!(stores[1].listen_addr.as_deref(), Some("localhost:11234")); + assert_eq!(stores[2].listen_addr.as_deref(), Some("192.0.2.30:12345")); + assert_eq!(stores[0].groups[0].remotes[0].endpoint, "192.0.2.20:10101"); + let cache = MonitorCache::new(); + cache + .set_node_report( + 1, + NodeRecord { + health: NodeHealth::Up, + stores: legacy_topology_to_node_stores(1, &stores), + ..Default::default() + }, + ) + .await; + assert_eq!( + cache.group0_leader_endpoint().await.as_deref(), + Some("localhost:10100") + ); +} diff --git a/lib/crowdb-kv-client/src/client/retry.rs b/lib/crowdb-kv-client/src/client/retry.rs index 55003c87f..79d8ff8bb 100644 --- a/lib/crowdb-kv-client/src/client/retry.rs +++ b/lib/crowdb-kv-client/src/client/retry.rs @@ -60,9 +60,12 @@ impl CrowdbKvClient { /// handled by [`Self::follow_not_leader`] before this is checked). /// Checks the structured `error_code` first, falling back to the /// string for old servers that don't set the code (default 0 = - /// `KvErrorNone`). + /// `KvErrorNone`). A removed replica may still have a cached route but no + /// local group; discover its successor instead of repeating that route. pub(crate) fn is_unknown_leader(error_code: i32, error: &str) -> bool { - error_code == KvErrorCode::KvErrorNotLeader as i32 || error == "not leader" + error_code == KvErrorCode::KvErrorNotLeader as i32 + || error == "not leader" + || error == "no kv group configured for request group_id" } /// After an [`Self::is_unknown_leader`] failure, give the election a diff --git a/lib/crowdb-kv-client/tests/conditional_retry_test.rs b/lib/crowdb-kv-client/tests/conditional_retry_test.rs index 6ac2a0155..cdc46190e 100644 --- a/lib/crowdb-kv-client/tests/conditional_retry_test.rs +++ b/lib/crowdb-kv-client/tests/conditional_retry_test.rs @@ -8,6 +8,48 @@ use conditional_servers::TestServers; use crowdb_kv::cluster::kv_server::KvServer; use crowdb_kv_client::{BatchOp, CrowdbKvClient, Error, KvRpcTransport}; +#[tokio::test] +async fn a_removed_replica_refreshes_its_stale_route_before_reading_or_writing() { + let servers = TestServers::start(true).await; + assert!(servers.follower.remove_group(1)); + servers + .client() + .put(1, 1, b"value", b"retained", None) + .await + .unwrap(); + let value = servers + .client() + .get(1, 1, b"value", crowdb_kv_client::ReadMode::Linearizable, None) + .await + .unwrap(); + assert!( + matches!(value, crowdb_kv_client::GetOutcome::Found { value, .. } if value.as_ref() == b"retained") + ); + servers + .client() + .put_cas(1, 1, b"conditional", b"value", 0) + .await + .unwrap(); + let scan = servers + .client() + .scan( + 1, + 1, + b"value", + b"", + b"", + 10, + crowdb_kv_client::ReadMode::Linearizable, + None, + false, + None, + ) + .await + .unwrap(); + assert_eq!(scan.items.len(), 1); + servers.client().delete(1, 1, b"value", None).await.unwrap(); +} + #[tokio::test] async fn conditional_writes_discover_leader_after_explicit_rejection_without_a_hint() { let servers = TestServers::start(true).await; diff --git a/lib/crowdb-kv/src/cluster/px_kv_store.rs b/lib/crowdb-kv/src/cluster/px_kv_store.rs index b4520effa..fc5fb779d 100644 --- a/lib/crowdb-kv/src/cluster/px_kv_store.rs +++ b/lib/crowdb-kv/src/cluster/px_kv_store.rs @@ -660,11 +660,15 @@ impl PxKvStore { } pub(crate) fn missing_group_response(request_id: u64, request_create_ms: u64) -> crate::rpc::KvResponse { - crate::rpc::KvResponse::err( + let mut response = crate::rpc::KvResponse::err( "no kv group configured for request group_id".to_string(), request_id, request_create_ms, - ) + ); + // A cached route may name a replica removed from this process. The caller + // must discover the current leader rather than repeat the obsolete route. + response.error_code = crate::rpc::KvErrorCode::KvErrorNotLeader as i32; + response } /// Build a failed [`crate::rpc::KvScanResponse`] carrying `error` and an diff --git a/lib/crowdb-kv/src/cluster/px_kv_store_handler.rs b/lib/crowdb-kv/src/cluster/px_kv_store_handler.rs index 208ee5072..acb5d93c5 100644 --- a/lib/crowdb-kv/src/cluster/px_kv_store_handler.rs +++ b/lib/crowdb-kv/src/cluster/px_kv_store_handler.rs @@ -237,12 +237,14 @@ impl KvStore for PxKvStore { request_create_ms: u64, ) -> crate::rpc::KvScanResponse { let Some(operations) = self.group_operations(group_id) else { - return scan_err( + let mut response = scan_err( format!("group {group_id} not found in store {}", self.store_id), String::new(), request_id, request_create_ms, ); + response.error_code = crate::rpc::KvErrorCode::KvErrorNotLeader as i32; + return response; }; let request = KvGroupScanRequest { prefix: Bytes::copy_from_slice(prefix), @@ -516,13 +518,15 @@ impl KvStore for PxKvStore { request_create_ms: u64, ) -> crate::rpc::KvJournalScanResponse { let Some(group) = self.get_group(group_id) else { - return journal_scan_err( + let mut response = journal_scan_err( format!("group {group_id} not found in store {}", self.store_id), String::new(), false, request_id, request_create_ms, ); + response.error_code = crate::rpc::KvErrorCode::KvErrorNotLeader as i32; + return response; }; // Read-mode routing mirrors `kv_scan`: linearizable runs the @@ -568,7 +572,8 @@ impl KvStore for PxKvStore { } else { max_slot.min(read_slot) }; - if effective_max < min_slot { + let first_slot = min_slot.max(1); + if effective_max < first_slot { // Empty range — return an empty success. return crate::rpc::KvJournalScanResponse { version: 1, @@ -592,7 +597,22 @@ impl KvStore for PxKvStore { // `end_slot_exclusive` = effective_max + 1 (effective_max is // inclusive per the proto contract). let end_exclusive = effective_max.saturating_add(1); - let entries = acceptor.accepted_iter_range(min_slot, end_exclusive); + let entries = acceptor.accepted_iter_range(first_slot, end_exclusive); + // Restart can restore the applied prefix from the durable engine while + // its earlier WAL segments are already gone. The live trim watermark + // has not necessarily been restored or advanced by maintenance yet. + // Also cover reclamation racing the trim check above: copies of every + // applied slot, including NoOps, must exist before filtering by key. + let expected_entries = effective_max - first_slot + 1; + if u64::try_from(entries.len()).unwrap_or(u64::MAX) != expected_entries { + return journal_scan_err( + format!("journal_scan history is incomplete in slots {first_slot}..={effective_max}"), + String::new(), + true, + request_id, + request_create_ms, + ); + } // Decode each slot's payload batch and emit matching ops in // slot order (within a slot, ops are in batch order). diff --git a/lib/crowdb-kv/tests/store_test.rs b/lib/crowdb-kv/tests/store_test.rs index b6ef1439e..784391953 100644 --- a/lib/crowdb-kv/tests/store_test.rs +++ b/lib/crowdb-kv/tests/store_test.rs @@ -61,3 +61,6 @@ mod bounded_scan; #[path = "store_test/group_operations_test.rs"] mod group_operations; + +#[path = "store_test/journal_recovery_test.rs"] +mod journal_recovery; diff --git a/lib/crowdb-kv/tests/store_test/journal_recovery_test.rs b/lib/crowdb-kv/tests/store_test/journal_recovery_test.rs new file mode 100644 index 000000000..61cf2b043 --- /dev/null +++ b/lib/crowdb-kv/tests/store_test/journal_recovery_test.rs @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_kv::cluster::group::PxGroup; +use crowdb_kv::cluster::kv_store::KvStore; +use crowdb_kv::cluster::{PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_kv::kv::{Batch, BatchOp, CrowdbTreeConfig, CrowdbTreeEngine, KVEngine, Op}; +use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_kv::rpc::{KvErrorCode, ReadMode}; +use crowdb_kv::wal::index::SegmentIndex; +use crowdb_kv::wal::record::WALRecord; +use crowdb_kv::wal::replay::ReplayResult; + +#[tokio::test] +async fn restored_journal_rejects_missing_durable_history_before_maintenance() { + let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig::default()).unwrap(); + for slot in 1..=3 { + engine + .apply( + slot, + &Batch { + ops: vec![BatchOp { + key: Bytes::from(format!("busy:{slot}")), + op: Op::Put(Bytes::from_static(b"occupied")), + }], + }, + ) + .await + .unwrap(); + } + engine.handle().flush().unwrap(); + assert_eq!(engine.resume_from_slot(), 3); + // Earlier WAL segments were reclaimed after the tree became durable. + // The surviving tail still seeds the applied frontier on restart. + let entry = PxLogEntry { + slot: 3, + ballot: PxBallot::new(0, 1), + term: 1, + payload: Bytes::new(), + }; + let replay = ReplayResult { + records: vec![WALRecord::from_accepted(1, &entry)], + index: SegmentIndex::new(), + max_segment_id: 1, + current_term: 1, + voted_for: None, + }; + let replica = PxLocalReplica::restore_from_replay_with_engine( + 1, + PxLocalReplicaRole::Leader, + &replay, + Box::new(engine), + ) + .await + .unwrap(); + assert_eq!(replica.contiguous_applied(), 3); + assert_eq!(replica.acceptor.trim_slot(), 0); + let store = PxKvStore::new(0, "127.0.0.1:0".parse().unwrap()); + store.add_group(PxGroup::new(1, replica)); + let response = store + .kv_journal_scan(1, 1, 0, b"busy:", 0, ReadMode::Linearizable as i32, 1, 1) + .await; + assert_eq!( + response.error_code, + KvErrorCode::KvErrorJournalScanGcGap as i32, + "missing allocation history must select full-scan recovery: {response:?}" + ); + assert!(!response.ok); + let retained_tail = store + .kv_journal_scan(1, 3, 0, b"busy:", 0, ReadMode::Linearizable as i32, 3, 3) + .await; + assert!( + retained_tail.ok, + "the retained NoOp is complete history: {retained_tail:?}" + ); + assert!(retained_tail.ops.is_empty()); + let retained = store + .kv_get(1, b"busy:1", ReadMode::Linearizable as i32, 0, 2, 2) + .await; + assert!(retained.ok && !retained.not_found); +} diff --git a/lib/crowdb-kv/tests/store_test/multi_group_test.rs b/lib/crowdb-kv/tests/store_test/multi_group_test.rs index 9acd02f43..15ae6feca 100644 --- a/lib/crowdb-kv/tests/store_test/multi_group_test.rs +++ b/lib/crowdb-kv/tests/store_test/multi_group_test.rs @@ -68,6 +68,10 @@ async fn missing_group_returns_error() { // Put to non-existent group let resp = store.kv_put(99, b"k", b"v", 1, 1, 100, 1000).await; assert!(!resp.ok); + assert_eq!( + resp.error_code, + crowdb_kv::rpc::KvErrorCode::KvErrorNotLeader as i32 + ); assert!( resp.error.contains("no kv group"), "error should mention missing group, got: {}", diff --git a/lib/crowdb-protocol/src/mgmt/node/deployment.rs b/lib/crowdb-protocol/src/mgmt/node/deployment.rs index 540d242c4..c014c6275 100644 --- a/lib/crowdb-protocol/src/mgmt/node/deployment.rs +++ b/lib/crowdb-protocol/src/mgmt/node/deployment.rs @@ -4,6 +4,12 @@ //! Accepted startup inputs, distinct from discovered or confirmed membership. use super::setup::{NodeSetupChoices, SetupService, SetupServiceKind, SetupStorage}; +use crate::{ + ACCESS_SERVER_DATASET_HTTP_BASE, ACCESS_SERVER_HEALTH_HTTP_BASE, ACCESS_SERVER_HTTP_BASE, + ACCESS_SERVER_ICEBERG_HTTP_BASE, CHUNKDB_HTTP_BASE, CHUNKDB_RPC_BASE, CHUNK_KV_HTTP_BASE, + CHUNK_KV_RPC_BASE, DISKDB_HTTP_BASE, DISKDB_LISTEN_BASE, DISKDB_RPC_BASE, DISKIO_RPC_BASE, + KV_SERVER_LISTEN_BASE, KV_SERVER_MGMT_BASE, +}; use serde::{Deserialize, Serialize}; mod public_access; @@ -73,24 +79,28 @@ impl Default for NodeDeploymentPolicy { let listeners = &mut service.listeners; match service.kind { SetupServiceKind::PaxosKv => { - listeners.http_port = Some(10000); - listeners.rpc_port = Some(10100); + listeners.http_port = Some(KV_SERVER_MGMT_BASE); + listeners.rpc_port = Some(KV_SERVER_LISTEN_BASE); } - SetupServiceKind::Diskdb => listeners.rpc_port = Some(7100), - SetupServiceKind::Diskio => listeners.rpc_port = Some(7200), + SetupServiceKind::Diskdb => { + listeners.listen_port = Some(DISKDB_LISTEN_BASE); + listeners.http_port = Some(DISKDB_HTTP_BASE); + listeners.rpc_port = Some(DISKDB_RPC_BASE); + } + SetupServiceKind::Diskio => listeners.rpc_port = Some(DISKIO_RPC_BASE), SetupServiceKind::Chunkdb => { - listeners.http_port = Some(7300); - listeners.rpc_port = Some(7301); + listeners.http_port = Some(CHUNKDB_HTTP_BASE); + listeners.rpc_port = Some(CHUNKDB_RPC_BASE); } SetupServiceKind::ChunkKv => { - listeners.http_port = Some(7400); - listeners.rpc_port = Some(7401); + listeners.http_port = Some(CHUNK_KV_HTTP_BASE); + listeners.rpc_port = Some(CHUNK_KV_RPC_BASE); } SetupServiceKind::AccessServer => { - listeners.s3_port = Some(9091); - listeners.http_port = Some(9092); - listeners.dataset_port = Some(9093); - listeners.health_port = Some(9094); + listeners.s3_port = Some(ACCESS_SERVER_HTTP_BASE); + listeners.http_port = Some(ACCESS_SERVER_ICEBERG_HTTP_BASE); + listeners.dataset_port = Some(ACCESS_SERVER_DATASET_HTTP_BASE); + listeners.health_port = Some(ACCESS_SERVER_HEALTH_HTTP_BASE); } } } @@ -101,9 +111,9 @@ impl Default for NodeDeploymentPolicy { initial_voters: 1, setup, public_access: PublicAccessUris { - s3: Some("http://127.0.0.1:9091".into()), - iceberg: Some("http://127.0.0.1:9092".into()), - dataset: Some("http://127.0.0.1:9093".into()), + s3: Some(format!("http://127.0.0.1:{ACCESS_SERVER_HTTP_BASE}")), + iceberg: Some(format!("http://127.0.0.1:{ACCESS_SERVER_ICEBERG_HTTP_BASE}")), + dataset: Some(format!("http://127.0.0.1:{ACCESS_SERVER_DATASET_HTTP_BASE}")), }, } } @@ -142,6 +152,7 @@ impl NodeDeploymentPolicy { for service in self.setup.services.iter().filter(|service| service.enabled) { let ports = &service.listeners; if [ + ports.listen_port, ports.http_port, ports.rpc_port, ports.s3_port, diff --git a/lib/crowdb-protocol/src/mgmt/node/setup.rs b/lib/crowdb-protocol/src/mgmt/node/setup.rs index b4f462be3..b9b8a26be 100644 --- a/lib/crowdb-protocol/src/mgmt/node/setup.rs +++ b/lib/crowdb-protocol/src/mgmt/node/setup.rs @@ -33,6 +33,8 @@ impl SetupServiceKind { #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SetupListeners { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub listen_port: Option, pub http_port: Option, pub rpc_port: Option, pub s3_port: Option, diff --git a/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs b/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs index fd6b5e4d9..4de75f2d7 100644 --- a/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs +++ b/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs @@ -144,7 +144,11 @@ fn valid_listeners(service: &SetupService, ports: &mut BTreeSet) -> bool { || listeners.health_port.is_some() || listeners.dataset_port.is_some())) || (service.kind == AccessServer && listeners.rpc_port.is_some()) - || (matches!(service.kind, Diskdb | Diskio) && listeners.http_port.is_some()) + || (service.kind != Diskdb && listeners.listen_port.is_some()) + || (service.kind == Diskio && listeners.http_port.is_some()) + || (service.kind == Diskdb + && (listeners.listen_port.is_some() != listeners.http_port.is_some() + || (listeners.listen_port.is_some() && listeners.rpc_port.is_none()))) { return false; } @@ -152,6 +156,7 @@ fn valid_listeners(service: &SetupService, ports: &mut BTreeSet) -> bool { return true; } for port in [ + listeners.listen_port, listeners.http_port, listeners.rpc_port, listeners.s3_port, @@ -165,7 +170,7 @@ fn valid_listeners(service: &SetupService, ports: &mut BTreeSet) -> bool { return false; } } - if service.kind == Diskdb { + if service.kind == Diskdb && listeners.listen_port.is_none() { if let Some(port) = listeners.rpc_port { return port <= 65533 && ports.insert(port + 1) && ports.insert(port + 2); } diff --git a/lib/crowdb-protocol/tests/node_deployment_test.rs b/lib/crowdb-protocol/tests/node_deployment_test.rs index 81068ec4f..84ed3ef8c 100644 --- a/lib/crowdb-protocol/tests/node_deployment_test.rs +++ b/lib/crowdb-protocol/tests/node_deployment_test.rs @@ -3,7 +3,10 @@ use crowdb_protocol::mgmt::node::{ deployment::{DeploymentMode, NodeDeploymentOverrides, NodeDeploymentPolicy, PublicAccessUris}, - setup::{SetupServiceKind, SetupStorage}, + setup::{NodeSetupChoices, SetupServiceKind, SetupStorage}, +}; +use crowdb_protocol::{ + ACCESS_SERVER_DATASET_HTTP_BASE, ACCESS_SERVER_HTTP_BASE, ACCESS_SERVER_ICEBERG_HTTP_BASE, }; #[test] @@ -19,15 +22,22 @@ fn fresh_local_policy_has_one_voter_and_all_public_access_listeners() { SetupStorage::Automatic { sparse: true, .. } )); assert!(policy.setup.services.iter().all(|service| service.enabled)); + let diskdb = &policy.setup.services[1].listeners; + assert_eq!(diskdb.listen_port, Some(11000)); + assert_eq!(diskdb.http_port, Some(11100)); + assert_eq!(diskdb.rpc_port, Some(11200)); let access = policy .setup .services .iter() .find(|service| service.kind == SetupServiceKind::AccessServer) .unwrap(); - assert_eq!(access.listeners.s3_port, Some(9091)); - assert_eq!(access.listeners.http_port, Some(9092)); - assert_eq!(access.listeners.dataset_port, Some(9093)); + assert_eq!(access.listeners.s3_port, Some(ACCESS_SERVER_HTTP_BASE)); + assert_eq!(access.listeners.http_port, Some(ACCESS_SERVER_ICEBERG_HTTP_BASE)); + assert_eq!( + access.listeners.dataset_port, + Some(ACCESS_SERVER_DATASET_HTTP_BASE) + ); } #[test] @@ -60,6 +70,19 @@ fn public_origins_resolve_per_field_and_survive_omitted_restart_inputs() { assert_eq!(accepted.setup, NodeDeploymentPolicy::default().setup); } +#[test] +fn shifted_setup_retains_the_group0_listener_pair_for_admission_metadata() { + let mut choices = NodeSetupChoices::default(); + let kv = choices + .services + .iter_mut() + .find(|service| service.kind == SetupServiceKind::PaxosKv) + .unwrap(); + kv.listeners.http_port = Some(10001); + kv.listeners.rpc_port = Some(10101); + assert_eq!(choices.paxos_kv_ports(), Some((10001, 10101))); +} + #[test] fn malformed_or_secret_bearing_public_origins_never_resolve() { for uri in [ @@ -175,7 +198,7 @@ fn invalid_policies_are_rejected_before_any_resolution_can_replace_them() { .is_err()); } let mut duplicate_port = valid; - duplicate_port.setup.services[5].listeners.dataset_port = Some(9092); + duplicate_port.setup.services[5].listeners.dataset_port = Some(ACCESS_SERVER_ICEBERG_HTTP_BASE); assert!(duplicate_port.validate().is_err()); duplicate_port.setup.services[5].listeners.dataset_port = None; assert!(duplicate_port.validate().is_err()); From 8885d2adc351d9c99f0219e18ecf0c4782bc3079 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 31 Oct 2026 10:21:20 +0800 Subject: [PATCH 07/24] Fix five-voter recovery and CI readiness regressions --- .github/workflows/ci.yml | 5 + app/crowdb-chunkdb/src/topology.rs | 29 +- .../src/topology/availability.rs | 67 ++++ .../tests/common/topology_availability.rs | 23 ++ .../tests/topology_availability_test.rs | 98 +++++ app/crowdb-web/src/node/admission.rs | 5 + .../src/node/auto_join/formation.rs | 2 +- app/crowdb-web/src/node/preflight.rs | 1 + app/crowdb-web/tests/auto_formation_test.rs | 52 +++ app/crowdb-web/tests/auto_join_test.rs | 1 + app/crowdb-web/tests/cluster_target_test.rs | 1 + .../tests/node_listener_recovery_test.rs | 1 + app/crowdb-web/tests/node_preflight_test.rs | 1 + container/crowdb-monitor/src/lib.rs | 4 +- container/crowdb-monitor/src/node.rs | 4 +- container/crowdb-monitor/src/node/runtime.rs | 12 +- .../src/node/runtime/control.rs | 14 +- .../src/node/runtime/recovery.rs | 2 +- .../src/node/runtime/recovery/discovery.rs | 64 +++- .../src/node/runtime/recovery/publication.rs | 4 +- .../src/node/runtime/services.rs | 3 + .../src/node/runtime/services/execution.rs | 12 +- .../src/node/runtime/services/receipt.rs | 46 +++ .../src/node/runtime/services/test_support.rs | 10 + .../src/node/runtime/setup/execution.rs | 10 +- .../tests/common/admitted_member.rs | 3 +- .../tests/common/member_trust.rs | 1 + .../crowdb-monitor/tests/common/node_setup.rs | 1 + .../tests/common/service_receipt.rs | 107 ++++++ .../tests/initialization_test.rs | 1 + .../tests/member_discovery_test.rs | 93 +++++ .../crowdb-monitor/tests/node_cleanup_test.rs | 19 +- .../tests/node_run_inputs_test.rs | 2 +- .../tests/node_service_receipt_test.rs | 109 ++++++ doc/design/chunkdb/design-crowdb-chunkdb.md | 6 + .../chunkds/design-crowdb-chunk-stream.md | 7 + doc/design/deploy/design-crowdb-deploy.md | 41 +- doc/design/kv/design-crowdb-kv-slot.md | 70 +++- doc/working/plan-two-physical-nodes.md | 174 --------- doc/working/track-two-physical-nodes.md | 349 +++++++++++++++++- e2e-test/README.md | 6 + .../e2e/flows/01-shell-ui-behaviors.spec.ts | 8 +- .../e2e/flows/55-chunk-kv-catalog.spec.ts | 47 ++- e2e-test/console_cli_client.py | 3 +- e2e-test/console_cli_fixture.py | 10 +- e2e-test/console_cli_io_fault.py | 91 +++++ e2e-test/console_cli_recovery.py | 48 +-- e2e-test/console_cli_target.py | 9 +- e2e-test/console_native_fixture.py | 9 + .../tests/access_iceberg/gc/convergence.rs | 4 +- .../access_iceberg/gc/convergence/pressure.rs | 37 +- .../access_iceberg/gc/notifications/matrix.rs | 2 +- .../tests/access_iceberg/gc/sdk_pressure.py | 10 +- e2e-test/tests/common/control.rs | 8 + e2e-test/tests/console/deployment.rs | 2 + e2e-test/tests/console/native_balance.rs | 26 ++ e2e-test/tests/console/native_browser.rs | 28 +- e2e-test/tests/console_cli_test.py | 72 +++- e2e-test/tests/deployment.rs | 3 + e2e-test/tests/deployment/kv_bootstrap.rs | 66 ++++ .../src/production_chunk.rs | 12 +- .../src/production_chunk/recovery.rs | 28 +- .../tests/production_chunk_test.rs | 3 + .../production_chunk_test/finalized_empty.rs | 145 ++++++++ .../src/deployment/admission.rs | 2 + .../src/deployment/preparation/record.rs | 4 +- .../src/deployment/registry.rs | 9 + .../src/ops/cluster/readiness.rs | 3 +- .../src/ops/cluster/readiness/collect.rs | 49 ++- .../ops/cluster/readiness/collect/members.rs | 10 +- .../tests/cluster_readiness_test.rs | 23 +- .../tests/cluster_storage_observation_test.rs | 104 ++++++ ...uster_supplied_storage_observation_test.rs | 1 + .../tests/common/member_evidence.rs | 1 + .../tests/default_preparation_test.rs | 47 +++ .../tests/member_evidence_test.rs | 21 ++ lib/crowdb-kv-client/src/lib.rs | 1 + lib/crowdb-kv-client/src/service/registry.rs | 2 +- lib/crowdb-kv/src/cluster/group.rs | 13 +- lib/crowdb-kv/src/cluster/group_election.rs | 161 +------- .../src/cluster/group_election/recovery.rs | 113 ++++++ .../src/cluster/group_election_leader.rs | 5 +- lib/crowdb-kv/src/cluster/group_fetchgap.rs | 66 +++- lib/crowdb-kv/src/cluster/group_tests.rs | 8 + .../src/cluster/local_replica_replay.rs | 61 ++- lib/crowdb-kv/src/common/config.rs | 9 +- lib/crowdb-kv/src/paxos/acceptor.rs | 23 ++ lib/crowdb-kv/tests/group_test.rs | 6 + .../group_test/fetch_gap_durability_test.rs | 153 ++++++++ .../tests/group_test/g3_leader_change_test.rs | 35 +- .../tests/group_test/recovery_backlog_test.rs | 92 +++++ lib/crowdb-kv/tests/rpc_migration_test.rs | 9 +- lib/crowdb-kv/tests/wal_test/replay_test.rs | 103 ++++-- .../src/mgmt/node/deployment.rs | 6 +- lib/crowdb-protocol/src/mgmt/node/setup.rs | 10 + .../tests/node_deployment_test.rs | 26 ++ 96 files changed, 2714 insertions(+), 593 deletions(-) create mode 100644 app/crowdb-chunkdb/src/topology/availability.rs create mode 100644 app/crowdb-chunkdb/tests/common/topology_availability.rs create mode 100644 app/crowdb-chunkdb/tests/topology_availability_test.rs create mode 100644 container/crowdb-monitor/src/node/runtime/services/receipt.rs create mode 100644 container/crowdb-monitor/tests/common/service_receipt.rs create mode 100644 container/crowdb-monitor/tests/member_discovery_test.rs create mode 100644 container/crowdb-monitor/tests/node_service_receipt_test.rs delete mode 100644 doc/working/plan-two-physical-nodes.md create mode 100644 e2e-test/console_cli_io_fault.py create mode 100644 lib/crowdb-chunk-stream/tests/production_chunk_test/finalized_empty.rs create mode 100644 lib/crowdb-kv/src/cluster/group_election/recovery.rs create mode 100644 lib/crowdb-kv/tests/group_test/fetch_gap_durability_test.rs create mode 100644 lib/crowdb-kv/tests/group_test/recovery_backlog_test.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 508b696da..e78d2ec85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -654,6 +654,11 @@ jobs: - uses: prefix-dev/setup-pixi@v0.8.1 with: pixi-version: v0.77.0 + - name: Install owned network fault tools + if: matrix.partition == 'console-api' + run: | + pixi run sudo apt-get update + pixi run sudo apt-get install -y iptables - uses: ./.github/actions/restore-compiled with: artifact: ${{ needs.BuildClients.outputs.artifact }} diff --git a/app/crowdb-chunkdb/src/topology.rs b/app/crowdb-chunkdb/src/topology.rs index c97fbd9bb..bb11af798 100644 --- a/app/crowdb-chunkdb/src/topology.rs +++ b/app/crowdb-chunkdb/src/topology.rs @@ -3,9 +3,9 @@ //! Topology cache for chunkdb placement decisions. //! -//! `TopologyCache` holds an `Arc>` with the +//! `TopologyCache` publishes an immutable `ArcSwap` with the //! current cluster hierarchy (racks, nodes, disk-groups). Placement -//! (R87) calls `snapshot()` to get a consistent point-in-time view. +//! calls `snapshot()` to get a consistent point-in-time view. //! //! Update path: periodic full refresh from group-0 via //! `HardwareClient` + watch/notify for immediate status changes. @@ -23,6 +23,7 @@ use crowdb_protocol::{DiskGroupId, NodeId, RackId}; use crate::selector::PlacementEntry; +mod availability; pub mod notify; pub mod refresh; @@ -33,7 +34,7 @@ pub const G0_GROUP: u64 = 0; /// Watch prefixes for chunkdb topology updates. Disk records carry the /// allocation geometry; without their notifications a ChunkDB started before /// DiskIO publishes hardware can retain a zero unit size indefinitely. -pub const CHUNKDB_WATCH_PREFIXES: &[&[u8]] = &[b"/hw/node/", b"/hw/dg/", b"/hw/disk/"]; +pub const CHUNKDB_WATCH_PREFIXES: &[&[u8]] = &[b"/hw/node/", b"/hw/dg/", b"/hw/disk/", b"/srv/diskio/"]; /// `HwStatus::Up` as `i32` (prost represents enums as i32 in messages). const HW_UP: i32 = HwStatus::Up as i32; @@ -52,6 +53,8 @@ pub struct TopologySnapshot { disk_groups: HashMap, /// disk_id → physical failure-domain location for post-allocation checks. disks: HashMap, + /// Located DiskIO observations expire independently of hardware intent. + diskio_deadlines: HashMap, /// Capacity observations joined to live disk-group membership. usage: HashMap, /// Monotonic local publication generation. @@ -139,6 +142,16 @@ impl Drop for InFlightReservation { impl TopologySnapshot { /// All disk-groups with healthy status (rack/node/dg all `Up`). pub fn healthy_disk_groups(&self) -> Vec<&DiskGroupEntry> { + self.healthy_disk_groups_at(availability::now_ms()) + } + + #[cfg(feature = "test-util")] + #[must_use] + pub fn healthy_disk_groups_at_for_tests(&self, now: u64) -> Vec<&DiskGroupEntry> { + self.healthy_disk_groups_at(now) + } + + fn healthy_disk_groups_at(&self, now: u64) -> Vec<&DiskGroupEntry> { self.disk_groups .values() .filter(|dg| { @@ -148,7 +161,13 @@ impl TopologySnapshot { .get(&(dg.rack_id, dg.node_id)) .is_some_and(|(s, _)| *s == HW_UP); let rack_ok = self.racks.get(&dg.rack_id).is_some_and(|(s, _)| *s == HW_UP); - dg_ok && node_ok && rack_ok + dg_ok + && node_ok + && rack_ok + && self + .diskio_deadlines + .get(&dg.dg_id) + .map_or(true, |deadline| *deadline >= now) }) .collect() } @@ -497,6 +516,7 @@ pub async fn build_snapshot(hw: &crowdb_kv_client::HardwareClient) -> Option Option +// Licensed under the Apache License, Version 2.0. + +//! Physical IO availability overlays durable hardware intent for placement. + +use crowdb_protocol::{common::InstanceValue, sysdata::DiskGroupEntry, DiskGroupId}; +use std::{ + collections::HashMap, + time::{SystemTime, UNIX_EPOCH}, +}; + +pub(super) async fn read( + service: &crowdb_kv_client::ServiceRegistryClient, +) -> Option> { + match service.read_all_instance_observations("diskio").await { + Ok(instances) => Some(instances), + Err(error) => { + tracing::warn!(%error, "topology refresh: DiskIO availability is unavailable"); + None + } + } +} + +pub(super) fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |time| u64::try_from(time.as_millis()).unwrap_or(u64::MAX)) +} + +pub(super) fn deadlines( + groups: &HashMap, + instances: &[(u64, InstanceValue)], +) -> HashMap { + let mut deadlines = HashMap::::new(); + for (_, instance) in instances { + let Some(extra) = instance.extra.as_ref().and_then(|extra| extra.diskdb.as_ref()) else { + continue; + }; + let (Some(rack), Some(node)) = (extra.rack_id, extra.node_id) else { + continue; + }; + for group in groups + .values() + .filter(|group| group.rack_id == rack && group.node_id == node) + { + if extra + .diskio_scope + .as_ref() + .is_some_and(|scope| scope.disk_group_id != 0 && scope.disk_group_id != group.dg_id) + { + continue; + } + let deadline = if extra.owned_dg_ids.contains(&group.dg_id) { + instance + .last_heartbeat_ms + .saturating_add(crowdb_kv_client::SERVICE_INSTANCE_TTL_MS) + } else { + 0 + }; + deadlines + .entry(group.dg_id) + .and_modify(|current| *current = (*current).max(deadline)) + .or_insert(deadline); + } + } + deadlines +} diff --git a/app/crowdb-chunkdb/tests/common/topology_availability.rs b/app/crowdb-chunkdb/tests/common/topology_availability.rs new file mode 100644 index 000000000..6efba4a20 --- /dev/null +++ b/app/crowdb-chunkdb/tests/common/topology_availability.rs @@ -0,0 +1,23 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_kv_client::ServiceRegistryClient; +use crowdb_protocol::{common::DiskioHardwareScope, sysdata::DiskGroupEntry}; + +pub async fn heartbeat(registry: &ServiceRegistryClient, group: &DiskGroupEntry) { + registry + .heartbeat_diskio_scoped( + group.dg_id, + "127.0.0.1:13000", + &DiskioHardwareScope { + version: 1, + rack_id: group.rack_id, + node_id: group.node_id, + disk_group_id: group.dg_id, + }, + &[group.dg_id], + &[], + ) + .await + .unwrap(); +} diff --git a/app/crowdb-chunkdb/tests/topology_availability_test.rs b/app/crowdb-chunkdb/tests/topology_availability_test.rs new file mode 100644 index 000000000..8bf884342 --- /dev/null +++ b/app/crowdb-chunkdb/tests/topology_availability_test.rs @@ -0,0 +1,98 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/topology_availability.rs"] +mod availability; +mod common; + +use crowdb_chunkdb::{ + selector::{MirrorPlacement, PlacementConstraints}, + topology::build_snapshot, +}; +use crowdb_kv_client::ServiceRegistryClient; +use crowdb_protocol::{ + common::{HwStatus, InstanceValue}, + key::InstanceKey, +}; + +#[tokio::test] +async fn expired_physical_io_is_excluded_without_changing_hardware_and_returns_after_heartbeat() { + let cluster = common::cluster::KvCluster::start_pinned().await; + let hardware = cluster.make_hardware_client(); + common::cluster::seed_hardware(&hardware).await; + let registry = ServiceRegistryClient::from_shared(cluster.make_crowdb_client()); + let groups = hardware.list_disk_groups().await.unwrap(); + assert!(groups.len() >= 3); + for group in &groups { + availability::heartbeat(®istry, group).await; + } + let before = build_snapshot(&hardware).await.unwrap(); + assert_eq!(before.healthy_disk_groups().len(), groups.len()); + let expired = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap() + + crowdb_kv_client::SERVICE_INSTANCE_TTL_MS + + 1; + assert!(before.healthy_disk_groups_at_for_tests(expired).is_empty()); + let stopped = &groups[0]; + let key = InstanceKey { + service: "diskio".into(), + instance_id: stopped.dg_id, + } + .to_path(); + let kv = cluster.make_crowdb_client(); + let value = kv + .get( + 0, + 0, + key.as_bytes(), + crowdb_kv_client::ReadMode::Linearizable, + None, + ) + .await + .unwrap(); + let crowdb_kv_client::GetOutcome::Found { value, .. } = value else { + panic!("missing IO registration") + }; + let mut observation: InstanceValue = serde_json::from_slice(&value).unwrap(); + observation.last_heartbeat_ms = 0; + kv.put( + 0, + 0, + key.as_bytes(), + &serde_json::to_vec(&observation).unwrap(), + None, + ) + .await + .unwrap(); + let unavailable = build_snapshot(&hardware).await.unwrap(); + assert!(unavailable + .healthy_disk_groups() + .iter() + .all(|group| group.dg_id != stopped.dg_id)); + let plan = MirrorPlacement::select(&unavailable, 2, &PlacementConstraints::default()).unwrap(); + assert!(plan + .entries + .iter() + .all(|entry| entry.disk_group_id != stopped.dg_id)); + assert_eq!(before.healthy_disk_groups().len(), groups.len()); + assert!(hardware + .list_disk_groups() + .await + .unwrap() + .iter() + .all(|group| group.value.status == HwStatus::Up as i32)); + availability::heartbeat(®istry, stopped).await; + assert_eq!( + build_snapshot(&hardware) + .await + .unwrap() + .healthy_disk_groups() + .len(), + groups.len() + ); +} diff --git a/app/crowdb-web/src/node/admission.rs b/app/crowdb-web/src/node/admission.rs index 8185a0162..69e67d691 100644 --- a/app/crowdb-web/src/node/admission.rs +++ b/app/crowdb-web/src/node/admission.rs @@ -179,6 +179,10 @@ async fn admit_observed( .await .map_err(map_config_err)?; record.pin_host_key(&identity.host_key).map_err(map_config_err)?; + if identity.web_port == 0 { + return Err(err_409("Authenticated target has an invalid Web listener")); + } + record.web_port = identity.web_port; if let Some(binding) = &binding { record = registry::admit( state.kv_client().await.as_ref(), @@ -436,6 +440,7 @@ fn initial_record( record } else { NodeRecord { + web_port: 9090, discovery_id: id.to_string(), node_id: next_node_id(config, &records(state)?)?, physical_host_id: physical_host.to_owned(), diff --git a/app/crowdb-web/src/node/auto_join/formation.rs b/app/crowdb-web/src/node/auto_join/formation.rs index 218a24fe0..1482c3b49 100644 --- a/app/crowdb-web/src/node/auto_join/formation.rs +++ b/app/crowdb-web/src/node/auto_join/formation.rs @@ -56,7 +56,7 @@ pub(super) fn candidates<'a>( "Too many fresh nodes in the cluster scope; select an unambiguous formation", )); } - if policy.initial_voters == 3 + if policy.initial_voters > 1 && snapshot.candidates.iter().any(|peer| { peer.state == CandidateState::ForeignCluster && peer.advertisement.formation.as_ref() == Some(&expected) diff --git a/app/crowdb-web/src/node/preflight.rs b/app/crowdb-web/src/node/preflight.rs index 32a7269fc..e7adcb8bb 100644 --- a/app/crowdb-web/src/node/preflight.rs +++ b/app/crowdb-web/src/node/preflight.rs @@ -113,6 +113,7 @@ pub(crate) async fn prepare( record.clone() } else { NodeRecord { + web_port: 9090, discovery_id: id.to_string(), node_id: super::admission::next_node_id(&config, &records)?, physical_host_id: handshake.physical_host_id, diff --git a/app/crowdb-web/tests/auto_formation_test.rs b/app/crowdb-web/tests/auto_formation_test.rs index 37a9e5ce3..c72a3e0f3 100644 --- a/app/crowdb-web/tests/auto_formation_test.rs +++ b/app/crowdb-web/tests/auto_formation_test.rs @@ -134,3 +134,55 @@ async fn missing_exposed_empty_or_corrupt_local_credentials_never_fall_back_to_a assert!(!fixture.root.join("accepted-node.json").exists()); } } + +#[tokio::test] +async fn five_voter_scope_preserves_incomplete_overfull_and_foreign_member_fences() { + let expected = policy(5, DeploymentMode::Local); + for count in 0..4 { + let peers = vec![(expected.formation(), CandidateState::Unbound); count]; + let fixture = TestFormationFixture::start(expected.clone(), peers, 1).await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "waiting", "{report}"); + assert_eq!(report["required_voters"], 5); + assert_eq!(report["observed_voters"], count + 1); + fixture.assert_no_admission(); + } + for state in [CandidateState::ForeignCluster, CandidateState::IdentityConflict] { + let fixture = + TestFormationFixture::start(expected.clone(), vec![(expected.formation(), state)], 1).await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "failed", "{report}"); + fixture.assert_no_admission(); + } + let fixture = TestFormationFixture::start( + expected.clone(), + vec![(expected.formation(), CandidateState::Unbound); 5], + 1, + ) + .await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "failed", "{report}"); + assert!(report["error"].as_str().unwrap().contains("Too many fresh nodes")); + fixture.assert_no_admission(); + let fixture = TestFormationFixture::start( + expected.clone(), + vec![(expected.formation(), CandidateState::Unbound); 4], + 99, + ) + .await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "waiting", "{report}"); + assert_eq!(report["coordinator"], "00000000-0000-4000-8000-000000000002"); + fixture.assert_no_admission(); + let production = policy(5, DeploymentMode::Production); + let fixture = TestFormationFixture::with_password( + production.clone(), + vec![(production.formation(), CandidateState::Unbound); 4], + 1, + Some((b"local-test-password", 0o600)), + ) + .await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "needs-rack", "{report}"); + fixture.assert_no_admission(); +} diff --git a/app/crowdb-web/tests/auto_join_test.rs b/app/crowdb-web/tests/auto_join_test.rs index 6597a4d40..e780fd419 100644 --- a/app/crowdb-web/tests/auto_join_test.rs +++ b/app/crowdb-web/tests/auto_join_test.rs @@ -189,6 +189,7 @@ async fn two_authenticated_nodes_wait_in_uuid_order_without_initializing() { ssh_credential_ref: Some("id_ed25519".into()), }); let record = NodeRecord { + web_port: 9090, discovery_id: discovery_id.into(), node_id, physical_host_id: "host".into(), diff --git a/app/crowdb-web/tests/cluster_target_test.rs b/app/crowdb-web/tests/cluster_target_test.rs index 13c5e66b0..17fdcefe0 100644 --- a/app/crowdb-web/tests/cluster_target_test.rs +++ b/app/crowdb-web/tests/cluster_target_test.rs @@ -143,6 +143,7 @@ async fn target_uses_confirmed_current_addresses_without_rewriting_the_manifest( let registry = NodeRegistry { cluster_id: fixture.prepared.identity.cluster_id.clone(), nodes: vec![NodeRecord { + web_port: 9090, discovery_id: uuid::Uuid::new_v4().to_string(), node_id: 2, physical_host_id: "test-host".into(), diff --git a/app/crowdb-web/tests/node_listener_recovery_test.rs b/app/crowdb-web/tests/node_listener_recovery_test.rs index bc0657501..3eb38f6bd 100644 --- a/app/crowdb-web/tests/node_listener_recovery_test.rs +++ b/app/crowdb-web/tests/node_listener_recovery_test.rs @@ -27,6 +27,7 @@ async fn background_protocol_reads_follow_confirmed_addresses_without_ui_refresh let registry = NodeRegistry { cluster_id: fixture.binding.bootstrap.cluster_id.clone(), nodes: vec![NodeRecord { + web_port: 9090, discovery_id: discovery.clone(), node_id: 2, physical_host_id: "retained-host".into(), diff --git a/app/crowdb-web/tests/node_preflight_test.rs b/app/crowdb-web/tests/node_preflight_test.rs index db4b955e6..5d048132e 100644 --- a/app/crowdb-web/tests/node_preflight_test.rs +++ b/app/crowdb-web/tests/node_preflight_test.rs @@ -19,6 +19,7 @@ async fn a_preflight_review_is_not_admission_and_discard_is_operation_fenced_and let discovery = uuid::Uuid::new_v4().to_string(); let operation = uuid::Uuid::new_v4().to_string(); let record = NodeRecord { + web_port: 9090, discovery_id: discovery.clone(), operation_id: operation.clone(), node_id: 1, diff --git a/container/crowdb-monitor/src/lib.rs b/container/crowdb-monitor/src/lib.rs index 772817db9..44e834dde 100644 --- a/container/crowdb-monitor/src/lib.rs +++ b/container/crowdb-monitor/src/lib.rs @@ -31,8 +31,6 @@ pub use liveness::{probe_liveness, LivenessError, LivenessServer}; pub use manifest::{BootstrapManifest, BootstrapSession, ManifestError, ManifestState}; pub use monitor_log::{MonitorEvent, MonitorEventKind, MonitorLog, MonitorLogError}; #[cfg(feature = "test-util")] -pub use node::unique_member_endpoints_for_tests; -#[cfg(feature = "test-util")] pub use node::{ accept_file_for_tests, accept_member_trust_for_tests, accept_startup_mode_for_tests, check_member_trust_for_tests, check_retained_medium_for_tests, file_operations_for_tests, @@ -48,6 +46,8 @@ pub use node::{ NodeManagementError, NodeRuntimeConfig, NODE_SERVICE_TYPE, }; #[cfg(feature = "test-util")] +pub use node::{member_recovery_destinations_for_tests, unique_member_endpoints_for_tests}; +#[cfg(feature = "test-util")] pub use preview::{preview_step_names_for_tests, run_preview_for_tests, PreviewError}; pub use probe::{ProbeError, ProbeExecutor}; pub use process::{ProcessError, ProcessManager}; diff --git a/container/crowdb-monitor/src/node.rs b/container/crowdb-monitor/src/node.rs index 5a3dd76c8..e43ac1144 100644 --- a/container/crowdb-monitor/src/node.rs +++ b/container/crowdb-monitor/src/node.rs @@ -31,8 +31,6 @@ pub use runtime::prepare_supplied_disk_for_tests; #[cfg(feature = "test-util")] pub use runtime::runtime_observation_for_tests; #[cfg(feature = "test-util")] -pub use runtime::unique_member_endpoints_for_tests; -#[cfg(feature = "test-util")] pub use runtime::InitializationForTests; #[cfg(feature = "test-util")] pub use runtime::NodeSetupForTests; @@ -42,6 +40,8 @@ pub use runtime::{ }; pub use runtime::{control_node, run_node, run_node_with_inputs, NodeRuntimeConfig}; #[cfg(feature = "test-util")] +pub use runtime::{member_recovery_destinations_for_tests, unique_member_endpoints_for_tests}; +#[cfg(feature = "test-util")] pub use runtime::{publish_file_for_tests, resume_file_operations_for_tests}; #[cfg(feature = "test-util")] pub use startup::accept_startup_mode_for_tests; diff --git a/container/crowdb-monitor/src/node/runtime.rs b/container/crowdb-monitor/src/node/runtime.rs index d5084a3e5..d1dc30f05 100644 --- a/container/crowdb-monitor/src/node/runtime.rs +++ b/container/crowdb-monitor/src/node/runtime.rs @@ -58,11 +58,11 @@ mod recovery; pub use ipc::control_node; use ipc::read_request; #[cfg(feature = "test-util")] -pub use recovery::unique_member_endpoints_for_tests; -#[cfg(feature = "test-util")] pub use recovery::{ accept_member_trust_for_tests, check_member_trust_for_tests, member_trust_ready_for_tests, }; +#[cfg(feature = "test-util")] +pub use recovery::{member_recovery_destinations_for_tests, unique_member_endpoints_for_tests}; type Result = std::result::Result>; @@ -128,7 +128,13 @@ pub async fn run_node_with_inputs( let web = web_service(&profile, config.bind.port())?; processes.start(&web, &BTreeMap::new()).await?; recovery::publish_local(&root, &config.discovery.addresses)?; - let endpoint_recovery = recovery::start(root.clone(), config.bind.port()); + let web_port = profile + .public_endpoints + .iter() + .find(|endpoint| endpoint.id == "web") + .ok_or("web public endpoint is absent from node profile")? + .port; + let endpoint_recovery = recovery::start(root.clone(), config.bind.port(), web_port); kv::resume(&profile, &mut processes).await?; let (stop, stopped) = tokio::sync::oneshot::channel(); let discovery_root = root.clone(); diff --git a/container/crowdb-monitor/src/node/runtime/control.rs b/container/crowdb-monitor/src/node/runtime/control.rs index 5c5114abf..a6ad0d5bb 100644 --- a/container/crowdb-monitor/src/node/runtime/control.rs +++ b/container/crowdb-monitor/src/node/runtime/control.rs @@ -61,8 +61,18 @@ pub(super) async fn execute( node_id, admission, } => admission::cancel(profile, processes, bootstrap, node_id, admission).await, - request @ (NodeControl::Identity - | NodeControl::InstallKey { .. } + NodeControl::Identity => { + let mut value = identity(root)?; + let port = profile + .public_endpoints + .iter() + .find(|endpoint| endpoint.id == "web") + .ok_or("web public endpoint is absent from node profile")? + .port; + value["web_port"] = port.into(); + Ok(value) + } + request @ (NodeControl::InstallKey { .. } | NodeControl::RemoveKey { .. } | NodeControl::TrustHost { .. }) => keys(root, request).await, NodeControl::Bind { binding } => kv::bind(profile, processes, binding).await, diff --git a/container/crowdb-monitor/src/node/runtime/recovery.rs b/container/crowdb-monitor/src/node/runtime/recovery.rs index a4808a88b..8bbaec613 100644 --- a/container/crowdb-monitor/src/node/runtime/recovery.rs +++ b/container/crowdb-monitor/src/node/runtime/recovery.rs @@ -20,7 +20,7 @@ mod publication; pub(super) use confirmed::{accept as accept_confirmed, prepare_admitted}; pub(super) use discovery::start; #[cfg(feature = "test-util")] -pub use discovery::unique_member_endpoints_for_tests; +pub use discovery::{member_recovery_destinations_for_tests, unique_member_endpoints_for_tests}; pub(super) fn publish_local(root: &Path, addresses: &[std::net::IpAddr]) -> Result<()> { durable::write( diff --git a/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs b/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs index bf050d51d..2f54d2416 100644 --- a/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs +++ b/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs @@ -25,20 +25,20 @@ impl Drop for Worker { } } -pub(in crate::node::runtime) fn start(root: std::path::PathBuf, port: u16) -> Worker { +pub(in crate::node::runtime) fn start(root: std::path::PathBuf, port: u16, web_port: u16) -> Worker { Worker(tokio::spawn(async move { let mut tick = tokio::time::interval(Duration::from_secs(3)); tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); loop { tick.tick().await; - if let Err(error) = refresh(&root, port).await { + if let Err(error) = refresh(&root, port, web_port).await { tracing::debug!(%error, "member endpoint recovery is pending"); } } })) } -async fn refresh(root: &Path, port: u16) -> Result<()> { +async fn refresh(root: &Path, port: u16, web_port: u16) -> Result<()> { let Some(accepted) = durable::read::(&root.join("accepted-node.json"))? else { return Ok(()); }; @@ -52,9 +52,14 @@ async fn refresh(root: &Path, port: u16) -> Result<()> { { return Err("member recovery is retired".into()); } + let authority_start = std::time::Instant::now(); if let Err(error) = super::confirmed::refresh(root, &accepted).await { tracing::debug!(%error, "confirmed recovery contacts await current authority"); } + // Confirmation and publication share their existing three-second budget. + // A ten-second offline SSH proof must not stretch route refresh past the + // fifteen-second freshness fence while Group 0 is unavailable. + let publication_budget = Duration::from_secs(3).saturating_sub(authority_start.elapsed()); super::check(root, &accepted)?; let operation = PreparedBootstrap::load(&root.join("prepared-bootstrap.json"))?; let (trust, config) = super::confirmed::contacts(root, &operation)?; @@ -95,8 +100,8 @@ async fn refresh(root: &Path, port: u16) -> Result<()> { }; durable::write(&root.join("member-endpoints.json"), &snapshot)?; match tokio::time::timeout( - Duration::from_secs(3), - super::publication::reconcile(root, &operation, &snapshot, &config), + publication_budget, + super::publication::reconcile(root, &operation, &snapshot, &config, web_port), ) .await { @@ -139,19 +144,40 @@ fn destinations( if destinations.len() >= 64 || !destinations.insert((member.node_id, host)) { continue; } - let mut node = config - .node(member.node_id) - .cloned() - .ok_or("member topology is absent")?; - node.host = host.to_string(); - node.ssh_password = None; - node.ssh_key = Some(root.join("ssh/id_ed25519").to_string_lossy().into_owned()); - probes.push(node); + probes.push(probe_node(root, config, member.node_id, host)?); + } + } + // Fresh discovery gets the bounded probe budget first so retained old + // addresses cannot crowd out a changed IP. Retained addresses are hints too: pinned SSH still proves the member + // before a route is published. Recovery must not depend on multicast or + // on Group 0 already being available after its majority restarts. + for member in trust.members.iter().take(64) { + if destinations.len() >= 64 { + break; + } + let node = config.node(member.node_id).ok_or("member topology is absent")?; + if let Ok(host) = node.host.parse::() { + if destinations.insert((member.node_id, host)) { + probes.push(probe_node(root, config, member.node_id, host)?); + } } } Ok(probes) } +fn probe_node( + root: &Path, + config: &ConsoleConfig, + node_id: u64, + host: std::net::IpAddr, +) -> Result { + let mut node = config.node(node_id).cloned().ok_or("member topology is absent")?; + node.host = host.to_string(); + node.ssh_password = None; + node.ssh_key = Some(root.join("ssh/id_ed25519").to_string_lossy().into_owned()); + Ok(node) +} + fn unique(endpoints: impl Iterator) -> (Vec, Vec) { let mut members = BTreeMap::::new(); let mut conflicts = BTreeSet::new(); @@ -181,3 +207,15 @@ pub fn unique_member_endpoints_for_tests( ) -> (Vec, Vec) { unique(endpoints.into_iter()) } + +#[cfg(feature = "test-util")] +/// # Errors +/// Rejects trusted members missing from the retained topology. +pub fn member_recovery_destinations_for_tests( + root: &Path, + config: &ConsoleConfig, + trust: &MemberTrust, + snapshot: CandidateSnapshot, +) -> Result> { + destinations(root, config, trust, snapshot) +} diff --git a/container/crowdb-monitor/src/node/runtime/recovery/publication.rs b/container/crowdb-monitor/src/node/runtime/recovery/publication.rs index c28571280..a72a043fc 100644 --- a/container/crowdb-monitor/src/node/runtime/recovery/publication.rs +++ b/container/crowdb-monitor/src/node/runtime/recovery/publication.rs @@ -18,6 +18,7 @@ pub(super) async fn reconcile( operation: &PreparedBootstrap, endpoints: &MemberEndpoints, topology: &ConsoleConfig, + web_port: u16, ) -> Result<()> { let local = super::identity(root)?.endpoint; let mut config = topology.clone(); @@ -60,8 +61,9 @@ pub(super) async fn reconcile( .ok_or("local node topology missing")? .host .clone(); - if record.host != current { + if record.host != current || record.web_port != web_port { record.host.clone_from(¤t); + record.web_port = web_port; node_update::apply(&ctx, &operation.identity.cluster_id, record).await?; } super::owners::capture(root, &ctx, &operation.identity, local.node_id).await?; diff --git a/container/crowdb-monitor/src/node/runtime/services.rs b/container/crowdb-monitor/src/node/runtime/services.rs index 2664eaae2..579a03b40 100644 --- a/container/crowdb-monitor/src/node/runtime/services.rs +++ b/container/crowdb-monitor/src/node/runtime/services.rs @@ -11,6 +11,8 @@ use std::time::Duration; #[derive(Serialize, Deserialize)] pub(super) struct Execution { pub(super) intent: NodeServiceIntent, + #[serde(default)] + bootstrap: Option, complete: bool, pid: Option, #[serde(default)] @@ -28,6 +30,7 @@ mod execution; pub(super) use dependencies::ready as dependencies_ready; mod observation; mod policy; +mod receipt; pub(super) use execution::execute; pub(super) use observation::observe; diff --git a/container/crowdb-monitor/src/node/runtime/services/execution.rs b/container/crowdb-monitor/src/node/runtime/services/execution.rs index 88f7d8a9c..a2db1442e 100644 --- a/container/crowdb-monitor/src/node/runtime/services/execution.rs +++ b/container/crowdb-monitor/src/node/runtime/services/execution.rs @@ -19,6 +19,12 @@ pub(in crate::node::runtime) async fn execute( if binding.node_id != intent.node_id || binding.bootstrap.cluster_id != intent.cluster_id { return Err("service intent belongs to another node or cluster".into()); } + let directory = profile.paths.data_root.join("services").join(&intent.service_id); + let state = directory.join("execution.json"); + let previous = durable::read::(&state)?; + if super::receipt::completed_kv_stop(profile, processes, &binding, &intent, previous.as_ref())? { + return Ok(json!({"pid": null, "operation_id": intent.operation_id})); + } tokio::time::timeout(Duration::from_secs(3), super::super::verify_binding(&binding)).await??; verify_intent(&binding, &intent).await?; let process_id = if intent.kind == "kv" { @@ -26,12 +32,10 @@ pub(in crate::node::runtime) async fn execute( } else { &intent.service_id }; - let directory = profile.paths.data_root.join("services").join(&intent.service_id); std::fs::create_dir_all(&directory)?; - let state = directory.join("execution.json"); let mut attempts = 0; let mut continuing = false; - if let Some(previous) = durable::read::(&state)? { + if let Some(previous) = previous { if previous.intent.operation_id == intent.operation_id { if previous.intent != intent { return Err("operation identity reused with other inputs".into()); @@ -68,6 +72,7 @@ pub(in crate::node::runtime) async fn execute( &state, &Execution { intent: intent.clone(), + bootstrap: Some(binding.bootstrap.clone()), complete: false, pid: None, attempts, @@ -93,6 +98,7 @@ pub(in crate::node::runtime) async fn execute( &state, &Execution { intent: intent.clone(), + bootstrap: Some(binding.bootstrap.clone()), complete, pid, attempts: if complete { 0 } else { attempts }, diff --git a/container/crowdb-monitor/src/node/runtime/services/receipt.rs b/container/crowdb-monitor/src/node/runtime/services/receipt.rs new file mode 100644 index 000000000..fcfa3ce14 --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/services/receipt.rs @@ -0,0 +1,46 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Completed stop receipts observe a prior effect; they never execute one. + +use super::{ + durable, DeploymentProfile, Execution, NodeServiceAction, NodeServiceIntent, ProcessManager, Result, +}; +use crowdb_protocol::mgmt::node::NodeBinding; + +pub(super) fn completed_kv_stop( + profile: &DeploymentProfile, + processes: &mut ProcessManager, + binding: &NodeBinding, + intent: &NodeServiceIntent, + previous: Option<&Execution>, +) -> Result { + let Some(previous) = previous else { + return Ok(false); + }; + if intent.kind != "kv" + || !matches!(intent.action, NodeServiceAction::Stop | NodeServiceAction::Delete) + || previous.intent != *intent + || previous.bootstrap.as_ref() != Some(&binding.bootstrap) + || !previous.complete + || previous.pid.is_some() + || processes.owns("kv") && processes.alive("kv")? + { + return Ok(false); + } + let root = &profile.paths.data_root; + let Some(accepted) = durable::read::(&root.join("accepted-node.json"))? + else { + return Ok(false); + }; + Ok(accepted.prepared + && accepted.node_id == binding.node_id + && accepted.bootstrap == binding.bootstrap + && !super::super::admission::is_cancelled(root, &accepted) + && !root + .join(format!( + "retired-bootstrap-{}.json", + binding.bootstrap.operation_id + )) + .exists()) +} diff --git a/container/crowdb-monitor/src/node/runtime/services/test_support.rs b/container/crowdb-monitor/src/node/runtime/services/test_support.rs index da898a879..fec2ae88b 100644 --- a/container/crowdb-monitor/src/node/runtime/services/test_support.rs +++ b/container/crowdb-monitor/src/node/runtime/services/test_support.rs @@ -72,6 +72,16 @@ impl NodeServiceForTests { .await } + /// # Errors + /// Propagates owned test KV child startup failures. + pub async fn start_kv_child_for_tests(&mut self) -> Result { + let service = super::super::service(&self.profile, "kv")?; + Ok(self + .processes + .start(&service, &std::collections::BTreeMap::new()) + .await?) + } + /// # Errors /// Propagates failure to stop the owned KV child. pub async fn stop_kv_for_tests(&mut self) -> Result<()> { diff --git a/container/crowdb-monitor/src/node/runtime/setup/execution.rs b/container/crowdb-monitor/src/node/runtime/setup/execution.rs index d239cf1af..d4c42e1d6 100644 --- a/container/crowdb-monitor/src/node/runtime/setup/execution.rs +++ b/container/crowdb-monitor/src/node/runtime/setup/execution.rs @@ -79,15 +79,7 @@ async fn advance(profile: &DeploymentProfile, state: &mut AcceptedSetup) -> Resu state.status.phase = NodeSetupPhase::Running; journal::save(root, state)?; let choices = &state.request.choices; - let preparation_needed = !matches!( - choices.storage, - crowdb_protocol::mgmt::node::setup::SetupStorage::Diskless - ) || choices.services.iter().any(|service| { - // The admitted KV process already supplies Group 0. Keeping that - // prerequisite enabled does not request ordinary storage metadata. - service.enabled && service.kind != crowdb_protocol::mgmt::node::setup::SetupServiceKind::PaxosKv - }); - if preparation_needed + if choices.requires_default_preparation() && crate::node::read_node_policy(root)?.is_some() && !metadata::ready(root, &binding, &kv).await? { diff --git a/container/crowdb-monitor/tests/common/admitted_member.rs b/container/crowdb-monitor/tests/common/admitted_member.rs index 1c6cad06a..2ce7abc4a 100644 --- a/container/crowdb-monitor/tests/common/admitted_member.rs +++ b/container/crowdb-monitor/tests/common/admitted_member.rs @@ -22,7 +22,7 @@ pub fn fixture(root: &Path) -> ConfirmedMemberTrust { let nodes: Vec<_> = (1_u64..=3).map(|id| { config.nodes.push(serde_json::from_value(json!({"id":id,"rack_id":1,"host":format!("192.0.2.{id}"),"ssh_user":"crowdb"})).unwrap()); config.servers.push(serde_json::from_value(json!({"id":format!("kv-{id}"),"node_id":id,"url":format!("http://192.0.2.{id}:10000"),"rpc_url":format!("192.0.2.{id}:10100")})).unwrap()); - NodeRecord { discovery_id: uuid::Uuid::new_v4().to_string(), node_id:id, physical_host_id:format!("host-{id}"), + NodeRecord { web_port: 9090, discovery_id: uuid::Uuid::new_v4().to_string(), node_id:id, physical_host_id:format!("host-{id}"), rack_id:1, host:format!("192.0.2.{id}"), ssh_port:22, ssh_user:"crowdb".into(), operation_id:uuid::Uuid::new_v4().to_string(), confirmed:false, cancelled:false, host_key:Some(super::trust::key(u8::try_from(id).unwrap())) } @@ -62,6 +62,7 @@ pub fn fixture(root: &Path) -> ConfirmedMemberTrust { node.confirmed = true; } confirmed.push(NodeRecord { + web_port: 9090, discovery_id: own.discovery_id, node_id: 4, physical_host_id: "host-4".into(), diff --git a/container/crowdb-monitor/tests/common/member_trust.rs b/container/crowdb-monitor/tests/common/member_trust.rs index 304be3256..6ffff6c9d 100644 --- a/container/crowdb-monitor/tests/common/member_trust.rs +++ b/container/crowdb-monitor/tests/common/member_trust.rs @@ -44,6 +44,7 @@ pub fn fixture(root: &Path) -> MemberTrust { ); config.servers.push(serde_json::from_value(json!({"id":format!("kv-{id}"),"node_id":id,"url":format!("http://192.0.2.{id}:10000"),"rpc_url":format!("192.0.2.{id}:10100")})).unwrap()); nodes.push(NodeRecord { + web_port: 9090, discovery_id: ids[usize::try_from(id - 1).unwrap()].clone(), node_id: id, physical_host_id: "test-host".into(), diff --git a/container/crowdb-monitor/tests/common/node_setup.rs b/container/crowdb-monitor/tests/common/node_setup.rs index d8d7de409..31f8a27e4 100644 --- a/container/crowdb-monitor/tests/common/node_setup.rs +++ b/container/crowdb-monitor/tests/common/node_setup.rs @@ -54,6 +54,7 @@ impl TestAuthority { let registry = NodeRegistry { cluster_id: binding.bootstrap.cluster_id.clone(), nodes: vec![NodeRecord { + web_port: 9090, discovery_id: request.discovery_id.clone(), operation_id: request.operation_id.clone(), node_id: 2, diff --git a/container/crowdb-monitor/tests/common/service_receipt.rs b/container/crowdb-monitor/tests/common/service_receipt.rs new file mode 100644 index 000000000..fb32ac200 --- /dev/null +++ b/container/crowdb-monitor/tests/common/service_receipt.rs @@ -0,0 +1,107 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::node_setup; +use crowdb_monitor::{DeploymentProfile, NodeIdentity, NodeServiceForTests}; +use crowdb_protocol::mgmt::node::{ + setup::{NodeSetupChoices, NodeSetupRequest}, + NodeServiceAction, NodeServiceIntent, +}; +use crowdb_test_harness::test_dirs::TestDir; +use std::{ + collections::BTreeMap, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, +}; + +pub struct TestStoppedKv { + pub directory: TestDir, + pub worker: NodeServiceForTests, + pub intent: NodeServiceIntent, +} + +impl TestStoppedKv { + pub async fn new() -> Self { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("completed-kv-stop"); + let root = directory.path(); + let request = NodeSetupRequest { + version: 1, + discovery_id: NodeIdentity::load_or_create(root).unwrap().uuid().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + choices: NodeSetupChoices::default(), + }; + let authority = node_setup::TestAuthority::start(root, &request).await; + assert!(authority.registry.nodes[0].confirmed); + std::fs::write( + root.join("accepted-node.json"), + serde_json::to_vec(&serde_json::json!({ + "node_id":authority.binding.node_id,"bootstrap":authority.binding.bootstrap,"prepared":true, + })) + .unwrap(), + ) + .unwrap(); + std::fs::set_permissions( + root.join("accepted-node.json"), + std::fs::Permissions::from_mode(0o600), + ) + .unwrap(); + let intent = NodeServiceIntent { + cluster_id: authority.binding.bootstrap.cluster_id.clone(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: authority.binding.node_id, + service_id: "kv-2".into(), + kind: "kv".into(), + action: NodeServiceAction::Stop, + configuration: "node_id = 2\n".into(), + environment: BTreeMap::new(), + }; + authority + .kv + .put( + 0, + 0, + b"/deployment/services/2/kv-2", + &serde_json::to_vec(&intent).unwrap(), + None, + ) + .await + .unwrap(); + let mut worker = NodeServiceForTests::new_for_tests(profile(root)).await.unwrap(); + let pid = worker.start_kv_child_for_tests().await.unwrap(); + let first = worker.execute_for_tests(intent.clone()).await.unwrap(); + assert!(first["pid"].is_null()); + assert!( + !Path::new(&format!("/proc/{pid}")).exists(), + "the acknowledged stop must reap its owned child" + ); + drop(authority); + Self { + directory, + worker, + intent, + } + } + + pub fn state(&self) -> PathBuf { + self.directory.path().join("services/kv-2/execution.json") + } +} + +fn profile(root: &Path) -> DeploymentProfile { + let mut profile = + DeploymentProfile::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml")) + .unwrap(); + profile.paths.data_root = root.to_path_buf(); + profile.paths.log_root = root.join("log"); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + let program = root.join("owned-kv"); + std::fs::write(&program, "#!/bin/sh\nexec sleep 60\n").unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap(); + profile + .services + .iter_mut() + .find(|s| s.id == "kv") + .unwrap() + .program = program; + profile +} diff --git a/container/crowdb-monitor/tests/initialization_test.rs b/container/crowdb-monitor/tests/initialization_test.rs index f3edb1d25..0efbd20d8 100644 --- a/container/crowdb-monitor/tests/initialization_test.rs +++ b/container/crowdb-monitor/tests/initialization_test.rs @@ -32,6 +32,7 @@ fn fixture(root: &Path) -> (DeploymentProfile, PreparedBootstrap, NodeServiceCre "server": [{"id": "kv-1", "node_id": 1, "url": "http://127.0.0.1:10000", "rpc_url": "127.0.0.1:10100"}] })).unwrap(); let node = NodeRecord { + web_port: 9090, discovery_id: identity.uuid().to_string(), node_id: 1, rack_id: 1, diff --git a/container/crowdb-monitor/tests/member_discovery_test.rs b/container/crowdb-monitor/tests/member_discovery_test.rs new file mode 100644 index 000000000..88155c7f9 --- /dev/null +++ b/container/crowdb-monitor/tests/member_discovery_test.rs @@ -0,0 +1,93 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/member_trust.rs"] +mod trust; + +#[test] +fn pinned_retained_contacts_are_probed_when_multicast_discovery_is_empty() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("member-recovery-contacts"); + let trust = trust::fixture(root.path()); + let operation = crowdb_console_shared::deployment::PreparedBootstrap::load( + &root.path().join("prepared-bootstrap.json"), + ) + .unwrap(); + let mut config = operation.intent.to_config(); + for node in &mut config.nodes { + node.ssh_port = 2200 + u16::try_from(node.id).unwrap(); + node.ssh_password = Some("discarded-test-password".into()); + } + let mut foreign = config.nodes[0].clone(); + foreign.id = 99; + foreign.host = "192.0.2.99".into(); + config.nodes.push(foreign); + let probes = crowdb_monitor::member_recovery_destinations_for_tests( + root.path(), + &config, + &trust, + crowdb_protocol::mgmt::node::CandidateSnapshot::default(), + ) + .unwrap(); + assert_eq!(probes.iter().map(|node| node.id).collect::>(), [1, 2, 3]); + for node in probes { + assert_eq!(node.host, format!("192.0.2.{}", node.id)); + assert_eq!(node.ssh_port, 2200 + u16::try_from(node.id).unwrap()); + assert!(node.ssh_password.is_none()); + assert_eq!( + node.ssh_key, + Some(root.path().join("ssh/id_ed25519").to_string_lossy().into_owned()) + ); + } +} + +#[test] +fn retained_contacts_do_not_crowd_a_changed_address_out_of_the_probe_budget() { + use crowdb_protocol::mgmt::node::{CandidateNode, CandidateSnapshot, CandidateState, NodeAdvertisement}; + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("member-recovery-probe-budget"); + let mut trust = trust::fixture(root.path()); + let operation = crowdb_console_shared::deployment::PreparedBootstrap::load( + &root.path().join("prepared-bootstrap.json"), + ) + .unwrap(); + let mut config = operation.intent.to_config(); + let template_node = config.nodes[0].clone(); + let template_member = trust.members[0].clone(); + config.nodes.clear(); + trust.members.clear(); + for id in 1_u8..=64 { + let mut node = template_node.clone(); + node.id = u64::from(id); + node.host = format!("192.0.2.{id}"); + config.nodes.push(node); + let mut member = template_member.clone(); + member.node_id = u64::from(id); + member.discovery_id = uuid::Uuid::new_v4().to_string(); + member.admission_operation_id = uuid::Uuid::new_v4().to_string(); + member.host_key = trust::key(id); + trust.members.push(member); + } + trust.validate().unwrap(); + let snapshot = CandidateSnapshot { + nodes: vec![CandidateNode { + advertisement: NodeAdvertisement { + discovery_id: trust.members[63].discovery_id.clone(), + protocol_version: 1, + monitor_endpoints: vec!["http://203.0.113.64:9095".into()], + cluster_id: Some(trust.bootstrap.cluster_id.clone()), + formation: None, + }, + state: CandidateState::SameCluster, + }], + discovery_diagnostics: Vec::new(), + }; + let probes = + crowdb_monitor::member_recovery_destinations_for_tests(root.path(), &config, &trust, snapshot) + .unwrap(); + assert_eq!(probes.len(), 64); + assert!( + probes + .iter() + .any(|node| node.id == 64 && node.host == "203.0.113.64"), + "a full retained-address cache must not suppress a member's discovered new address" + ); +} diff --git a/container/crowdb-monitor/tests/node_cleanup_test.rs b/container/crowdb-monitor/tests/node_cleanup_test.rs index 40c979cae..14e37871b 100644 --- a/container/crowdb-monitor/tests/node_cleanup_test.rs +++ b/container/crowdb-monitor/tests/node_cleanup_test.rs @@ -2,7 +2,10 @@ // Licensed under the Apache License, Version 2.0. use crowdb_monitor::{accept_node_policy, cleanup_node_for_tests, DeploymentProfile, ProcessManager}; -use crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides; +use crowdb_protocol::mgmt::node::{ + deployment::{NodeDeploymentOverrides, NodeDeploymentPolicy}, + setup::SetupServiceKind, +}; use crowdb_protocol::mgmt::SystemBootstrapIdentity; use crowdb_test_harness::test_dirs::tempdir_in_test_data; use std::{os::unix::fs::PermissionsExt, path::Path}; @@ -12,7 +15,18 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; async fn retired_cleanup_restarts_its_local_worker_after_a_failed_deletion() { let directory = tempdir_in_test_data("monitor-retired-cleanup"); let root = directory.path(); - let defaults = NodeDeploymentOverrides::default(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut services = NodeDeploymentPolicy::default().setup.services; + services + .iter_mut() + .find(|service| service.kind == SetupServiceKind::PaxosKv) + .unwrap() + .listeners + .http_port = Some(listener.local_addr().unwrap().port()); + let defaults = NodeDeploymentOverrides { + services: Some(services), + ..Default::default() + }; accept_node_policy(root, &defaults, &defaults).unwrap(); let bootstrap = SystemBootstrapIdentity { cluster_id: uuid::Uuid::new_v4().to_string(), @@ -65,7 +79,6 @@ async fn retired_cleanup_restarts_its_local_worker_after_a_failed_deletion() { assert!(!retired.exists()); assert!(!manager.owns("kv")); - let listener = tokio::net::TcpListener::bind("127.0.0.1:10000").await.unwrap(); let endpoint = tokio::spawn(serve_deletions(listener, bootstrap.clone(), retired.clone())); assert!( cleanup_node_for_tests(&profile, &mut manager, bootstrap.clone(), true) diff --git a/container/crowdb-monitor/tests/node_run_inputs_test.rs b/container/crowdb-monitor/tests/node_run_inputs_test.rs index d53b66e50..d3f62fc6b 100644 --- a/container/crowdb-monitor/tests/node_run_inputs_test.rs +++ b/container/crowdb-monitor/tests/node_run_inputs_test.rs @@ -139,7 +139,7 @@ fn invalid_voter_override_fails_before_node_identity_and_storage_effects() { .output() .unwrap(); assert!(!output.status.success()); - assert!(String::from_utf8_lossy(&output.stderr).contains("initial voters must be one or three")); + assert!(String::from_utf8_lossy(&output.stderr).contains("initial voters must be one, three or five")); assert_eq!(fs::read_dir(root.path().join("install/data")).unwrap().count(), 0); } diff --git a/container/crowdb-monitor/tests/node_service_receipt_test.rs b/container/crowdb-monitor/tests/node_service_receipt_test.rs new file mode 100644 index 000000000..f06e553ac --- /dev/null +++ b/container/crowdb-monitor/tests/node_service_receipt_test.rs @@ -0,0 +1,109 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/node_setup.rs"] +mod node_setup; +#[path = "common/service_receipt.rs"] +mod service_receipt; + +use service_receipt::TestStoppedKv; +use std::time::Duration; + +#[tokio::test] +async fn completed_kv_stop_can_report_its_receipt_after_authority_loss() { + let mut test = TestStoppedKv::new().await; + let before = std::fs::read(test.state()).unwrap(); + let receipt = tokio::time::timeout( + Duration::from_millis(200), + test.worker.execute_for_tests(test.intent.clone()), + ) + .await + .expect("reporting a completed stop must not wait for the authority it stopped") + .unwrap(); + assert_eq!(receipt["operation_id"], test.intent.operation_id); + assert!(receipt["pid"].is_null()); + assert_eq!( + std::fs::read(test.state()).unwrap(), + before, + "a receipt must not repeat execution" + ); +} + +#[tokio::test] +async fn incomplete_foreign_and_legacy_receipts_cannot_authorize_an_offline_operation() { + let mut test = TestStoppedKv::new().await; + let original: serde_json::Value = serde_json::from_slice(&std::fs::read(test.state()).unwrap()).unwrap(); + for fault in [ + "legacy", + "foreign-bootstrap", + "incomplete", + "pid", + "newer-operation", + ] { + let mut record = original.clone(); + match fault { + "legacy" => { + record.as_object_mut().unwrap().remove("bootstrap"); + } + "foreign-bootstrap" => { + record["bootstrap"]["operation_id"] = uuid::Uuid::new_v4().to_string().into(); + } + "incomplete" => record["complete"] = false.into(), + "pid" => record["pid"] = 123.into(), + "newer-operation" => record["intent"]["operation_id"] = uuid::Uuid::new_v4().to_string().into(), + _ => unreachable!(), + } + let bytes = serde_json::to_vec(&record).unwrap(); + std::fs::write(test.state(), &bytes).unwrap(); + let result = tokio::time::timeout( + Duration::from_millis(200), + test.worker.execute_for_tests(test.intent.clone()), + ) + .await; + assert!( + !matches!(result, Ok(Ok(_))), + "invalid {fault} receipt acknowledged without authority" + ); + assert_eq!(std::fs::read(test.state()).unwrap(), bytes); + } +} + +#[tokio::test] +async fn a_completed_receipt_cannot_stop_a_live_replacement_without_authority() { + let mut test = TestStoppedKv::new().await; + let pid = test.worker.start_kv_child_for_tests().await.unwrap(); + let result = tokio::time::timeout( + Duration::from_millis(200), + test.worker.execute_for_tests(test.intent.clone()), + ) + .await; + assert!( + !matches!(result, Ok(Ok(_))), + "a live replacement requires current authorization" + ); + assert!(std::path::Path::new(&format!("/proc/{pid}")).exists()); + test.worker.stop_kv_for_tests().await.unwrap(); +} + +#[tokio::test] +async fn retired_bootstrap_does_not_report_a_completed_receipt() { + let mut test = TestStoppedKv::new().await; + let record: serde_json::Value = serde_json::from_slice(&std::fs::read(test.state()).unwrap()).unwrap(); + let operation = record["bootstrap"]["operation_id"].as_str().unwrap(); + std::fs::write( + test.directory + .path() + .join(format!("retired-bootstrap-{operation}.json")), + b"{}", + ) + .unwrap(); + let result = tokio::time::timeout( + Duration::from_millis(200), + test.worker.execute_for_tests(test.intent.clone()), + ) + .await; + assert!( + !matches!(result, Ok(Ok(_))), + "retired local authority cannot serve the receipt" + ); +} diff --git a/doc/design/chunkdb/design-crowdb-chunkdb.md b/doc/design/chunkdb/design-crowdb-chunkdb.md index 2746d0f2e..f8d04e00c 100644 --- a/doc/design/chunkdb/design-crowdb-chunkdb.md +++ b/doc/design/chunkdb/design-crowdb-chunkdb.md @@ -477,6 +477,12 @@ Site (data center) - healthy physical-disk locations, allocatable capacity observations, and a monotonic publication generation. +Located DiskIO registrations overlay hardware intent with current placement +availability. A configured disk group is eligible only while a matching live +process reports it opened. Retained expired registrations exclude stopped +physical owners; expiry is evaluated at selection time even without a refresh. +This does not change durable hardware status or relax failure-domain constraints. + Usage observations include only allocatable disks and carry their sample time. Missing or stale observations remove capacity ranking information; they never relax a rack, node, or disk safety constraint. Per-disk-group in-flight bytes diff --git a/doc/design/chunkds/design-crowdb-chunk-stream.md b/doc/design/chunkds/design-crowdb-chunk-stream.md index 2d47b4822..42664a0e3 100644 --- a/doc/design/chunkds/design-crowdb-chunk-stream.md +++ b/doc/design/chunkds/design-crowdb-chunk-stream.md @@ -364,6 +364,13 @@ Only the last referenced chunk may require content parsing. Earlier chunks have finalized bounds and chunk record mappings and are not scanned. A last chunk whose seal and final mappings are already published needs no tail scan. Recovery after an interrupted seal may still reconstruct that one last chunk. +The ChunkDB finalizer may reclaim and tombstone a physically empty tail before +the Stream publishes its successor. Recovery accepts that exact tombstone only +with its seal timestamp, valid purpose and nonzero writer epoch, zero capacity, +zero sealed/acknowledged/checkpoint boundaries, and no remaining strips or cleanup +intents. It retains the logical position, leaves the tombstone unchanged, and +attaches a fresh chunk under current writer authority. A missing chunk or a +nonempty or unproven tombstone never becomes an empty prefix. Bound reads, buffered bytes, parsed frames, and total scan bytes by the active chunk geometry, independently of retained Stream length. diff --git a/doc/design/deploy/design-crowdb-deploy.md b/doc/design/deploy/design-crowdb-deploy.md index caad22017..425f8a3ad 100644 --- a/doc/design/deploy/design-crowdb-deploy.md +++ b/doc/design/deploy/design-crowdb-deploy.md @@ -96,6 +96,11 @@ numeric node ID through Group 0 and durably records the mapping. Allocation and mapping publication are conditional and idempotent under concurrent admission; IDs cannot collide or be silently reassigned. Endpoint and rack changes do not change either identity. Duplicate live UUID claims are identity conflicts. +Confirmed admission retains the Web listener proved over authenticated SSH. +Cluster observation uses that listener for each UUID, including containers with +shared host addresses and different port offsets. Retained default-port records +keep their existing serialized identity; each monitor republishes its own current +listener through the existing confirmed-node update operation. ## 3. Invariants @@ -255,7 +260,7 @@ requires an explicit recovery/decommission procedure, not an automatic timeout. 1. Any UI collects compatible candidates and the operator's rack assignments. 2. The operator selects initial Group-0 voting members, or Auto join selects - the explicitly selected one or three authenticated eligible candidates in + the explicitly selected one, three or five authenticated eligible candidates in discovery-UUID order. Candidate count is not voting-member count; application nodes need not all vote. 3. Submit the bootstrap identity, immutable initial manifest and initial data @@ -311,16 +316,18 @@ Production requires configured rack assignments and access. Missing Production credentials remain Needs access, without a Local fallback. Multiple bound cluster UUIDs require explicit target selection rather than an arbitrary advertisement. -The accepted initial voter count is one or three. An incomplete three-node set -remains pending; an observed overfull fresh set reports conflict before acceptance. +The accepted initial voter count is one, three or five. An incomplete selected +set remains pending; an observed overfull fresh set reports conflict before acceptance. Eligible UUID ordering selects the coordinator within that set, while monitor and KV-server accepted-manifest fencing prevent conflicting execution. Fixed voters never expand from later advertisements. Turning Auto join off stops new scheduling while already accepted monitor work continues. After Group-0 publication, one committed default preparation intent owns the -ordinary metadata group, replica mapping and shared access credentials. Authorized -workers resume that intent across Web/coordinator loss. Each monitor interleaves +ordinary metadata group, replica mapping and shared access credentials. The +replica mapping covers every fixed initial voting member; each receives its own +initial storage binding. Authorized workers resume that intent across +Web/coordinator loss. Each monitor interleaves disk preparation with dependent service startup; DiskIO may start before Online because its ownership proof enables Online. Explicit later service stops or group retirement supersede historical setup and prevent automatic recreation. @@ -395,8 +402,12 @@ trust or automatic key replacement for an existing identity. Recovery proceeds independently of Group-0 availability: -1. mDNS/DNS-SD or a configured seed reveals a candidate's current management - endpoint. Its advertised UUIDs and node ID are untrusted lookup hints. +1. mDNS/DNS-SD, a configured seed or retained accepted contact supplies a + candidate management address. Retained contacts remain probe hints when + multicast or Group 0 is unavailable. Advertised identities and addresses + never authorize a route without fresh proof. Fresh discovery hints receive + the bounded probe budget before retained addresses, so a full old-address + cache cannot suppress a member's changed IP. 2. The monitor selects an already accepted member binding and opens SSH to that address with the expected host key, rather than trusting the IP's previous occupant or accepting a new key. SSH proves possession of the private key. @@ -423,6 +434,12 @@ Recovery proceeds independently of Group-0 availability: conditional new service operations; later stop/delete commands win. Public client URIs are independent of these internal addresses. +The contact worker checks current authority before probing peers so retirement +still takes precedence. Confirmation and later address publication share a +three-second authority budget; SSH proofs retain their ten-second bound. A +failed authority lookup must not add another full publication wait and expire +otherwise valid routes at their fifteen-second freshness fence. + This permits all members to restart with changed IPs at once. Neither stable DNS names nor fixed IPs nor a complete peer-address list is required. Multicast must reach the selected management scope; across discovery boundaries, seed discovery @@ -578,6 +595,16 @@ process and before starting its replacement. A superseded operation or unavailab authority rejects execution. Recovery reads current desired intents through the same authority path and does not execute a retained command solely from disk. +An exact completed KV Stop/Delete can return its retained receipt while that +child remains stopped. The receipt carries the bootstrap identity under which +the operation was authorized; it must match the current accepted local binding +and complete intent, with no live replacement, cancellation or retirement. +This reports a prior effect without executing another one. It must not require +the Group-0 leader that the operation already stopped: background recovery may +finish the intent before its identical SSH request reaches the executor. +Legacy receipts without that bootstrap proof still require current authority. +Incomplete, superseded and new operations retain every authority fence above. + Admission commands similarly match the current cluster publication and UUID, numeric ID, operation ID and cancellation state. Local durable cancellation and retirement markers reject delayed commands after cleanup or restart. Long-running diff --git a/doc/design/kv/design-crowdb-kv-slot.md b/doc/design/kv/design-crowdb-kv-slot.md index da3130fc4..29b010a89 100644 --- a/doc/design/kv/design-crowdb-kv-slot.md +++ b/doc/design/kv/design-crowdb-kv-slot.md @@ -199,22 +199,44 @@ The `ballot_round` field in the ChosenNotice flatbuffer enables this verificatio When a follower detects a gap (missing or stale slot in the chosen range), it sends a unary `FetchGap(slot)` request to the leader. The leader: -- Has the chosen value locally → replies with the full entry (payload + ballot + term). -- Does not have it → runs classic Paxos (`repair_once`, §9) to resolve the slot, then replies with the resolved value (or NoOp). - -On receipt, the follower overwrites any stale accepted value with the chosen value, updates its chosen frontier, and wakes the apply loop. FetchGap is bounded by `MAX_INFLIGHT_FETCHGAP` (default 16) to avoid flooding the leader. +- Has completed leader recovery and retains the value within its contiguous + chosen prefix → replies with the full entry (payload + ballot + term). +- Otherwise → returns pending/not found. Normal leader recovery or independent + gap repair must establish the chosen value before a later request can fetch it. + +On receipt, a durable follower first writes the original Accepted entry to its +WAL, then installs it, advances the chosen frontier and wakes the apply loop. +Historical entries retain their original election term; a newer local term +does not reject committed history. Failed persistence leaves the gap pending. +Learning a proven chosen value does not answer a new Accept request and must +not discard a later promise. It retains the original entry alongside that +higher promise; an already higher accepted ballot must carry the same value. +Replay rebuilds highest accepted values before restoring promise fences, so +the physical order of historical fetch and later promise records cannot lose +either durable fact. Accepted history alone is never a FetchGap chosen proof. +`MAX_INFLIGHT_FETCHGAP` (default 16) bounds the entire fetch/persist/install +operation, including error and cancellation paths. This replaces the previous leader-driven catch-up that ran inline in `run_heartbeat_round` (up to 64 synchronous `send_accept().await` calls per lagging follower). That inline catch-up could exceed `heartbeat_interval` and trigger spurious elections. The heartbeat round is now pure liveness + lease: one RPC round-trip, no catch-up work. ### 9A.4 Snapshot Fallback -If a follower's gap count exceeds `catchup_snapshot_threshold` (default `bulk_prepare_window` = 1024), the follower stops issuing FetchGap requests and logs a warning. The full snapshot-install path for running replicas is deferred; the threshold gate prevents FetchGap storms against the leader when a follower is severely lagging (e.g. after a long network partition). +If a follower's gap count exceeds `catchup_snapshot_threshold` (default 1024), +it records the large backlog and continues FetchGap under the same fixed +16-request limit. A missing automatic snapshot path must not disable catch-up. The bounded snapshot protocol is used only while bootstrapping a fresh, unpublished member. It does not replace the engine of a running follower and -is not invoked by this threshold. A live replacement requires a separate +is not invoked by this diagnostic threshold. A live replacement requires a separate atomic engine-generation publication design. +A recovered durable engine's gap-free resume watermark remains authoritative +when WAL reclamation removed the boundary record or all preceding segments. +Restore retains that complete chosen/applied prefix without capping it to the +remaining WAL tip. If the exact boundary term is unavailable, its historical +term is reported as zero (unknown); voting freshness still uses the durable +acceptor log tip. Higher retained entries replay after the snapshot prefix. + ### 9A.5 Apply Loop The follower's apply loop targets `max(known_commit_slot, last_chosen_slot)`. For each slot in the apply range: @@ -315,19 +337,29 @@ See [WAL garbage collection](design-crowdb-kv-wal.md#7-garbage-collection). ## 12. Tunables and Defaults -| Parameter | Default | Where it lives | -| --- | --- | --- | -| `max_inflight_proposals` | 32 | `PaxosConfig` (total semaphore permits) | -| `reject_on_inflight_window_full` | `false` | `PaxosConfig` (internal, not CLI-exposed) | -| `coalesce_max_keys` | 32 | `PaxosConfig` (max ops per coalesced batch; 0 = off) | -| `coalesce_drain_threshold` | 1 | `PaxosConfig` (skip drain at this inflight count) | -| `max_paxos_retries` | 3 | `PaxosConfig` (per-slot Phase-2 retries) | -| `max_slot_retries` | 3 | `PaxosConfig` (new-slot retries before giving up) | -| `retry_base_backoff_ms` | 5 | `PaxosConfig` (exponential backoff base) | -| `bulk_prepare_window` | 1024 | `PxElectionConfig` (bulk Phase-1 batch size) | -| `catchup_snapshot_threshold` | 1024 | `PxElectionConfig` (gap count above which FetchGap is skipped in favor of snapshot fallback) | - -The bulk Phase-1 typically resolves open gaps in a single RTT. Steady-state gaps are closed one-at-a-time after each heartbeat round, so worst-case gap-clear time is bounded by `heartbeat_interval × gap_count`. Follower-side gaps (missing or stale chosen slots) are closed by FetchGap at the follower's own pace, bounded by `MAX_INFLIGHT_FETCHGAP` (default 16) concurrent requests. +| Parameter | Default | Where it lives | +| -------------------------------- | ------- | --------------------------------------------------------------------- | +| `max_inflight_proposals` | 32 | `PaxosConfig` (total semaphore permits) | +| `reject_on_inflight_window_full` | `false` | `PaxosConfig` (internal, not CLI-exposed) | +| `coalesce_max_keys` | 32 | `PaxosConfig` (max ops per coalesced batch; 0 = off) | +| `coalesce_drain_threshold` | 1 | `PaxosConfig` (skip drain at this inflight count) | +| `max_paxos_retries` | 3 | `PaxosConfig` (per-slot Phase-2 retries) | +| `max_slot_retries` | 3 | `PaxosConfig` (new-slot retries before giving up) | +| `retry_base_backoff_ms` | 5 | `PaxosConfig` (exponential backoff base) | +| `bulk_prepare_window` | 1024 | `PxElectionConfig` (bulk Phase-1 batch size) | +| `catchup_snapshot_threshold` | 1024 | `PxElectionConfig` (diagnostic threshold; bounded FetchGap continues) | + +Bulk Phase 1 recovers independent slots concurrently, with at most +`bulk_prepare_window` complete Prepare/Accept operations in flight. It starts +at the new leader's own contiguous frontier, adopts quorum values and opens +reads only after the entire observed prefix has resolved and applied. Tenure +cancellation prevents prepared slots from proceeding to Accept. + +Steady-state gap repair starts only after initial recovery has completed; +the two paths never compete for the same slot. Each background pass drains +its observed backlog independently of heartbeat cadence. Follower-side gaps +use at most `MAX_INFLIGHT_FETCHGAP` (default 16) concurrent requests, including +backlogs above the snapshot diagnostic threshold. --- diff --git a/doc/working/plan-two-physical-nodes.md b/doc/working/plan-two-physical-nodes.md deleted file mode 100644 index b8c538477..000000000 --- a/doc/working/plan-two-physical-nodes.md +++ /dev/null @@ -1,174 +0,0 @@ - - - -# Two Physical Node Validation Plan - -Upstream: [deployment design](../design/deploy/design-crowdb-deploy.md), -[OCI image guide](../design/deploy/design-crowdb-oci-image.md), -[KV quorum design](../design/kv/design-crowdb-kv-group0.md), -[Chunk-KV recovery design](../design/chunkds/design-crowdb-chunk-kv-server.md). - -Goal: validate discovery, key-only admission, host-network composition, -service readiness, bounded data operations and recovery on two real hosts. -Results and receipts: [track-two-physical-nodes.md](track-two-physical-nodes.md). -The physical three-container matrix and requested CI selections pass. The plan -remains open for authorized publication and the five-container physical follow-up. - -## Fixture and constraints - -- Host A: `u24-amd`, `192.168.31.129`; host B: `cj7960`, `192.168.31.184`. -- Final placement: two logical voters on A, one on B; host-network offsets - `0/1` on A and `0` on B. Each node owns a separate labeled volume. -- All remote work now uses `cj` and `id_ed25519`, key-only SSH. Existing - authorized keys are preserved; passwords are absent from run artifacts. -- Build and execute through `pixi run`. Record immutable OCI receipts and - compare execution configuration plus ordered rootfs layers on both hosts. -- Never alter shared firewall, Docker, NIC, routes or kernel settings. - Stop/remove only resources verified to carry the exact run label. -- Preserve the first failing operation and complete logs. Do not lengthen - deadlines or add caller retries to turn a failure into a pass. - -## Completed prerequisites and baseline - -- [x] **Inventory and reachability**: capture hosts, interfaces, Docker versions, - resources, endpoint ownership and cross-host SSH/TCP reachability. -- [x] **Repair host SSH**: install the existing public key on B, preserve other - keys, verify permissions and key-only `BatchMode` login. -- [x] **Compose offset nodes**: verify all canonical listener blocks and - offset-resolved setup choices. Local offsets `0/1/2` reached active Group 0. - Files: `container/oci-image/run-node.py`, - `container/single-node-container/entrypoint.sh`, - `lib/crowdb-protocol/src/mgmt/node/deployment.rs`, - `container/crowdb-monitor/src/node/startup/policy.rs`. -- [x] **Persist accepted admission choices**: use the target-resolved choices - for endpoint publication when the request omits setup. Files: - `app/crowdb-web/src/node/{admission,setup}.rs`. -- [x] **Separate DiskDB listener blocks**: support explicit main/HTTP/RPC ports, - shift each once, retain custom legacy base behavior, and prove adjacent - instances disjoint. Files: protocol node setup/validation/deployment, - Web DiskDB lifecycle/configuration and planned deployment, monitor policy. -- [x] **Exact mixed-placement image and bootstrap**: import the verified image - on both hosts, admit all three nodes without passwords, initialize - three-voter Group 0, provision production disk media, and reach seven - healthy services per node. Files: attempt-12 receipts and snapshots. -- [x] **Bounded baseline corpus**: KV put/get/batch, 96 KiB S3 object, - Iceberg metadata, and a small mirrored ChunkDB allocation; cross-host exact - reads. Files: attempt-12 corpus and native-client evidence. -- [x] **Retained-volume recreation**: recreate all three nodes with the - corrected image and original identities, without a password-file mount. - -## Recovery verification and remaining work - -- [x] **Managed listener recovery**: fault one owned Chunk-KV child, require a - changed PID within the existing bound, all services ready and cross-host - corpus reads. Strengthen Access startup to probe every catalog owner before - another launch. Files: monitor service dependencies and regression tests. -- [x] **Allocation retry contract regression**: use real RPC responses to prove - known `Unavailable` retries the same chunk ID and a lost reply submits once. - File: `app/crowdb-chunkdb/tests/allocation_retry_test.rs`. -- [x] **Observe remote loss**: stop B, verify KV quorum write/read, retain the - S3 `ServiceUnavailable` result and failed 180-second catalog failover wait, - then restore B. This completed observation is a failed availability case. -- [x] **Diagnose recovery defects**: reproduce the unfinished owner-2-to-owner-1 - `TargetPreparing` transfer and explain why it cannot finish; inspect the - planner's busy-owner exclusion and the target preparation dependency. - DiskDB stale-owner discovery and released-source WAL/grant fixes pass focused - regressions. Attempt 15 also reproduced empty sealed-WAL recovery rejection; - the production adapter regression passes. The upgrade then exposed reclaimed-journal recovery losing live DiskDB occupancy; KV and DiskDB restart regressions now pass. Preserve the corrupted attempt-15 evidence and run a fresh dynamic-ownership matrix after rebuilding. Files: Chunk-KV transition worker/monitor, DiskDB ownership lifecycle, - and focused regressions selected from first divergence evidence. -- [x] **Repeat physical allocation retry**: attempt 15 retained three voters, - injected Group-1-only leadership unavailability, observed the explicit - allocation-level `Unavailable` retry, and completed same-ID allocation/query - within the original five-retry/10-second limit. The KV processes and Group 0 - stayed live. Earlier process-pause attempts are separate failed observations. -- [x] **Repeat remote storage recovery**: after repairing the defects, stop B, - require bounded range recovery and one-shot S3/Iceberg reads, recreate B with - its volume/key-only admission, and recheck identities and exact data. -- [x] **Repeat expected two-voter host loss**: after preceding recovery passes, - stop both A nodes, record expected Group-0 unavailability, restore them and - verify all services and durable data. Earlier-image evidence is historical. -- [x] **Pass final health/data/allocation gate**: require stable healthy PIDs, - existing KV/S3/Iceberg reads, a fresh batch and fresh allocation/query. - Attempt 12 passed restored health, existing reads and batch, but failed fresh - allocation with `disk-group not owned`. - -## Evidence and cleanup - -- [x] **Freeze diagnostics**: retain status, inspect, policies, receipts, - transitions, workload errors and complete logs. Capture final log archives - after container stop to avoid concurrent-write tar failures. -- [x] **Clean exact owned resources**: remove attempt-12 through attempt-16 containers and volumes, - delete its remote transfer archives, verify all owned endpoints closed, - retain artifacts outside deleted volumes and retain host-B SSH access. -- [x] **Report actual coverage**: distinguish passes, failed recovery cases, - earlier-image observations and the physical retry coverage gap. - -## Verification groups - -- Protocol: node deployment/setup contract tests and full crate tests. -- Web: DiskDB deployment configuration tests and full crate tests. -- Monitor: node policy, service dependencies and service readiness tests. -- ChunkDB: allocation retry tests and dynamic retry tests. -- Quality: Rust fmt, affected crates' clippy with warnings denied, launcher - Python tests and shell syntax; immutable OCI build for production changes. -- Physical: repeat only the affected matrix after the first failures are - repaired, preserving successful baseline evidence and default limits. - -## After physical validation: requested CI jobs - -- [x] **Fetch requested CI evidence**: retain all eight job logs and map the - exact failed cases in run 41003611925 after completing physical attempt 16. -- [x] **Repair retained Monitor fixtures**: accept valid startup policy before - identity, resume and cleanup operations. The exact two failures pass; the - earlier passing Monitor targets and the remaining targets are retained. - Files: `container/crowdb-monitor/tests/{kv_resume,node_cleanup}_test.rs`. -- [x] **Repair common Console/bootstrap policy**: the exact inspection case - reproduces container exit due to disabled Paxos KV, followed by no route to - host. Keep mandatory KV listeners enabled while skipping storage setup; - verify the original case and affected Console/deployment selections. - Files: `e2e-test/profiles/setup_choices.py`, native policy, bridge-node and - reference-recovery fixtures. -- [x] **Preserve live services during readiness recovery**: native inspection - passed serially after bootstrap repair but a concurrent-load run exposed a - monitor restart during normal transfer readiness withdrawal. Recorded HTTP - evidence distinguishes `/ready` 503 from `/health` 200. Keep readiness for - admission/status and liveness for restart decisions; preserve dead-listener - restart assertions and verify the new image with the affected case. - Files: monitor runtime health/probes and service readiness regression. -- [x] **Verify Go SDK failure cleanup**: focused and complete S3 selections - pass. Retain safe operation/error diagnostics; the original opaque CI - cleanup cause cannot be established from its artifact. -- [x] **Skip ordinary metadata for diskless KV-only setup**: the mandatory - Group-0 prerequisite must not reserve a default Group 1. Reproduce the - metadata race, retain both disabled/KV-only unit coverage, repair the - preparation predicate, rebuild and resume the stopped native-data group. -- [x] **Resolve remote wildcard topology addresses**: HTTP Console topology - reports must inherit the reporting server management host before entering - the shared cache. Reproduce false loopback Group-0 authority loss after - stopping a local follower, reuse the existing endpoint resolver, add an - HTTP/cache regression and rerun the affected lifecycle case. -- [x] **Publish canonical protocol listeners with retained-input compatibility**: - repair Pixi/CLI/Compose mappings and image metadata, retain version-1 launch - ports while sealing fresh version-2 inputs, verify mapping/receipt regressions - and CLI quality gates; the image rebuild is complete before reference verification. -- [x] **Complete requested Console partitions**: browser, native-data, - native-operations, transition and inspection, with original coverage and - fail-fast progression. Keep browser fixture networks isolated from other jobs. -- [x] **Complete deployment reference**: rerun its original case set after the - shared bootstrap repair, then diagnose any remaining failure. -- [x] **Close AccessTests and quality gates**: run the requested Access component - and SDK commands once, fmt, affected clippy and required C++ lint before the - first coherent commit/push. -- Job IDs: `123893403740`, `123899615935`, `123899616023`, `123899615992`, - `123899616153`, `123899616076`, `123899616075`, `123899615882`. -- Run: https://github.com/buzzcrow/crowdb/actions/runs/41003611925 - -## Authorized publication and five-container follow-up - -- [~] **First publication**: user authorized a single coherent commit and push - after the physical matrix, requested CI repairs and relevant gates pass. -- [ ] **Five-container physical matrix**: after that push, run a fresh cluster on the same two physical - hosts, retaining immutable image receipts, cross-host data witnesses, fault - and recovery results, and exact owned-resource cleanup. -- [ ] **Final publication**: repair failures and verify affected groups, then commit and push the - resulting five-node validation changes and evidence summary a second time. diff --git a/doc/working/track-two-physical-nodes.md b/doc/working/track-two-physical-nodes.md index f6b20a2e3..c65a80654 100644 --- a/doc/working/track-two-physical-nodes.md +++ b/doc/working/track-two-physical-nodes.md @@ -3,14 +3,19 @@ # Two Physical Node Test Track -Plan: [plan-two-physical-nodes.md](plan-two-physical-nodes.md) - Run ID: `two-physical-nodes-20261030T010205Z`. -Latest fixture: `physical-20261030-r16`; result: **physical recovery matrix passed**. -SSH access is repaired. All owned physical fixtures are cleaned up. Attempt 15 -forensic metadata/WAL, raw mirror reads and complete logs remain archived; -attempt 16 passed using fresh test data and the retained preflight policy -volumes. Historical failures remain recorded below. +Latest fixture: `physical-20261030-r17`; result: **five-container recovery matrix +passed across two physical hosts**, including all five replicas' acknowledged +Group-0/1/3 checkpoints. All owned physical fixtures are cleaned up and key-only +SSH works. First publication is `25059cf7d`; the follow-up CI repairs pass local +verification and are prepared for the authorized second publication. Historical +CI runs remain failed; the next published commit requires its own remote result. + +The separate 512-record/default-five-second capacity probe failed; 294 confirmed +records are exact recovery witnesses, not a capacity pass. User disk cleanup +deleted the earlier local archives, including attempt 15 forensic files. Later +sections describe that loss and the surviving/reconstructed evidence explicitly; +historical archive descriptions below refer to their state before that cleanup. ## Attempt 12 placement and image (historical) @@ -438,3 +443,333 @@ Root: `.crowdb-runtime/artifacts/two-physical-nodes-20261030T010205Z/`. both host-loss cases and owned-resource cleanup. - Verify repairs and commit/push the five-node changes and evidence summary a second time. The execution plan remains active until then. + +## Five-container physical continuation + +- First publication completed: commit + `25059cf7d9a5f08d61479f4749659aa6f3fe119f` pushed to `task-fix-build`. + PR #55 CI run `41128190276` completed: 22 jobs passed; five follow-up + failures are recorded below and remain part of this task. +- Selected five initial voters across two physical hosts: A offsets `0/1/2`, + B offsets `0/1`, with a separate volume and seven services per container. +- New regressions reproduced three entry restrictions: accepted voter count, + default preparation membership and bound-member conflict handling. Policy now + accepts only `1/3/5`; preparation maps every admitted voter to its own initial + storage binding; all multi-node scopes retain the existing formation fences. + Policy, all nine preparation tests and all nine formation tests pass. +- Full affected tests/clippy and a fresh image precede physical deployment. + The physical matrix will include 512 exact 64-KiB ChunkKV records, all five + serving owners, cross-host KV/S3/Iceberg reads, retained-volume recreation, + unresponsive-child recovery, remote-leader KV stop/start, both host-loss cases, + bounded allocation retry and a final no-overlapping-allocation scan. + +- Full affected Protocol, Console shared, Web and Monitor tests, all-target + clippy and workspace formatting pass. One Monitor CLI test retained the old + invalid-voter error text; it now asserts the `1/3/5` contract and retains all + no-identity/no-storage side-effect checks. +- The first remote CI run also exposed `g3_leader_change::leader_change_simulation`: + the second one-shot write returned failure. Its original artifact omitted the + response, so the precise original reply remains unknown. The focused case, + a complete 131-case group run and twenty logged repetitions pass before the + helper repair. A deterministic regression then proves that the fixture's + role-only wait can return before a current quorum lease exists. The helper + now waits within the same three-second bound for current proposing tenure + and quorum lease; writes remain one-shot and report the complete response on + failure. The three affected cases, complete `test-core` gate and KV clippy + pass. Production Paxos logic is unchanged. Evidence: + `ci/run-41128190276/core-verification.json`, `ci-leader-ready-{red,green}.log`. + + +### Attempt 17: five voters on two physical hosts + +- Immutable image `0ef175e40bef7a152fda9abff2929f41df3830d39bedf555f04de53776dcab51` + is verified on both hosts. All five Group-0/default-metadata/manual-data voters, + 35 service health observations and KV/S3/Iceberg witnesses pass initially. +- The additional 512-record capacity case failed its original five-second batch + deadline during split/cutover. A full read-only audit found records 0–293 exact + (19,267,584 bytes), with 294–511 absent. No timed-out mutation was replayed. + Preserve this failed capacity case separately from the 294-record recovery + witness. No process OOM or PID replacement occurred during the capacity failure. +- All-five retained-volume recreation passes with exact identities and witness + bytes. SIGSTOP ChunkKV is replaced within 48.3 seconds; the other 34 PIDs and + all witnesses are retained. Cross-host local-KV stop/start passes after fixing + the harness to accept an omitted stopped-service row as well as a null PID. +- Stopping both B containers preserves the manual-data KV quorum write, but + catalog recovery fails its original 180-second bound. WAL recovery drains + schedule repair, then new active WAL initialization still selects stopped + physical IO targets. This remains a failed case until repaired and rerun. +- Whole-cluster readiness also exposes a fixed-9090 lookup and a `1/3` reducer + restriction. Repair accepted Web-listener publication and five-voter reduction; + verify the original volume set after upgrading the image. +- First-push CI exposed four additional groups: KV-only readiness collection, + stale CLI Access ports, journal selection during normal owner transfer, and GC + foreground overlap. Keep the original assertions/deadlines and verify each + group before final publication. Browser trace and GC operation timestamps are + retained under `ci/run-41128190276/`. + +- Repaired-image round 1 preserved the original volumes but exposed finalized + empty WAL tails: the current Stream record still referenced a zero-boundary + chunk that the ChunkDB finalizer had legitimately sealed, reclaimed and + tombstoned. Exact records under `upgrade-round1/` show `Deleted`, a seal time, + zero capacity/cursors and no remaining strips. Recovery previously accepted + only `Sealed` for that empty case. Both a cold adapter and complete writer + reopen regression reproduce the error, then pass with exact finalization + proof. Reopen preserves the tombstone and allocates a fresh chunk; eleven + invalid/unproven states remain rejected. Full Stream and ChunkKV server tests + pass; quality gates and the second image upgrade precede resumed faults. +- The same round confirms the independent cluster-observation repair: all five + UUIDs resolve to the accepted per-host `9090/9091/9092` listeners, with no + invalid-voter or foreign-member reasons. Remaining service reasons correctly + report the blocked WAL recovery. See `upgrade-round1/partial-cluster-status.json`. + +- Round 2 image `cde2868ade7b3437c27f01c67c7eaf407a02c15b53b874e580c04d0611d1b1ea` + restores all original volumes, 35 services, all five cluster-readiness views + and exact witnesses. B-host loss now passes Catalog takeover, original data, + and one-shot S3/Iceberg writes. Both B containers recreate with retained + identities and all five nodes verify the outage writes and 294 exact records. +- The subsequent majority-host loss correctly rejects writes, but restoration + fails the original 180-second bound. KV role/lease health returns while new + leaders' applied frontiers remain behind. `upgrade-round2/` preserves the + exact failure, current per-replica health and complete logs. They show serial + bulk Phase 1 competing with background repair, plus follower FetchGap stopping + permanently above the 1024-gap snapshot threshold despite having no automatic + running-replica snapshot replacement. Two regressions reproduce these paths. + Bounded concurrent recovery, mutually exclusive initial/background repair and + continued 16-request FetchGap pass three regressions, including cancellation + before Accept. Full core gates and a rebuilt image precede the resumed matrix. + +- The complete `test-core` gate, KV all-target clippy and workspace formatting + pass after the backlog repair. Evidence: `ci/run-41128190276/backlog-gates.json`. + +- During round 3 the user deleted `.crowdb-runtime` for disk cleanup. Historical + artifact archives, including the earlier frozen corruption records, are no + longer available. Surviving `/tmp` logs/scripts were copied back into the same + runtime artifact root, as requested; current state and complete service logs + were recaptured from the five original Docker volumes. The cleanup also + interrupted artifact publication after the already-failed restoration wait. + Do not treat reconstructed expectations or new captures as recovered archives. +- Round 3 image `64454951913004b00d5d8dad480e8ddeb65088ce34cb427b64c3d0269ec48a71` + again passes original-volume upgrade and B-host loss/recreation. Majority loss + rejects writes, but restoration still fails the original 180-second bound. + Further regressions show FetchGap failing to persist recovered entries and + restart discarding a valid durable snapshot prefix when its WAL boundary is + absent. Both failures reproduce before their fixes, then focused restart and + durable-tree regressions pass. Recovery also needs to probe retained pinned + contacts when multicast is absent; an empty-discovery regression reproduces + the missing probes. All route publication continues to require pinned SSH + identity/bootstrap proof. Complete gates and another image precede resumption. + +- After the durable catch-up and retained-contact fixes, complete `test-core`, + Monitor tests, both crates' all-target clippy and workspace formatting pass. + Confirmation/publication share their existing three-second authority budget, + keeping offline SSH proof refresh below the fifteen-second route fence. + Evidence: `ci/run-41128190276/durable-gates.json`. + +- Round 4 image `2989a3a0dc7729126f005330ee58f1e701a87d3696164ce750a259d874755f03` + restores all 35 services, five cluster-readiness views and exact witnesses. + The stronger per-replica check then fails its 180-second bound: B replicas + remain at Group-0 applied slots 77005 and 34771 while A advances beyond 140000. + The matrix stops before new fault injection. Current receipts, checkpoints and + complete logs are retained under attempt 17's `round4/upgrade-failure/`. +- A real-RPC regression reproduces historical chosen replay being rejected by + a higher promise; another proves FetchGap incorrectly exposed accepted-only + values. Both fail before the repair. Learning now preserves higher promises, + WAL replay retains accepted history independently of promise record order, + and the source requires a recovered contiguous chosen proof. Focused tests + pass; complete core gates and another retained-volume upgrade remain pending. +- An actual zero-ballot probe confirms both blocked boundary slots retain a + higher promise (`(1,4)` / `(1,5)`) than the source's historical chosen ballot + `(0,3)`. Both probes are rejected without changing those promises. The + retained receipt, probe source and complete response are under `round4/`. +- The repaired FetchGap cases preserve exact nonempty values and higher + promises through WAL restart. The existing RPC migration fixture now + publishes its actual two-voter quorum decision before fetching; response + assertions remain intact. Complete core coverage passes through its retained + prefix and resumed remainder; affected all-target clippy and formatting pass. + Evidence: `ci/run-41128190276/promise-resumed-gates.json` and + `ci-fetch-gap-promise-{red,green}.log`. +- Round 5 image `51d0a67638ee9102664710eb95711dae3d201d066c09e44bb8a3aa1ae7e9996d` + restores all services, five readiness views and exact data. All five replicas + reach the fixed Group-0/1/3 checkpoints within the original 180-second bound; + the Group-0 check completes in 110.3 seconds after both B replicas had been + tens of thousands of slots behind. Per-second checkpoint observations and + immutable receipts accompany the retained-volume upgrade. Host faults follow. +- Review found that 64 retained address hints could consume the whole probe + budget and exclude a discovered changed IP. A 64-member regression fails + before prioritizing fresh hints, then both discovery regressions and Monitor + clippy pass. The five-member physical fixture probes the same complete set + in either order; the next CI image includes this budget-boundary repair. +- Round 5 B-host loss/recreation passes in 338.6 seconds for the whole + sequence: surviving quorum write/read, Catalog takeover within 180 seconds, + new unique acknowledged KV/S3/Iceberg witnesses, exact original reads, + retained identity recreation of both B containers and all-five reads. + Majority-host loss, allocation fault, final audit and cleanup remain pending. +- Majority-host loss/restoration passes in 175.3 seconds for the whole sequence: + the two surviving voters reject the write, then all 35 services, all-five + replica checkpoints and exact witnesses recover within their original bounds. +- The first allocation fault observation failed a harness race: its wait ended + on the first follower's no-op reply before the leader's accepted reply arrived. + It now waits for actual acceptance within the same three-second bound and + persists every reply. The resumed case observes explicit allocation-only safe + retry, completes same-ID allocation/query in 3.12 seconds, keeps every KV PID + and verifies Group-0 lease availability during the fault. Earlier passes are + retained; final fresh-outage writes, allocation audit and cleanup follow. +- The resumed final gate passes: a second unique outage witness, all-five exact + reads, stable service PIDs, fresh native batch/allocation and all Group-0/1/3 + applied checkpoints. The final scan finds 1774 live mirrored segments with + no cross-chunk physical overlap. All five labeled containers/volumes are + removed, all 85 owned endpoints are closed, and key-only host SSH still works. + Final frozen service logs were archived after stopping each container, with + no concurrent-write tar caveat. The separate 512-record capacity deadline + failure remains a measured limit; 294 exact records are the recovery witness. + Evidence: `attempt17/repaired-matrix-sequence.json`, `final-closed/`, + `final-all-replicas-caught-up.json`, `physical-five-occupancy.log`, `cleanup.json`. +- Complete storage tests and formatting pass after the physical matrix. The + current isolated Journal browser case passes in 3268 ms; its original ordered + native-operations group then stops at Shell's KV-stop command. Three earlier + cases pass and are retained. Whole-workspace clippy and all 277 framework + tests pass before the newly diagnosed stop-receipt repair. +- Shell's trace shows Stop requested at `00:45:35.108Z`, KV exiting about + 0.1 seconds later, and the identical SSH request timing out after another + three-second authority read. The target had been Group-0 leader; its durable + execution record already says complete. The isolated Shell case passes in + 4.3 seconds, then a real-authority/owned-child regression reproduces receipt + failure after authority loss. Exact completed KV-stop receipts now carry + bootstrap proof and return without re-execution; legacy, foreign, incomplete, + retired and live-replacement states still require current authority. + +### Stop-receipt and settled-Journal verification + +- Full Monitor tests, affected Monitor/E2E clippy, the 277 fixture tests, + bundled native-browser type checks, workspace fmt and required `tree-lint` + pass. The C++ gate exited zero with existing warnings in unchanged sources. +- The release image containing the completed-stop receipt fix built in 139.7 s; + Docker execution identity is + `sha256:fea066994ab45918a04335fe93b00044aebb3fe73b958278806536a90aa05cba`. +- Exact Shell recovery passes: Playwright 6.5 s, initial election 3385 ms; + the stopped leader's duplicate operation returns its immutable receipt. +- Exact Journal recovery passes: placement settles in 502 ms before the browser, + Playwright 4.6 s, owner restart response 2290 ms. The measured pre-change + browser baseline was 3.268 s; the new time is below twice that baseline. + All navigation, identity, owner-loss/restart, stale-cursor and cleanup assertions + retain their original deadlines. +- Evidence: `ci/run-41128190276/receipt-fixture-gates.json` and its named logs. + The initial direct-source TypeScript invocation lacked the serializer snapshot + supplied by `bundle_client`; the passing check uses that same snapshot layout. +- Continue the native-operations selection in its original relative order: + retain unchanged binding/CRUD passes and rerun the KV-stop lifecycle case, + failed Shell case and all fifteen previously unrun cases (17 remaining). + +### Complete native operations and explicit KV-only witness setup + +- The native-operations closing run actually executed all original 19 cases, + despite the launcher-only remainder filter described above. All 19 passed; + `native-operations-complete-coverage.json` validates the exact original case + set against `results.json`. `native-resume-orchestration-note.json` preserves + the planned/actual selection difference. The resume helper now filters both + launcher and executor selections; no assertions or resources changed. +- The deployment-services closing selection stopped at + `deployment::kv_bootstrap::monitor_creates_real_kv_groups_then_validates_after_restart`: + `/stores/0/groups/1/ready` returned 404 (`group 1 not found in store 0`). Seven + earlier/in-flight cases passed and remain valid. +- This KV-only fixture must explicitly request its ordinary witness Group 1: + `/system/prepare` accepts bootstrap ownership; it does not create that group. + Both KV-only witness tests now assert no default-preparation record and no + existing Group 1, create the group once through the production management API, + and await its leader under the existing management four-second setup bound. + Group-0/1 identity, readiness, foreign-bootstrap rejection, exact data and + retained initialization checks remain. Restart does not recreate the group. +- Repair sequence: affected E2E clippy/fmt, both exact KV-only cases, then the + 24 unrun deployment cases in original relative order. Combined with the seven + valid passes and two focused cases, this preserves all 33 deployment cases. + +### Sustained terminal-writer fault versus automatic DiskIO recovery + +- Console API stopped at `console::cli_container::terminal_chunk_writer_recovery`. + The correct health endpoints returned fresh `ready: true` after the nominal + forty-second pause. Six independent cases passed, including the formerly + failing plain Docker attachment; retain them for the resumed selection. +- Monitor lifecycle logs prove the fault healed before the thirty-second I/O + deadline: DiskIO PIDs 1414/594/631 were replaced by 2344/1522/1515 around + 19–27 seconds after fault submission. The healthy responses were not stale. +- The injector now holds each verified owned Monitor while pausing DiskIO, + verifies all six original processes remain stopped after forty seconds, + and resumes DiskIO before Monitor. Production recovery policy, original fault + duration, health assertions, terminal-replay proof and restart/data checks + remain intact. No host-wide settings change. +- An `ExitStack` restores every paused process even if another resume fails; + client reaping still runs on failure. Two cleanup regressions and all 279 + fixture tests pass. The focused terminal-writer test is running. +- Deployment's complete 33-case union and Native Operations' exact 19-case set + are verified by the named `*-complete-coverage.json` records. + +### Replace supervisor suspension with an owned network fault + +- Holding Monitor also suspends its child-log pumps. That attempted injector + produced health-listener curl timeouts while all six PIDs remained stopped; + it is a failed observation, not a pass. Preserve `terminal-fault-held-focused.log` + and `run-fd6b4633ab9c4f0e981d5a62da2387fa` for that first divergence. +- Use a forty-second packet drop on each accepted DiskIO TCP listener instead. + The helper joins only an authenticated owned CLI bridge container's network + namespace, receives only NET_ADMIN, and mounts a fingerprinted host netfilter + tool snapshot read-only. It never joins the host namespace or edits host rules. + Existing Monitor, log pumping and health samplers continue during the outage; + automatic DiskIO replacement cannot heal the blocked listener early. +- A disposable private-namespace probe passed add/check/delete of the exact + DROP rule using the current release image. ExitStack cleanup handles each + jump, rule and chain independently, including partial setup and a failed + removal; focused unit coverage also rejects host-network targets before any + helper execution. The original health/currentness, terminal writer restart, + unchanged Access PIDs, exact data and operation-fencing assertions remain. + +### Bound the sustained GC load at its original cadence + +- Native Operations 19/19, deployment-services 33/33 and Console API 92/92 + complete coverage is verified. GC cleanup's six cases passed; convergence + failed at 96/128 files after otherwise monotonic revisions. +- The convergence failure is an explained process exit: the sole Chunk-KV + container has `OOMKilled=true`, exit 137, with its unchanged 512 MiB limit. + Four zero-delay SDK workers created 45,828 table cycles before that exit, + versus the original minimum 48. Keep the frozen state and counters in + `gc-zero-delay-oom-proof.json`; do not count this overload as a passing run. +- Retain the original 500 ms inter-table interval while running until cleanup + completes. Stagger four workers by quarter-cycle offsets so 250 ms deletion + observations do not repeatedly alias all workers' gaps. Sleeps stay outside + latency measurements; the minimum counts, real deletion overlap, 64 small + writes and original test deadlines remain. The separate zero-delay foreground + probe keeps its existing zero-delay/yield behavior. +- Repeat convergence and the restart matrix with that shared pressure change. + The five other cleanup cases retain valid passes because their load inputs + are unchanged. Preserve the complete failed long-run evidence. + +### Closing verification and publication + +- Both paced GC repetitions pass: restart matrix 132.54 s; convergence 479.44 s + including fixture lifecycle, with 471.46 s client execution. The convergence + task reaches `Complete`, deletes all 128 source files and verifies their 1554 + logical bytes. The original 64 foreground writes, minimum SDK operation counts + and measured overlap with actual deletion intervals all pass. The failed + zero-delay overload remains separate evidence. +- Complete affected container coverage is 151 distinct cases: Native Operations + 19, deployment-services 33, Console API 92, GC cleanup 6 and GC convergence 1. + Their `*-complete-coverage.json` records identify the actual result files and + distinguish retained unchanged passes from repeated changed cases. The two + shared GC pressure consumers were both repeated after restoring the cadence. +- Core, storage, Protocol, Web, Monitor, Console Shared, KV Client, ChunkDB and + Chunk Stream verification pass, along with workspace fmt, whole-workspace and + subsequent affected-crate clippy, native-browser type checks, all 280 fixture + tests and required C++ tree lint. The final post-pressure E2E clippy/fmt and + framework checks pass. No production changes followed the final release-image + build; later changes are tests, fixture tooling, CI prerequisites and docs. +- Rechecked run `41003611925`: eight failed jobs, repaired and locally verified + before the first publication. Rechecked run `41128190276`: five failed jobs, + repaired and verified by the follow-up coverage above. The historical job + conclusions are retained; local passes do not change those remote conclusions. +- The user renewed commit/push authorization after temporarily deferring push. + The completed execution plan is removed for this publication. Cleanup-tool + changes belonging to the user's concurrent work are excluded from the commit. +- Physical scope remains five containers across two hosts, with complete + recovery/cleanup results. The separate 512-record/default-five-second capacity + failure is unresolved as a capacity limit; the 294 acknowledged records are + verified recovery witnesses only. Earlier archives deleted during user disk + cleanup cannot be recovered or claimed as retained. diff --git a/e2e-test/README.md b/e2e-test/README.md index 5aae427c7..3c957ac21 100644 --- a/e2e-test/README.md +++ b/e2e-test/README.md @@ -15,6 +15,12 @@ Test clients run on the host against inspected node-container IPs and fixed ports, with three node containers for a three-node cluster. Fresh one- through eight-node profiles exercise management, membership, async operations and multi-store routing. Host bridge IP reachability is required; remote Docker and Linux-VM routing are not implicitly supported. +The CLI terminal-writer fault also requires host `iptables-nft`, its shared +libraries and xtables extensions. A short-lived release-image helper receives +NET_ADMIN only inside each verified owned CLI bridge namespace and drops the +accepted DiskIO listener for forty seconds. Monitor and logging keep running; +the helper never enters the host network namespace. Tool fingerprints, rule +verification and failure-safe rule cleanup remain in the case artifacts. Containers supply server subsets or clusters from the formal release image; they are not test execution environments. Domain profiles configure that same server artifact rather than introducing separate test-server builds. diff --git a/e2e-test/clients/console-ui/e2e/flows/01-shell-ui-behaviors.spec.ts b/e2e-test/clients/console-ui/e2e/flows/01-shell-ui-behaviors.spec.ts index 263847bbf..f420d8fa5 100644 --- a/e2e-test/clients/console-ui/e2e/flows/01-shell-ui-behaviors.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/01-shell-ui-behaviors.spec.ts @@ -4,10 +4,10 @@ import { test, expect } from '../fixtures/realBackend'; import { diskMenu } from '../fixtures/capacity'; import { step } from '../fixtures/stepTimer'; -// Baseline: 6.0s (2026-10-15). +// Baseline: 6.5s (2026-10-31). test('shell defaults and tree interactions distinguish live and stopped release nodes', async ({ page, request }) => { test.setTimeout(90_000); - const stopped = await request.post('/api/nodes/4/server/stop'); + const stopped = await step('shell KV stop response', () => request.post('/api/nodes/4/server/stop')); expect(stopped.ok(), await stopped.text()).toBe(true); await expect.poll(async () => { const response = await request.get('/api/node/status'); @@ -21,14 +21,14 @@ test('shell defaults and tree interactions distinguish live and stopped release return nodes.filter((node: { id: number; kv_server?: { health: string } }) => [1, 2, 3].includes(node.id) && node.kv_server?.health === 'up').length; }, { timeout: 10_000, intervals: [100] }).toBe(3); - const store = await request.post('/api/stores', { data: { store_id: 207, nodes: [1, 2] } }); + const store = await step('shell Store mutation response', () => request.post('/api/stores', { data: { store_id: 207, nodes: [1, 2] } })); expect(store.status(), await store.text()).toBe(201); const diskInventoryRequests: number[] = []; page.on('request', request => { const match = new URL(request.url()).pathname.match(/^\/api\/nodes\/(\d+)\/disk-groups$/); if (match) diskInventoryRequests.push(Number(match[1])); }); - await page.goto('/?domain=KV'); + await step('shell DOM setup', () => page.goto('/?domain=KV')); const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); await sidebar.getByRole('button', { name: 'datacenter', exact: true }).click({ button: 'right' }); await page.getByRole('menuitem', { name: 'Add Store', exact: true }).click(); diff --git a/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts b/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts index e8d89e8c2..bbef568a2 100644 --- a/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts @@ -1,21 +1,24 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -// Baseline: partition/overlay 1.4s, unavailable 0.208s, journal 0.534s, placement 0.321s (2026-10-03). +// Baseline: journal 4.6s (2026-10-31); partition/overlay 1.4s, unavailable 0.208s, placement 0.321s (2026-10-03). import { test, expect } from '../fixtures/realBackend'; +import { step } from '../fixtures/stepTimer'; import type { APIRequestContext, APIResponse, Page, Response } from '@playwright/test'; import type { CatalogPage, Partition } from '../../src/chunk-kv/catalog'; test('native diagnostics: journal identity survives Chunk navigation and owner interruption', async ({ page, request }) => { - const bucket = `journal-${process.pid}-${Date.now()}`; + const bucket = process.env.CROWDB_NATIVE_JOURNAL_BUCKET; + if (!bucket) throw new Error("Journal diagnostics require the settled owned object"); const object = `/api/access/s3/${bucket}/value`; try { - expect((await request.put(`/api/access/s3/${bucket}`)).ok()).toBe(true); - expect((await request.put(object, { data: 'native journal observation' })).ok()).toBe(true); + const witness = await step('journal fixture witness', () => request.get(object)); + expect(witness.ok(), await witness.text()).toBe(true); + expect(await witness.text()).toBe('native journal observation'); const catalogObservation = page.waitForResponse(response => new URL(response.url()).pathname === '/api/chunk-kv/catalog'); - await page.goto('/?domain=Chunk-KV'); + await step('journal DOM setup', () => page.goto('/?domain=Chunk-KV')); const response = await catalogObservation; expect(response.ok(), await response.text()).toBe(true); - const { partition, observed } = await selectActiveJournal(page, request, await response.json()); + const { partition, observed } = await step('journal initial lifecycle', async () => selectActiveJournal(page, request, await response.json())); const runtime = await observed.json(); expect(runtime.journal.chunk_record).not.toBeNull(); await page.getByRole('tab', { name: 'Journal', exact: true }).click(); @@ -36,26 +39,28 @@ test('native diagnostics: journal identity survives Chunk navigation and owner i await page.getByRole('button', { name: 'Refresh catalog', exact: true }).click(); const currentResponse = await currentMap; expect(currentResponse.ok(), await currentResponse.text()).toBe(true); - const current = await selectActiveJournal(page, request, await currentResponse.json(), partition.id); + const current = await step('journal current owner lifecycle', async () => selectActiveJournal(page, request, await currentResponse.json(), partition.id)); expect(current.partition.id).toBe(partition.id); await page.getByRole('tab', { name: 'Journal', exact: true }).click(); await expect(page.getByRole('region', { name: 'Partition runtime', exact: true }).getByRole('button', { name: 'Refresh runtime', exact: true })).toBeEnabled(); const owner = `chunk-kv-${current.partition.owner_id}`; - expect((await request.post(`/api/services/${owner}/stop`, { data: {} })).ok()).toBe(true); + expect((await step('journal owner stop response', () => request.post(`/api/services/${owner}/stop`, { data: {} }))).ok()).toBe(true); try { const observation = page.getByRole('region', { name: 'Partition runtime', exact: true }); - await observation.getByRole('button', { name: 'Refresh runtime', exact: true }).click(); - await expect(observation).toContainText('Runtime unavailable'); - await expect(journal.getByLabel('Stream chunk record', { exact: true })).toHaveCount(0); - await expect(journal.getByRole('button', { name: 'Inspect Chunk', exact: true })).toHaveCount(0); + await step('journal owner loss DOM refresh', async () => { + await observation.getByRole('button', { name: 'Refresh runtime', exact: true }).click(); + await expect(observation).toContainText('Runtime unavailable'); + await expect(journal.getByLabel('Stream chunk record', { exact: true })).toHaveCount(0); + await expect(journal.getByRole('button', { name: 'Inspect Chunk', exact: true })).toHaveCount(0); + }); } finally { - expect((await request.post(`/api/services/${owner}/restart`, { data: {} })).ok()).toBe(true); + expect((await step('journal owner restart response', () => request.post(`/api/services/${owner}/restart`, { data: {} }))).ok()).toBe(true); } const refreshedMap = page.waitForResponse(response => new URL(response.url()).pathname === '/api/chunk-kv/catalog'); await page.getByRole('button', { name: 'Refresh catalog', exact: true }).click(); const mapResponse = await refreshedMap; expect(mapResponse.ok(), await mapResponse.text()).toBe(true); - const restored = await selectActiveJournal(page, request, await mapResponse.json(), partition.id); + const restored = await step('journal restored owner lifecycle', async () => selectActiveJournal(page, request, await mapResponse.json(), partition.id)); expect(restored.partition.id).toBe(partition.id); await page.getByRole('tab', { name: 'Journal', exact: true }).click(); const observation = page.getByRole('region', { name: 'Partition runtime', exact: true }); @@ -67,9 +72,9 @@ test('native diagnostics: journal identity survives Chunk navigation and owner i expect((await request.get(stale.toString())).status()).toBe(409); } finally { try { - expect((await request.delete(object)).status()).toBe(204); + expect((await step('journal object teardown', () => request.delete(object))).status()).toBe(204); } finally { - expect((await request.delete(`/api/access/s3/${bucket}`)).status()).toBe(204); + expect((await step('journal bucket teardown', () => request.delete(`/api/access/s3/${bucket}`))).status()).toBe(204); } } }); @@ -180,7 +185,7 @@ async function selectActiveJournal(page: Page, request: APIRequestContext, initi let selected: { partition: Partition; observed: Response } | null = null; const refresh = async (action: () => Promise) => { const observation = page.waitForResponse(response => new URL(response.url()).pathname === '/api/chunk-kv/catalog'); - await action(); + await step('journal catalog refresh response', action); const response = await observation; expect(response.ok(), await response.text()).toBe(true); catalog = await response.json(); @@ -206,7 +211,7 @@ async function selectActiveJournal(page: Page, request: APIRequestContext, initi } if (entry.state !== 'Serving') continue; const query = new URLSearchParams({ id: entry.id, epoch: entry.epoch, generation: catalog.generation, page: String(catalog.page), offset: String(catalog.offset) }); - const response = await request.get(`/api/chunk-kv/runtime?${query}`); + const response = await step('journal runtime API response', () => request.get(`/api/chunk-kv/runtime?${query}`)); if (await changed(response)) return false; expect(response.ok(), await response.text()).toBe(true); if (!(await response.json()).journal.chunk_record) continue; @@ -214,8 +219,10 @@ async function selectActiveJournal(page: Page, request: APIRequestContext, initi // Selecting the same partition leaves its identity unchanged. Explicitly // refresh runtime so owner restart is observed even without a new catalog. const observation = page.waitForResponse(response => new URL(response.url()).pathname === '/api/chunk-kv/runtime'); - await page.getByRole('region', { name: 'Partition runtime', exact: true }).getByRole('button', { name: 'Refresh runtime', exact: true }).click(); - const observed = await observation; + const observed = await step('journal runtime DOM refresh', async () => { + await page.getByRole('region', { name: 'Partition runtime', exact: true }).getByRole('button', { name: 'Refresh runtime', exact: true }).click(); + return observation; + }); if (await changed(observed)) return false; expect(observed.ok(), await observed.text()).toBe(true); selected = { partition: entry, observed }; diff --git a/e2e-test/console_cli_client.py b/e2e-test/console_cli_client.py index ef6f578df..fa1339482 100644 --- a/e2e-test/console_cli_client.py +++ b/e2e-test/console_cli_client.py @@ -19,7 +19,8 @@ def diagnostics(self): opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) for index, address in enumerate(self.addresses): try: - with opener.open(f"http://{address}:9091/_crowdb/metrics", timeout=2) as response: + port = self.access_listeners(index)["s3_port"] + with opener.open(f"http://{address}:{port}/_crowdb/metrics", timeout=2) as response: metrics = response.read(1024 * 1024 + 1) if len(metrics) > 1024 * 1024: raise ValueError("Access metrics exceeded the diagnostic limit") diff --git a/e2e-test/console_cli_fixture.py b/e2e-test/console_cli_fixture.py index e6b45eb2f..63b4a8922 100644 --- a/e2e-test/console_cli_fixture.py +++ b/e2e-test/console_cli_fixture.py @@ -203,7 +203,7 @@ def refresh(self): "published_console_ports": [self.port_base + index * 10 for index in range(3)]}, indent=2)) def read_owned(self, path, index=0): - if path not in ("node-binding.json", "diskdb-execution"): + if path not in ("node-binding.json", "node-policy.json", "diskdb-execution"): raise ValueError("unsupported CLI recovery observation") if type(index) is not int or not 0 <= index < len(getattr(self, "servers", ())): raise ValueError("CLI recovery node is outside the owned cluster") @@ -216,6 +216,14 @@ def read_owned(self, path, index=0): path = f"services/diskdb-{node_id}/execution.json" return json.loads(self.command("exec", info["Id"], "cat", "/opt/crowdb/data/" + path)) + def access_listeners(self, index): + services = self.read_owned("node-policy.json", index)["setup"]["services"] + selected = [service["listeners"] for service in services + if service["kind"] == "access-server" and service["enabled"]] + if len(selected) != 1: + raise ValueError("owned node needs one accepted Access listener selection") + return selected[0] + def diagnostics(self): if not (self.root / "persistent/oci-clusters" / f"{self.run_id}.json").exists(): return diff --git a/e2e-test/console_cli_io_fault.py b/e2e-test/console_cli_io_fault.py new file mode 100644 index 000000000..09c9e73f2 --- /dev/null +++ b/e2e-test/console_cli_io_fault.py @@ -0,0 +1,91 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Bounded DiskIO packet loss inside verified CLI-owned network namespaces.""" + +from contextlib import ExitStack +import hashlib +import json +from pathlib import Path +import re +import shutil +import subprocess + + +class DiskIoOutage: + def __init__(self, cluster): + self.cluster = cluster + self.chain = "CROWDB_E2E_" + hashlib.sha256(cluster.run_id.encode()).hexdigest()[:12] + self.nodes = [] + for index, name in enumerate(cluster.servers): + policy = cluster.read_owned("node-policy.json", index) + services = [service for service in policy["setup"]["services"] + if service["kind"] == "diskio" and service["enabled"]] + if len(services) != 1: + raise ValueError("fault needs one accepted DiskIO listener per owned node") + port = services[0]["listeners"]["rpc_port"] + if type(port) is not int or not 0 < port < 65536: + raise ValueError("invalid owned DiskIO listener") + self.nodes.append((name, port)) + if len(self.nodes) != 3: + raise ValueError("terminal writer fault requires three owned nodes") + + def _prepare_tools(self): + binary = shutil.which("iptables-nft") + if not binary: + raise RuntimeError("owned network fault requires the host iptables-nft executable") + self.tools = self.cluster.artifacts / "fault-network-tools" + self.tools.mkdir() + shutil.copy2(binary, self.tools / "iptables-nft") + result = subprocess.run(["ldd", binary], capture_output=True, text=True, check=False, timeout=10) + if result.returncode: + raise RuntimeError(f"netfilter dependency inspection failed: {result.stdout}{result.stderr}") + libraries = [Path(path) for path in re.findall(r"(?:=>\s*|^\s*)(/\S+)", result.stdout, re.M)] + loaders = [path for path in libraries if path.name.startswith("ld-linux")] + extensions = [path.parent / "xtables" for path in libraries if path.name.startswith("libxtables")] + if len(loaders) != 1 or len(extensions) != 1 or not extensions[0].is_dir(): + raise RuntimeError(f"cannot locate the host netfilter loader/extensions: {result.stdout}") + self.loader = loaders[0].name + for library in libraries: + shutil.copy2(library, self.tools / library.name) + shutil.copytree(extensions[0], self.tools / "xtables") + files = {str(path.relative_to(self.tools)): hashlib.sha256(path.read_bytes()).hexdigest() + for path in self.tools.rglob("*") if path.is_file()} + (self.cluster.artifacts / "fault-network-tools.json").write_text(json.dumps(files, indent=2)) + + def _command(self, node, *arguments): + name, _port = node + info = self.cluster.inspect_owned(name) + if info["HostConfig"]["NetworkMode"] != "crowdb-" + self.cluster.run_id: + raise ValueError("fault must remain inside the owned CLI bridge namespace") + root = "/run/crowdb-netfilter" + return self.cluster.command("run", "--rm", "--network", "container:" + info["Id"], + "--cap-drop", "ALL", "--cap-add", "NET_ADMIN", "--memory", "64m", "--pids-limit", "32", + "--cpu-shares", "1024", "--label", "org.crowdb.e2e.run=" + self.cluster.run_id, + "--mount", f"type=bind,source={self.tools.resolve()},target={root},readonly", + "--env", f"XTABLES_LIBDIR={root}/xtables", "--entrypoint", f"{root}/{self.loader}", + self.cluster.image, "--library-path", root, f"{root}/iptables-nft", "-w", "2", *arguments) + + def __enter__(self): + self._prepare_tools() + with ExitStack() as cleanup: + for node in self.nodes: + self._command(node, "-N", self.chain) + cleanup.callback(self._command, node, "-X", self.chain) + self._command(node, "-A", self.chain, "-p", "tcp", "--dport", str(node[1]), "-j", "DROP") + cleanup.callback(self._command, node, "-F", self.chain) + self._command(node, "-I", "INPUT", "1", "-j", self.chain) + cleanup.callback(self._command, node, "-D", "INPUT", "-j", self.chain) + self.cleanup = cleanup.pop_all() + return self + + def verify(self): + proof = [] + for node in self.nodes: + self._command(node, "-C", "INPUT", "-j", self.chain) + self._command(node, "-C", self.chain, "-p", "tcp", "--dport", str(node[1]), "-j", "DROP") + proof.append({"container": self.cluster.inspect_owned(node[0])["Id"], + "diskio_port": node[1], "chain": self.chain, "blocked": True}) + (self.cluster.artifacts / "fault-network-blocked.json").write_text(json.dumps(proof, indent=2)) + + def __exit__(self, *exception): + return self.cleanup.__exit__(*exception) diff --git a/e2e-test/console_cli_recovery.py b/e2e-test/console_cli_recovery.py index 0d1b35c07..dd6808bb7 100644 --- a/e2e-test/console_cli_recovery.py +++ b/e2e-test/console_cli_recovery.py @@ -7,21 +7,15 @@ import time from fixture import request - - -def signal_diskio(cluster, identity, pid, signal): - expected = "/opt/crowdb/bin/crowdb-diskio" - check = f'test "$(readlink /proc/{pid}/exe)" = {expected} && kill -{signal} {pid}' - if signal == "CONT": - check = f"if test -e /proc/{pid}/exe; then {check}; fi" - cluster.command("exec", "--user", "crowdb", identity, "sh", "-c", check) +from console_cli_io_fault import DiskIoOutage def health_snapshot(cluster, label): observed = [] for index, name in enumerate(cluster.servers): identity = cluster.inspect_owned(name)["Id"] - url = f"http://{cluster.addresses[index]}:9094/_crowdb/health/ready" + port = cluster.access_listeners(index)["health_port"] + url = f"http://{cluster.addresses[index]}:{port}/_crowdb/health/ready" try: value = cluster.command("exec", identity, "curl", "--silent", "--show-error", "--max-time", "3", url) @@ -32,27 +26,24 @@ def health_snapshot(cluster, label): return observed -def pause_and_write(cluster, pauses): +def block_and_write(cluster, outage): with (cluster.artifacts / "fault-s3.log").open("w") as output: clients = [] try: - for identity, pid in pauses: - signal_diskio(cluster, identity, pid, "STOP") - status = cluster.command("exec", identity, "cat", f"/proc/{pid}/status") - assert "State:\tT" in status, status - print("Paused three verified DiskIO processes across the 30-second I/O deadline", flush=True) - for number in range(3): - clients.append(subprocess.Popen(cluster.arguments("s3", "object", "put", - "writer-recovery", f"during-{number}", "--text", "during-fault"), - env=cluster.environment(), stdout=output, stderr=output)) - time.sleep(40) - unavailable = health_snapshot(cluster, "paused") - assert all(value.get("ready") is False for value in unavailable), unavailable - assert all(value.get("observation", {}).get("state") == "current" - for value in unavailable), unavailable + with outage: + outage.verify() + print("Blocked three owned DiskIO listeners across the 30-second I/O deadline", flush=True) + for number in range(3): + clients.append(subprocess.Popen(cluster.arguments("s3", "object", "put", + "writer-recovery", f"during-{number}", "--text", "during-fault"), + env=cluster.environment(), stdout=output, stderr=output)) + time.sleep(40) + outage.verify() + unavailable = health_snapshot(cluster, "paused") + assert all(value.get("ready") is False for value in unavailable), unavailable + assert all(value.get("observation", {}).get("state") == "current" + for value in unavailable), unavailable finally: - for identity, pid in pauses: - signal_diskio(cluster, identity, pid, "CONT") for client in clients: try: client.wait(timeout=60) @@ -95,8 +86,9 @@ def terminal_writer_recovery(cluster, cluster_id): assert type(pid) is int and pid > 1, service pauses.append((identities[index], pid)) (cluster.artifacts / "fault-injection.json").write_text(json.dumps({ - "diskio_processes": pauses, "pause_seconds": 40}, indent=2)) - pause_and_write(cluster, pauses) + "diskio_processes": pauses, "fault": "owned-namespace DiskIO packet drop", + "outage_seconds": 40}, indent=2)) + block_and_write(cluster, DiskIoOutage(cluster)) # No application traffic until independent health observation reports recovery. started = time.monotonic() try: diff --git a/e2e-test/console_cli_target.py b/e2e-test/console_cli_target.py index 4f24af1a5..9f6e95ede 100644 --- a/e2e-test/console_cli_target.py +++ b/e2e-test/console_cli_target.py @@ -40,9 +40,9 @@ def start(self): f"CROWDB_CLUSTER={self.run_id}", "CROWDB_INITIAL_VOTERS=3", "CROWDB_DEPLOYMENT_MODE=local", f"CROWDB_PHYSICAL_HOST_ID={host}", - f"CROWDB_S3_PUBLIC_URI=http://{address}:9091", - f"CROWDB_ICEBERG_PUBLIC_URI=http://{address}:9092", - f"CROWDB_DATASET_PUBLIC_URI=http://{address}:9093", + f"CROWDB_S3_PUBLIC_URI=http://{address}:16000", + f"CROWDB_ICEBERG_PUBLIC_URI=http://{address}:17000", + f"CROWDB_DATASET_PUBLIC_URI=http://{address}:18000", "CROWDB_DISCOVERY_SEEDS=" + ",".join(f"http://{peer}:9095" for peer in peers)))) self.origins = [f"http://{address}:10000" for address in self.addresses] for name in self.servers: @@ -62,6 +62,9 @@ def start(self): def endpoint(self, index): return f"http://{self.addresses[index]}:9090" + def access_listeners(self, index): + return {"s3_port": 16000, "health_port": 19000} + def inventory(self): containers = [json.loads(self.command("inspect", name))[0] for name in self.servers] assert all(item["Config"]["Labels"].get(LABEL) == self.run_id for item in containers) diff --git a/e2e-test/console_native_fixture.py b/e2e-test/console_native_fixture.py index abed7d913..4581716de 100644 --- a/e2e-test/console_native_fixture.py +++ b/e2e-test/console_native_fixture.py @@ -3,6 +3,7 @@ """Production-policy release nodes with explicit admission and fixed ports.""" import json +import re import urllib.request import urllib.error from urllib.parse import urlparse @@ -268,6 +269,14 @@ def control(self, body): ), config="native.config.ts", expected=4, extra_env={"CROWDB_NATIVE_CHUNK_FIXTURE": json.dumps(chunks)}) return {"passed": True} + if self.case_name == "console::native_browser::journal_identity_and_owner_recovery": + bucket = body.get("journal_bucket") + if not isinstance(bucket, str) or not re.fullmatch(r"journal-[a-f0-9]{32}", bucket): + raise ValueError("journal diagnostics require their settled owned object") + self.browser_timeout = 180 + self.browser("native diagnostics: journal identity survives Chunk navigation and owner interruption", + config="native.config.ts", extra_env={"CROWDB_NATIVE_JOURNAL_BUCKET": bucket}) + return {"passed": True} cases = { "console::native_browser::catalog_resource_navigation": ( "native diagnostics: Iceberg empty Catalog omits the first-success guide", diff --git a/e2e-test/tests/access_iceberg/gc/convergence.rs b/e2e-test/tests/access_iceberg/gc/convergence.rs index 13a3457f8..99f3d3d03 100644 --- a/e2e-test/tests/access_iceberg/gc/convergence.rs +++ b/e2e-test/tests/access_iceberg/gc/convergence.rs @@ -88,7 +88,9 @@ impl TestGcConvergence { { self.observe(&task)?; if let Some(pressure) = &mut self.pressure { - pressure.observe(task.deleted).await?; + pressure + .observe(task.deleted, task.phase == GcPhase::Complete) + .await?; } else if task.deleted > 0 { if task.phase == GcPhase::Complete { return Err("GC completed before the foreground overlap could be started".into()); diff --git a/e2e-test/tests/access_iceberg/gc/convergence/pressure.rs b/e2e-test/tests/access_iceberg/gc/convergence/pressure.rs index a29500daa..d66b5069f 100644 --- a/e2e-test/tests/access_iceberg/gc/convergence/pressure.rs +++ b/e2e-test/tests/access_iceberg/gc/convergence/pressure.rs @@ -15,6 +15,7 @@ use std::{ pub(in crate::access_iceberg::gc) struct TestGcPressure { sdk: std::process::Child, stdout: PathBuf, + control: PathBuf, writes: Option>, progress: Arc, previous_active: Option<(u64, u64)>, @@ -27,9 +28,13 @@ impl TestGcPressure { let stdout = PathBuf::from(std::env::var("CROWDB_E2E_ARTIFACTS").map_err(|error| error.to_string())?) .join(format!("gc-inline-sdk-{}.log", OperationId::random())); let output = std::fs::File::create(&stdout).map_err(|error| error.to_string())?; + let control = stdout.with_extension("control"); + std::fs::create_dir(&control).map_err(|error| error.to_string())?; + std::fs::write(control.join("run"), b"").map_err(|error| error.to_string())?; let sdk = std::process::Command::new("pixi") .args(["run", "-e", "iceberg-e2e", "python", "-c", SDK_PRESSURE_SCRIPT]) .args([origin, "12", "0.5", "gc-convergence"]) + .arg(&control) .stdout(output) .spawn() .map_err(|error| error.to_string())?; @@ -42,6 +47,7 @@ impl TestGcPressure { Ok(Self { sdk, stdout, + control, writes: Some(writes), progress, previous_active: None, @@ -50,13 +56,20 @@ impl TestGcPressure { }) } - pub(in crate::access_iceberg::gc) async fn observe(&mut self, deleted: u64) -> Result<(), String> { + pub(in crate::access_iceberg::gc) async fn observe( + &mut self, + deleted: u64, + gc_complete: bool, + ) -> Result<(), String> { if let Some(status) = self.sdk.try_wait().map_err(|error| error.to_string())? { if !status.success() { return Err(format!("official SDK foreground failed: {status}")); } self.sdk_complete = true; } + if gc_complete && self.writes.is_none() { + std::fs::write(self.control.join("stop"), b"").map_err(|error| error.to_string())?; + } if self .writes .as_ref() @@ -90,6 +103,7 @@ impl TestGcPressure { } pub(in crate::access_iceberg::gc) async fn stop(&mut self) { + let _ = std::fs::write(self.control.join("stop"), b""); if let Some(writes) = self.writes.take() { writes.abort(); let _ = writes.await; @@ -118,6 +132,23 @@ impl TestGcPressure { .find_map(|line| line.strip_prefix("gc_foreground_official_sdk ")) .ok_or("official SDK latency report is missing")?; let report: serde_json::Value = serde_json::from_str(report).map_err(|error| error.to_string())?; + for operation in [ + "create_table", + "commit", + "load_table", + "credentials", + "read_metadata", + "drop_table", + ] { + if report[operation]["count"] + .as_u64() + .map_or(true, |count| count < 48) + { + return Err(format!( + "official SDK did not complete its minimum foreground workload: {operation}" + )); + } + } let overlapping = report .as_object() .ok_or("SDK report must be an object")? @@ -143,10 +174,10 @@ impl TestGcPressure { "gc_foreground_deletion_overlap {}", serde_json::json!({ "deletion_intervals_unix_ns":self.deletion_intervals,"small_writes":64, - "sdk_workers":4,"tables_per_worker":12,"inter_table_sleep_ms":500, + "sdk_workers":4,"minimum_tables_per_worker":12,"inter_table_sleep_ms":500, "small_write_interval_ms":250,"sleeps_in_latency_samples":false, "sdk_stdout_artifact":self.stdout, - "scope":"independent fixed load; latency is not directly comparable to the original three-table, 50-ms probe", + "scope":"continuous foreground through cleanup and small-writer completion; latency is not a fixed-load baseline", }) ); Ok(()) diff --git a/e2e-test/tests/access_iceberg/gc/notifications/matrix.rs b/e2e-test/tests/access_iceberg/gc/notifications/matrix.rs index df69e69e1..e969916e9 100644 --- a/e2e-test/tests/access_iceberg/gc/notifications/matrix.rs +++ b/e2e-test/tests/access_iceberg/gc/notifications/matrix.rs @@ -125,7 +125,7 @@ impl TestMatrix { let deleted = task.deleted; self.observe(&task, multipart.as_ref()); if let Some(pressure) = &mut self.pressure { - pressure.observe(deleted).await?; + pressure.observe(deleted, task.phase == GcPhase::Complete).await?; } else if deleted > 0 { let remaining = self.records.remaining().await?; if task.phase == GcPhase::Complete || remaining == 0 { diff --git a/e2e-test/tests/access_iceberg/gc/sdk_pressure.py b/e2e-test/tests/access_iceberg/gc/sdk_pressure.py index 9e90f4f9f..f9571512e 100644 --- a/e2e-test/tests/access_iceberg/gc/sdk_pressure.py +++ b/e2e-test/tests/access_iceberg/gc/sdk_pressure.py @@ -33,8 +33,14 @@ def measure(operation, call): raise RuntimeError("foreground cancelled before release") time.sleep(0.01) count = int(sys.argv[2]) if len(sys.argv) > 2 else 3 + interval = float(sys.argv[3]) if len(sys.argv) > 3 else 0 + # Keep paced workers spread across a cycle so a short deletion interval + # cannot repeatedly fall into all four workers' inter-table gaps. + if interval: + time.sleep(worker * interval / 4) index = 0 - while index < count or (count == 0 and (index < 3 or not (control / "stop").exists())): + minimum = count if count else 3 + while index < minimum or (control and not (control / "stop").exists()): identifier = namespace + (f"events-{index}",) table = measure("create_table", lambda: catalog.create_table(identifier, Schema(NestedField(field_id=1, name="id", field_type=LongType(), required=True)))) measure("commit", lambda: table.transaction().set_properties({"gc-probe": str(index)}).commit_transaction()) @@ -58,7 +64,7 @@ def read_metadata(): temporary.write_text(str(index)) temporary.replace(control / f"progress-{worker}") if len(sys.argv) > 3: - time.sleep(float(sys.argv[3])) + time.sleep(interval) measure("drop_namespace", lambda: catalog.drop_namespace(namespace)) return samples, windows diff --git a/e2e-test/tests/common/control.rs b/e2e-test/tests/common/control.rs index d4d0c21d4..fa86e9246 100644 --- a/e2e-test/tests/common/control.rs +++ b/e2e-test/tests/common/control.rs @@ -26,6 +26,14 @@ pub(crate) async fn native_browser() -> Value { request_with_timeout(json!({"action": "run-native-browser"}), Duration::from_secs(305)).await } +pub(crate) async fn native_browser_with_journal(bucket: &str) -> Value { + request_with_timeout( + json!({"action":"run-native-browser", "journal_bucket":bucket}), + Duration::from_secs(305), + ) + .await +} + pub(crate) async fn native_browser_with_chunks(chunks: Value) -> Value { request_with_timeout( json!({"action":"run-native-browser", "chunks": chunks}), diff --git a/e2e-test/tests/console/deployment.rs b/e2e-test/tests/console/deployment.rs index 0b8d27a8a..b7d93976d 100644 --- a/e2e-test/tests/console/deployment.rs +++ b/e2e-test/tests/console/deployment.rs @@ -36,6 +36,7 @@ async fn concurrent_admissions_allocate_unique_stable_ids_through_group_zero() { .await .unwrap(); let node = |uuid: &str, host: &str| NodeRecord { + web_port: 9090, discovery_id: uuid.into(), node_id: 0, physical_host_id: "same-host".into(), @@ -173,6 +174,7 @@ async fn authenticated_node_update_preserves_ids_and_converges_from_another_cons .await .unwrap(); let source = NodeRecord { + web_port: 9090, discovery_id: uuid::Uuid::new_v4().to_string(), node_id: 7, physical_host_id: "same-host".into(), diff --git a/e2e-test/tests/console/native_balance.rs b/e2e-test/tests/console/native_balance.rs index dc701c7ad..277a8b0ac 100644 --- a/e2e-test/tests/console/native_balance.rs +++ b/e2e-test/tests/console/native_balance.rs @@ -34,6 +34,32 @@ pub(super) async fn settle(context: &TestNativeContext) { verify_values(&client).await; } +pub(super) async fn settle_existing(context: &TestNativeContext) { + let source = Group0ChunkKvRangeCatalogSource::from_shared(context.kv.clone()); + let started = Instant::now(); + let mut interval = tokio::time::interval(Duration::from_millis(100)); + loop { + interval.tick().await; + let catalog = load(&source).await; + if catalog.entries().iter().all(|entry| { + entry.state == ChunkKvRangeCatalogPartitionState::Serving + && entry.transition_id.is_none() + && entry.artifact.tail_overlay.is_none() + }) && placement_ready(&context.kv, catalog.generation()).await + { + eprintln!( + "[PHASE] journal fixture placement settled in {:?}", + started.elapsed() + ); + return; + } + assert!( + started.elapsed() < Duration::from_secs(40), + "journal fixture's normal placement did not settle" + ); + } +} + async fn seed_values(client: &ChunkKvClient) { for batch in 0..32 { let items = (batch * 16..(batch + 1) * 16) diff --git a/e2e-test/tests/console/native_browser.rs b/e2e-test/tests/console/native_browser.rs index 60a10bd5a..33d56af98 100644 --- a/e2e-test/tests/console/native_browser.rs +++ b/e2e-test/tests/console/native_browser.rs @@ -44,7 +44,33 @@ async fn storage_windows_and_access_recovery() { #[tokio::test] #[ignore = "requires normal storage and a host browser for owner recovery"] async fn journal_identity_and_owner_recovery() { - browse_storage().await; + let context = TestNativeContext::connect().await; + super::native_storage::start(&context).await; + let bucket = format!("journal-{}", uuid::Uuid::new_v4().simple()); + for (path, body) in [ + (format!("/api/access/s3/{bucket}"), ""), + ( + format!("/api/access/s3/{bucket}/value"), + "native journal observation", + ), + ] { + let response = context + .http + .put(format!("{}{path}", context.base)) + .body(body) + .timeout(std::time::Duration::from_secs(10)) + .send() + .await + .unwrap(); + assert!( + response.status().is_success(), + "journal fixture write: {}", + response.text().await.unwrap() + ); + } + super::native_balance::settle_existing(&context).await; + let result = crate::common::control::native_browser_with_journal(&bucket).await; + assert_eq!(result["passed"], true); } #[tokio::test] diff --git a/e2e-test/tests/console_cli_test.py b/e2e-test/tests/console_cli_test.py index 32ec73096..71b16868c 100644 --- a/e2e-test/tests/console_cli_test.py +++ b/e2e-test/tests/console_cli_test.py @@ -9,6 +9,8 @@ from console_cli_fixture import CliContainerCluster, LABEL, OWNER_LABEL from console_cli_target import CliAttachedCluster +from console_cli_recovery import health_snapshot +from console_cli_io_fault import DiskIoOutage from fixture import Cluster @@ -25,7 +27,7 @@ def test_diagnostics_export_bounded_access_metrics_before_teardown(self): opener.open.return_value = MagicMock() opener.open.return_value.__enter__.return_value = response cluster.diagnostics() - opener.open.assert_called_once_with("http://192.0.2.1:9091/_crowdb/metrics", timeout=2) + opener.open.assert_called_once_with("http://192.0.2.1:16000/_crowdb/metrics", timeout=2) response.read.assert_called_once_with(1024 * 1024 + 1) collect.assert_called_once() self.assertIn('phase="diskio"', (cluster.artifacts / "access-metrics-0.prom").read_text()) @@ -97,3 +99,71 @@ def test_recovery_observation_rejects_invalid_node_before_inspection(self): with self.assertRaisesRegex(ValueError, "outside"): cluster.read_owned("node-binding.json", index) cluster.command.assert_not_called() + + +class CliAcceptedListenersTest(unittest.TestCase): + def test_health_uses_each_owned_nodes_accepted_listener(self): + for port in (9094, 19000, 19002): + with self.subTest(port=port), tempfile.TemporaryDirectory() as directory: + cluster = object.__new__(CliContainerCluster) + cluster.servers = ["owned"] + cluster.addresses = ["192.0.2.1"] + cluster.artifacts = Path(directory) + cluster.inspect_owned = Mock(return_value={"Id": "verified"}) + cluster.read_owned = Mock(return_value={"setup": {"services": [ + {"kind": "access-server", "enabled": True, "listeners": { + "health_port": port, "s3_port": 16000}}]}}) + cluster.command = Mock(return_value='{"ready":true,"observation":{"state":"current"}}') + self.assertTrue(health_snapshot(cluster, "test")[0]["ready"]) + cluster.read_owned.assert_called_once_with("node-policy.json", 0) + cluster.command.assert_called_once_with("exec", "verified", "curl", "--silent", + "--show-error", "--max-time", "3", f"http://192.0.2.1:{port}/_crowdb/health/ready") + + +class CliFaultCleanupTest(unittest.TestCase): + def outage(self): + outage = object.__new__(DiskIoOutage) + outage.chain = "OWNED_FAULT" + outage.nodes = [("a", 13000), ("b", 13000), ("c", 13000)] + outage._prepare_tools = Mock() + return outage + + def test_rule_removal_failure_still_flushes_all_owned_chains(self): + outage = self.outage() + calls = [] + + def command(node, *arguments): + calls.append((node, arguments)) + if node[0] == "c" and arguments[0] == "-D": + raise RuntimeError("one removal failed") + + outage._command = command + with self.assertRaisesRegex(RuntimeError, "one removal failed"): + with outage: + pass + for operation in ("-F", "-X"): + self.assertEqual({node[0] for node, args in calls if args[0] == operation}, {"a", "b", "c"}) + + def test_partial_setup_failure_cleans_previous_and_partial_nodes(self): + outage = self.outage() + calls = [] + + def command(node, *arguments): + calls.append((node, arguments)) + if node[0] == "b" and arguments[0] == "-A": + raise RuntimeError("rule setup failed") + + outage._command = command + with self.assertRaisesRegex(RuntimeError, "rule setup failed"): + with outage: + self.fail("partial setup must not admit the workload") + self.assertEqual({node[0] for node, args in calls if args[0] == "-X"}, {"a", "b"}) + self.assertFalse(any(node[0] == "c" for node, _args in calls)) + + def test_network_fault_rejects_host_network_before_executing_helpers(self): + outage = self.outage() + outage.cluster = Mock(run_id="owned") + outage.cluster.inspect_owned.return_value = {"HostConfig": {"NetworkMode": "host"}} + with self.assertRaisesRegex(ValueError, "owned CLI bridge"): + DiskIoOutage._command(outage, outage.nodes[0], "-N", outage.chain) + outage.cluster.command.assert_not_called() diff --git a/e2e-test/tests/deployment.rs b/e2e-test/tests/deployment.rs index 6a7793c36..6183c9053 100644 --- a/e2e-test/tests/deployment.rs +++ b/e2e-test/tests/deployment.rs @@ -18,6 +18,9 @@ mod validation; #[tokio::test] #[ignore = "requires the normal release-monitor container fixture"] async fn selected_kv_services_recover_acknowledged_data() { + if std::env::var("CROWDB_E2E_PHASE").as_deref() == Ok("write") { + kv_bootstrap::create_witness_group().await; + } witness(false).await; } diff --git a/e2e-test/tests/deployment/kv_bootstrap.rs b/e2e-test/tests/deployment/kv_bootstrap.rs index d601feebd..6dc8b00ac 100644 --- a/e2e-test/tests/deployment/kv_bootstrap.rs +++ b/e2e-test/tests/deployment/kv_bootstrap.rs @@ -33,6 +33,9 @@ async fn monitor_creates_real_kv_groups_then_validates_after_restart() { ) .await; assert_eq!(result["bootstrap"], accepted["bootstrap"]); + if write { + create_witness_group().await; + } for group in [0, 1] { let ready = request( &http, @@ -70,3 +73,66 @@ async fn monitor_creates_real_kv_groups_then_validates_after_restart() { assert_eq!(retained["bootstrap"], accepted["bootstrap"]); super::witness_with_write(false, write).await; } + +pub(super) async fn create_witness_group() { + use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; + use std::time::{Duration, Instant}; + + let origin = format!("http://{}:10000", crate::common::management::addresses(1)[0]); + let kv = CrowdbKvClient::new(ClientConfig::new(vec![origin.clone()])); + assert!(matches!( + kv.get( + 0, + 0, + crowdb_console_shared::deployment::preparation::DEFAULT_PREPARATION_KEY, + ReadMode::Linearizable, + None, + ) + .await + .unwrap(), + GetOutcome::NotFound + )); + let http = client(); + // KV-only admission creates Group 0. This case explicitly requests its + // ordinary witness group instead of depending on storage preparation. + request( + &http, + &origin, + Method::GET, + "/stores/0/groups/1/ready", + Value::Null, + 404, + ) + .await; + request( + &http, + &origin, + Method::POST, + "/stores/0/groups", + json!({"group_id":1,"replica_id":1}), + 201, + ) + .await; + let deadline = Instant::now() + Duration::from_secs(4); + loop { + let ready = request( + &http, + &origin, + Method::GET, + "/stores/0/groups/1/ready", + Value::Null, + 200, + ) + .await; + if ready["ready"] == true { + assert_eq!(ready["leader_id"], 1); + assert_eq!(ready["voting_replicas"], 1); + return; + } + assert!( + Instant::now() < deadline, + "explicit witness group did not become ready: {ready}" + ); + tokio::time::sleep(Duration::from_millis(25)).await; + } +} diff --git a/lib/crowdb-chunk-stream/src/production_chunk.rs b/lib/crowdb-chunk-stream/src/production_chunk.rs index 4b7527735..2c55905d3 100644 --- a/lib/crowdb-chunk-stream/src/production_chunk.rs +++ b/lib/crowdb-chunk-stream/src/production_chunk.rs @@ -490,7 +490,17 @@ impl StreamChunkStore for ProductionStreamChunkStore { } let chunk = state.chunk.load_full(); let name = StreamName::from_chunk_owner_key(&chunk.owner_key).ok_or(StreamError::StaleWriter)?; - if let Some(context) = self.owner_context(name, writer_epoch)? { + let context = self.owner_context(name, writer_epoch)?; + if chunk.state == ChunkState::Deleted as i32 { + if cursor != 0 || !recovery::finalized_empty_chunk(&chunk) { + return Err(StreamError::Corruption( + "deleted stream chunk is not a proven empty tail".into(), + )); + } + context.ok_or(StreamError::StaleWriter)?.check_io()?; + return Ok(()); + } + if let Some(context) = context { if chunk.state != ChunkState::Sealed as i32 { self.drain_for_seal(&chunk, &context).await?; } diff --git a/lib/crowdb-chunk-stream/src/production_chunk/recovery.rs b/lib/crowdb-chunk-stream/src/production_chunk/recovery.rs index 8d9b2b2e1..ba4c884b1 100644 --- a/lib/crowdb-chunk-stream/src/production_chunk/recovery.rs +++ b/lib/crowdb-chunk-stream/src/production_chunk/recovery.rs @@ -47,6 +47,11 @@ impl ProductionStreamChunkStore { { return Err(StreamError::StaleWriter); } + if chunk.state == ChunkState::Deleted as i32 && !finalized_empty_chunk(&chunk) { + return Err(StreamError::Corruption( + "deleted stream chunk lacks empty finalization proof".into(), + )); + } let state = self.install(chunk.clone())?; if chunk.strips.is_empty() { let recovered = empty_sealed_tail(&chunk, active)?; @@ -121,11 +126,11 @@ fn empty_sealed_tail(chunk: &Chunk, active: &ActiveChunkDescriptor) -> Result Result bool { + use crowdb_protocol::chunk_stream::StreamPurpose; + chunk.state == ChunkState::Deleted as i32 + && chunk.sealed_ts_ms != 0 + && chunk.writer_epoch != 0 + && chunk.strips.is_empty() + && chunk.cleanup_intents.is_empty() + && chunk.capacity == 0 + && chunk.sealed_length == 0 + && chunk.acknowledged_cursor == 0 + && chunk.id.is_some_and(|id| { + [StreamPurpose::Wal, StreamPurpose::Stream] + .into_iter() + .any(|purpose| purpose.matches(id) && chunk.chunk_type == purpose.chunk_type() as i32) + }) +} + fn observation_error(error: &crowdb_chunk_client::FrameObservationError) -> StreamError { use crowdb_chunk_client::FrameObservationError as E; match error { diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test.rs index cc25d1e0c..ec21915da 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test.rs @@ -25,6 +25,9 @@ use crowdb_protocol::chunkdb::rpc::{ use crowdb_protocol::common::{ChunkId, DiskId}; use crowdb_protocol::diskdb::rpc::Segment; +#[path = "production_chunk_test/finalized_empty.rs"] +mod finalized_empty; + #[path = "production_chunk_test/recovery.rs"] mod recovery; diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/finalized_empty.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/finalized_empty.rs new file mode 100644 index 000000000..ccd9f195a --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/finalized_empty.rs @@ -0,0 +1,145 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test] +async fn finalized_empty_tail_recovers_without_reviving_the_deleted_chunk() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let make = || { + ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap() + }; + let name = StreamName { high: 813, low: 99 }; + let writer = make(); + let mut active = writer + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Wal) + .await + .unwrap(); + active.logical_start = 42; + let allocated = allocator.chunk.lock().unwrap().clone().unwrap(); + let deleted = { + let mut guard = allocator.chunk.lock().unwrap(); + let chunk = guard.as_mut().unwrap(); + chunk.state = ChunkState::Deleted as i32; + chunk.sealed_ts_ms = 123; + chunk.modify_ts += 2; + chunk.capacity = 0; + chunk.strips.clear(); + chunk.clone() + }; + drop(writer); + let cold = make(); + assert!(matches!( + cold.seal(active.chunk_id, 10, 0).await, + Err(crowdb_chunk_stream::StreamError::StaleWriter) + )); + let metadata = MemoryStreamStore::new(1024); + cold.bind_writer(name, 10, metadata.claim_writer(name, 10).await.unwrap()); + let recovered = cold.recover_tail(name, 10, &active).await.unwrap(); + assert_eq!((recovered.physical_end, recovered.logical_end), (0, 42)); + assert!(recovered.extents.is_empty()); + cold.seal(active.chunk_id, 10, 0).await.unwrap(); + assert_eq!(*allocator.chunk.lock().unwrap(), Some(deleted.clone())); + for field in 0..11 { + let mut invalid = deleted.clone(); + let mut descriptor = active.clone(); + match field { + 0 => invalid.sealed_ts_ms = 0, + 1 => invalid.capacity = 1, + 2 => invalid.sealed_length = 1, + 3 => invalid.acknowledged_cursor = 1, + 4 => descriptor.physical_start = 1, + 5 => descriptor.acknowledged_cursor = 1, + 6 => invalid.writer_epoch = 0, + 7 => invalid.chunk_type = crowdb_protocol::chunkdb::rpc::ChunkType::S3 as i32, + 8 => invalid + .cleanup_intents + .push(crowdb_protocol::chunkdb::rpc::StripCleanupIntent::default()), + 9 => invalid.strips.clone_from(&allocated.strips), + _ => invalid.state = ChunkState::Active as i32, + } + *allocator.chunk.lock().unwrap() = Some(invalid); + assert!(matches!( + cold.recover_tail(name, 10, &descriptor).await, + Err(crowdb_chunk_stream::StreamError::Corruption(_)) + )); + } +} + +#[tokio::test] +async fn finalized_empty_stream_tail_reopens_under_a_new_writer_with_a_fresh_chunk() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let make = || { + Arc::new( + ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(), + ) + }; + let metadata = Arc::new(MemoryStreamStore::new(1024)); + let name = StreamName { high: 814, low: 100 }; + let stream = ChunkStream::create( + StreamRecord { + purpose: crowdb_chunk_stream::StreamPurpose::Wal, + stream_name: name, + metadata_group_id: 7, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: None, + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + 9, + StreamConfig::default(), + metadata.clone(), + metadata.clone(), + make(), + ) + .await + .unwrap(); + stream.release_writer().await.unwrap(); + let old = { + let mut guard = allocator.chunk.lock().unwrap(); + let chunk = guard.as_mut().unwrap(); + chunk.state = ChunkState::Deleted as i32; + chunk.sealed_ts_ms = 123; + chunk.modify_ts += 2; + chunk.capacity = 0; + chunk.strips.clear(); + chunk.clone() + }; + let reopened = ChunkStream::open( + name, + 10, + StreamConfig::default(), + metadata.clone(), + metadata, + make(), + ) + .await + .unwrap(); + assert_eq!(reopened.tail(), 0); + assert_eq!(*allocator.chunk.lock().unwrap(), Some(old.clone())); + let appended = reopened + .append(&[Bytes::from_static(b"fresh-writer")]) + .await + .unwrap(); + assert_ne!(appended.chunk_id, old.id); + assert_eq!( + reopened.read_at(0, 12).await.unwrap(), + Bytes::from_static(b"fresh-writer") + ); + reopened.release_writer().await.unwrap(); +} diff --git a/lib/crowdb-console-shared/src/deployment/admission.rs b/lib/crowdb-console-shared/src/deployment/admission.rs index 270b7e689..4ef0d5f3d 100644 --- a/lib/crowdb-console-shared/src/deployment/admission.rs +++ b/lib/crowdb-console-shared/src/deployment/admission.rs @@ -20,6 +20,8 @@ pub struct SshNodeIdentity { pub discovery_id: String, pub public_key: String, pub host_key: String, + #[serde(default = "super::registry::default_web_port")] + pub web_port: u16, } /// # Errors diff --git a/lib/crowdb-console-shared/src/deployment/preparation/record.rs b/lib/crowdb-console-shared/src/deployment/preparation/record.rs index b60d58586..5102cacda 100644 --- a/lib/crowdb-console-shared/src/deployment/preparation/record.rs +++ b/lib/crowdb-console-shared/src/deployment/preparation/record.rs @@ -69,9 +69,9 @@ impl DefaultPreparation { fn with_operation(bootstrap: &PreparedBootstrap, operation_id: String) -> Result { let members = bootstrap.intent.members(); - if !matches!(members.len(), 1 | 3) { + if !matches!(members.len(), 1 | 3 | 5) { return Err(Error::Config( - "default preparation requires one or three admitted voters".into(), + "default preparation requires one, three or five admitted voters".into(), )); } let config = bootstrap.intent.to_config(); diff --git a/lib/crowdb-console-shared/src/deployment/registry.rs b/lib/crowdb-console-shared/src/deployment/registry.rs index 87b0cd3b7..1ebe3b1db 100644 --- a/lib/crowdb-console-shared/src/deployment/registry.rs +++ b/lib/crowdb-console-shared/src/deployment/registry.rs @@ -16,6 +16,8 @@ pub struct NodeRecord { pub physical_host_id: String, pub rack_id: u64, pub host: String, + #[serde(default = "default_web_port", skip_serializing_if = "IS_DEFAULT_WEB_PORT")] + pub web_port: u16, pub ssh_port: u16, pub ssh_user: String, pub operation_id: String, @@ -27,6 +29,13 @@ pub struct NodeRecord { pub cancelled: bool, } +pub(super) const fn default_web_port() -> u16 { + // Retained packaged-node records predate explicit Web listener publication. + 9090 +} + +const IS_DEFAULT_WEB_PORT: fn(&u16) -> bool = |port| *port == default_web_port(); + #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub struct NodeRegistry { pub cluster_id: String, diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness.rs index 767ccd3de..8c9ba0571 100644 --- a/lib/crowdb-console-shared/src/ops/cluster/readiness.rs +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness.rs @@ -185,6 +185,7 @@ fn required(observation: &ClusterObservation, kind: DependencyKind) -> bool { return true; }; match kind { + DependencyKind::Metadata => policy.setup.requires_default_preparation(), DependencyKind::Storage => !matches!(policy.setup.storage, SetupStorage::Diskless), DependencyKind::Protocol => policy .setup @@ -197,7 +198,7 @@ fn required(observation: &ClusterObservation, kind: DependencyKind) -> bool { fn invalid(observation: &ClusterObservation) -> bool { observation.version != 1 - || !matches!(observation.initial_voters, 1 | 3) + || !matches!(observation.initial_voters, 1 | 3 | 5) || (observation.bound && observation.cluster_id.is_none()) || observation.cluster_id.as_ref().is_some_and(|identity| { uuid::Uuid::parse_str(identity).map_or(true, |identity| identity.is_nil()) diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs index 3105a2587..d96d3477e 100644 --- a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs @@ -129,20 +129,36 @@ async fn gather( let (nodes, intents, metadata, storage) = tokio::join!( tokio::time::timeout_at(deadline, registry::read(ctx.kv())), tokio::time::timeout_at(deadline, services::list(ctx.kv())), - tokio::time::timeout_at(deadline, storage::metadata(ctx, bootstrap)), - tokio::time::timeout_at(deadline, storage::read(ctx, bootstrap)), + async { + if local.policy.setup.requires_default_preparation() { + Some(tokio::time::timeout_at(deadline, storage::metadata(ctx, bootstrap)).await) + } else { + None + } + }, + async { + if local.policy.setup.requires_default_preparation() { + tokio::time::timeout_at(deadline, storage::read(ctx, bootstrap)) + .await + .ok() + .and_then(Result::ok) + } else { + None + } + }, ); - observation - .dependencies - .push(metadata.ok().and_then(Result::ok).unwrap_or_else(|| { - fact( - "cluster/metadata", - DependencyKind::Metadata, - DependencyState::Unknown, - "metadata-unavailable", - ) - })); - let storage = storage.ok().and_then(Result::ok); + if let Some(metadata) = metadata { + observation + .dependencies + .push(metadata.ok().and_then(Result::ok).unwrap_or_else(|| { + fact( + "cluster/metadata", + DependencyKind::Metadata, + DependencyState::Unknown, + "metadata-unavailable", + ) + })); + } let Ok(Ok(Some((registry, _)))) = nodes else { append_storage(observation, storage.as_ref(), &std::collections::BTreeMap::new()); return; @@ -173,6 +189,13 @@ fn append_storage( storage: Option<&storage::StorageSnapshot>, nodes: &std::collections::BTreeMap, ) { + if observation + .policy + .as_ref() + .is_some_and(|policy| !policy.setup.requires_default_preparation()) + { + return; + } observation.dependencies.extend(storage.map_or_else( || { vec![fact( diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/members.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/members.rs index e94cc2721..223f31deb 100644 --- a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/members.rs +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/members.rs @@ -67,12 +67,13 @@ pub(super) async fn read( discovery_id: node.discovery_id.clone(), node_id: Some(node.node_id), management_endpoints: vec![format!( - "http://{}:9090", + "http://{}:{}", if host.contains(':') { format!("[{host}]") } else { host.into() - } + }, + node.web_port, )], }); let now = now_ms(); @@ -109,7 +110,10 @@ async fn member(client: &reqwest::Client, node: ®istry::NodeRecord) -> Option host.into() }; let mut reply = client - .get(format!("http://{host}:9090/api/cluster/node-observation")) + .get(format!( + "http://{host}:{}/api/cluster/node-observation", + node.web_port + )) .send() .await .ok()? diff --git a/lib/crowdb-console-shared/tests/cluster_readiness_test.rs b/lib/crowdb-console-shared/tests/cluster_readiness_test.rs index a29b9fc53..3bab3a90e 100644 --- a/lib/crowdb-console-shared/tests/cluster_readiness_test.rs +++ b/lib/crowdb-console-shared/tests/cluster_readiness_test.rs @@ -269,12 +269,29 @@ fn diskless_kv_only_policy_requires_its_selected_services_without_inventing_prot } policy.validate().unwrap(); input.policy = Some(policy); - input - .dependencies - .retain(|fact| !matches!(fact.kind, DependencyKind::Storage | DependencyKind::Protocol)); + input.dependencies.retain(|fact| { + !matches!( + fact.kind, + DependencyKind::Metadata | DependencyKind::Storage | DependencyKind::Protocol + ) + }); assert_eq!(reduce(input.clone(), 1000).phase, ClusterPhase::Ready); let mut remote_protocol = fact(DependencyKind::Protocol); remote_protocol.state = DependencyState::Unknown; input.dependencies.push(remote_protocol); assert_eq!(reduce(input, 1000).phase, ClusterPhase::Degraded); } + +#[test] +fn five_voter_readiness_preserves_required_evidence_and_rejects_other_counts() { + let mut input = observation(); + input.initial_voters = 5; + assert_eq!(reduce(input.clone(), 1000).phase, ClusterPhase::Ready); + input.dependencies[0].state = DependencyState::Unknown; + assert_eq!(reduce(input, 1000).phase, ClusterPhase::Unavailable); + for count in [0, 2, 4, 6, 7] { + let mut input = observation(); + input.initial_voters = count; + assert_eq!(reduce(input, 1000).phase, ClusterPhase::Failed); + } +} diff --git a/lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs b/lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs index 5159309ff..2bf45d7eb 100644 --- a/lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs +++ b/lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs @@ -280,6 +280,7 @@ async fn diskless_member_does_not_require_an_unused_storage_reservation() { &NodeRegistry { cluster_id: authority.bootstrap.identity.cluster_id.clone(), nodes: vec![NodeRecord { + web_port: 9090, discovery_id: node.discovery_id.clone(), node_id: 2, rack_id: 4, @@ -312,3 +313,106 @@ async fn diskless_member_does_not_require_an_unused_storage_reservation() { report.observation.dependencies ); } + +#[tokio::test] +async fn kv_only_collection_uses_confirmed_web_ports_without_preparing_storage() { + use crowdb_console_shared::deployment::registry::{self, NodeRecord, NodeRegistry}; + use crowdb_protocol::mgmt::{ + cluster::{ClusterPhase, DependencyKind}, + node::{ + observation::NodeServiceObservation, + setup::{SetupServiceKind, SetupStorage}, + }, + }; + let authority = TestPreparationAuthority::start(&[2]).await; + let mut node = local(&authority); + node.policy.setup.storage = SetupStorage::Diskless; + for service in &mut node.policy.setup.services { + service.enabled = service.kind == SetupServiceKind::PaxosKv; + } + node.observed_at_unix_ms = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap(); + node.services = vec![NodeServiceObservation { + service_id: "kv".into(), + kind: "kv".into(), + operation_id: None, + state: DependencyState::Ready, + detail: "current supervised KV".into(), + observed_at_unix_ms: node.observed_at_unix_ms, + max_age_ms: 3000, + reason_code: None, + }]; + let mut remote = node.clone(); + remote.discovery_id = uuid::Uuid::new_v4().to_string(); + remote.binding.as_mut().unwrap().node_id = 7; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let response = remote.clone(); + let task = tokio::spawn(async move { + axum::serve( + listener, + axum::Router::new().route( + "/api/cluster/node-observation", + axum::routing::get(move || { + let value = response.clone(); + async move { axum::Json(value) } + }), + ), + ) + .await + .unwrap(); + }); + let records = [ + (2, node.discovery_id.clone(), 9090), + (7, remote.discovery_id.clone(), port), + ] + .into_iter() + .map(|(node_id, discovery_id, web_port)| NodeRecord { + node_id, + discovery_id, + web_port, + rack_id: 4, + physical_host_id: "owned".into(), + host: "127.0.0.1".into(), + ssh_port: 2222, + ssh_user: "crowdb".into(), + operation_id: uuid::Uuid::new_v4().to_string(), + confirmed: true, + host_key: None, + cancelled: false, + }) + .collect(); + registry::publish( + authority.ctx.kv(), + &NodeRegistry { + cluster_id: authority.bootstrap.identity.cluster_id.clone(), + nodes: records, + }, + ) + .await + .unwrap(); + let report = collect(&authority.ctx, node).await.unwrap(); + task.abort(); + assert_eq!(report.phase, ClusterPhase::Ready, "{:?}", report.reasons); + assert_eq!(report.observation.members.len(), 2); + assert!(report + .observation + .members + .iter() + .any(|member| member.node_id == Some(7) + && member.management_endpoints == [format!("http://127.0.0.1:{port}")])); + assert!(!report.observation.dependencies.iter().any(|fact| matches!( + fact.kind, + DependencyKind::Metadata | DependencyKind::Storage | DependencyKind::Protocol + ))); + assert!(preparation::read(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap() + .is_none()); + assert!(authority.ctx.sysmd().get_group(0, 1).await.unwrap().is_none()); +} diff --git a/lib/crowdb-console-shared/tests/cluster_supplied_storage_observation_test.rs b/lib/crowdb-console-shared/tests/cluster_supplied_storage_observation_test.rs index 276b5ee80..837f72c94 100644 --- a/lib/crowdb-console-shared/tests/cluster_supplied_storage_observation_test.rs +++ b/lib/crowdb-console-shared/tests/cluster_supplied_storage_observation_test.rs @@ -61,6 +61,7 @@ async fn setup() -> (TestPreparationAuthority, NodeRuntimeObservation) { &NodeRegistry { cluster_id: authority.bootstrap.identity.cluster_id.clone(), nodes: vec![NodeRecord { + web_port: 9090, discovery_id: discovery_id.clone(), node_id: 2, physical_host_id: "owned-host".into(), diff --git a/lib/crowdb-console-shared/tests/common/member_evidence.rs b/lib/crowdb-console-shared/tests/common/member_evidence.rs index ed5221eb5..82c60d442 100644 --- a/lib/crowdb-console-shared/tests/common/member_evidence.rs +++ b/lib/crowdb-console-shared/tests/common/member_evidence.rs @@ -34,6 +34,7 @@ impl TestMemberEvidence { configuration_digest: "accepted-digest".into(), }; let expected = NodeRecord { + web_port: 9090, discovery_id: uuid::Uuid::new_v4().to_string(), node_id: 1, physical_host_id: "test-host".into(), diff --git a/lib/crowdb-console-shared/tests/default_preparation_test.rs b/lib/crowdb-console-shared/tests/default_preparation_test.rs index 3e1851b2d..abe1b0090 100644 --- a/lib/crowdb-console-shared/tests/default_preparation_test.rs +++ b/lib/crowdb-console-shared/tests/default_preparation_test.rs @@ -362,3 +362,50 @@ async fn explicit_retirement_is_idempotent_and_prevents_automatic_recreation() { ); assert_eq!(snapshot(&authority).await.unwrap(), retired); } + +#[tokio::test] +async fn five_voter_preparation_retains_every_admitted_identity_and_fixed_mapping() { + let members = [2, 7, 9, 12, 15]; + let authority = TestPreparationAuthority::start(&members).await; + let winner = preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap(); + assert_eq!( + winner + .metadata + .replicas + .iter() + .map(|r| (r.node_id, r.replica_id)) + .collect::>(), + vec![(2, 1), (7, 2), (9, 3), (12, 4), (15, 5)] + ); + assert_eq!( + winner + .storage + .iter() + .map(|b| (b.node_id, b.disk_group_id, b.rack_id, b.store_id, b.group_id)) + .collect::>(), + vec![ + (2, 1, 4, 0, 1), + (7, 2, 4, 0, 1), + (9, 3, 4, 0, 1), + (12, 4, 4, 0, 1), + (15, 5, 4, 0, 1) + ] + ); + assert_eq!(winner.chunk_kv.node_id, 2); + let before = snapshot(&authority).await.unwrap(); + let recovered = preparation::read(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap() + .unwrap(); + assert_eq!(recovered.0, winner); + assert_eq!(recovered.1, before.1); + assert_eq!( + preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap(), + winner + ); + assert_eq!(snapshot(&authority).await.unwrap(), before); +} diff --git a/lib/crowdb-console-shared/tests/member_evidence_test.rs b/lib/crowdb-console-shared/tests/member_evidence_test.rs index 0dc09521c..88cd89882 100644 --- a/lib/crowdb-console-shared/tests/member_evidence_test.rs +++ b/lib/crowdb-console-shared/tests/member_evidence_test.rs @@ -172,3 +172,24 @@ fn duplicate_service_or_protocol_identity_is_a_conflict() { assert_eq!(fixture.project()[0].state, DependencyState::Failed); } } + +#[test] +fn default_web_port_preserves_legacy_records_and_offset_ports_round_trip() { + let mut record = TestMemberEvidence::new().expected; + let legacy = serde_json::to_value(&record).unwrap(); + assert!(legacy.get("web_port").is_none()); + assert_eq!( + serde_json::from_value::(legacy.clone()) + .unwrap(), + record + ); + record.web_port = 9092; + let shifted = serde_json::to_value(&record).unwrap(); + assert_eq!(shifted["web_port"], 9092); + assert_eq!( + serde_json::from_value::(shifted).unwrap(), + record + ); + record.web_port = 9090; + assert_eq!(serde_json::to_value(record).unwrap(), legacy); +} diff --git a/lib/crowdb-kv-client/src/lib.rs b/lib/crowdb-kv-client/src/lib.rs index 6a8cfa8e7..f0aa40e1c 100644 --- a/lib/crowdb-kv-client/src/lib.rs +++ b/lib/crowdb-kv-client/src/lib.rs @@ -50,6 +50,7 @@ pub use hardware::{ }; pub use membership::{GroupMembershipClient, GroupMembershipSnapshot}; pub use metrics::{ClientMetrics, ClientMetricsSnapshot, LeaderChangeEpisode, WindowLatencySnapshot}; +pub use service::registry::DEFAULT_TTL_MS as SERVICE_INSTANCE_TTL_MS; pub use service::{ DomainMonitorClient, ServiceDiscoveryClient, ServiceRegistryClient, StreamProcessAuthority, StreamProcessContext, StreamProcessLeaseClient, StreamProcessLeaseError, StreamProcessLeaseStatus, diff --git a/lib/crowdb-kv-client/src/service/registry.rs b/lib/crowdb-kv-client/src/service/registry.rs index c119f4671..8d0fff807 100644 --- a/lib/crowdb-kv-client/src/service/registry.rs +++ b/lib/crowdb-kv-client/src/service/registry.rs @@ -29,7 +29,7 @@ const G0_STORE: u64 = 0; const G0_GROUP: u64 = 0; /// Default liveness TTL: 3× a 5s heartbeat interval = 15s. -const DEFAULT_TTL_MS: u64 = 15_000; +pub const DEFAULT_TTL_MS: u64 = 15_000; fn now_ms() -> u64 { SystemTime::now() diff --git a/lib/crowdb-kv/src/cluster/group.rs b/lib/crowdb-kv/src/cluster/group.rs index 1bb131239..3bd786527 100644 --- a/lib/crowdb-kv/src/cluster/group.rs +++ b/lib/crowdb-kv/src/cluster/group.rs @@ -939,16 +939,13 @@ impl PxGroup { } /// R65: Leader-side `FetchGap` handler. Looks up the requested slot - /// in the local acceptor. If the leader has an accepted value, replies - /// with it (payload + ballot + term). If the leader doesn't have the - /// value, returns `None` — the slot is not yet chosen from this - /// leader's perspective, and the follower should retry later (the - /// leader's `repair_once` will eventually fill it, or a newer - /// `ChosenNotice` will re-notify). + /// in the local acceptor, restricted to the recovered leader's proven + /// contiguous chosen prefix. An accepted value alone is insufficient + /// evidence for a follower to learn it. pub(crate) fn handle_fetch_gap(&self, slot: SlotIndex) -> Option { let replica = &self.local_replica; - if !replica.is_leader() { - debug!(slot, role = ?replica.role(), leader = ?replica.believed_leader_id(), "handle_fetch_gap: not leader, ignoring"); + if !replica.is_leader() || !self.leader_read_ready() || slot > replica.contiguous_chosen() { + debug!(slot, role = ?replica.role(), leader = ?replica.believed_leader_id(), "handle_fetch_gap: recovered chosen proof unavailable"); return None; } let entry = replica.acceptor.accepted_at(slot)?; diff --git a/lib/crowdb-kv/src/cluster/group_election.rs b/lib/crowdb-kv/src/cluster/group_election.rs index f208dd947..7755a1821 100644 --- a/lib/crowdb-kv/src/cluster/group_election.rs +++ b/lib/crowdb-kv/src/cluster/group_election.rs @@ -23,14 +23,16 @@ use std::time::Duration; use tokio::task::JoinHandle; use tokio::time::Instant; use tokio_util::sync::CancellationToken; -use tracing::{debug, info, trace, warn, Instrument}; +use tracing::{debug, info, trace, Instrument}; -use crate::cluster::group::{AcceptAttempt, PrepareAttempt, PxGroup, RemoteReplicaKind}; +use crate::cluster::group::{PxGroup, RemoteReplicaKind}; use crate::cluster::local_replica::PxLocalReplicaRole; use crate::common::config::PxElectionConfig; use crate::paxos::roles::SlotIndex; use crate::paxos::PxNodeId; +mod recovery; + /// Bundle handed off from `run_candidate_election` to `run_leader_state` /// when a candidate wins quorum. Carries the floor / ceiling needed by /// [`LeaderElection::run_bulk_phase1`] under the new tenure's cancel @@ -84,9 +86,8 @@ pub trait LeaderElection { /// /// Inputs: /// - /// - `floor` = `max(local.contiguous_chosen, peer_contiguous_chosen_max)` - /// — values from peer `RequestVote` / `PreVote` replies (the election - /// driver supplies the aggregate via `peer_contiguous_chosen_max`). + /// - `floor` = the local contiguous-chosen frontier. A peer's higher + /// frontier does not prove that its payloads are present locally. /// - `ceiling` = `max(local.acceptor.highest_seen_slot, /// self.next_slot - 1, /// peer_highest_seen_slot_max)`. @@ -100,9 +101,9 @@ pub trait LeaderElection { /// is decided (and the contiguous-chosen watermark can advance). /// 3. Re-Accept via the existing `run_accept_phase`. /// - /// After issuing (not waiting on completion of) the batch, `next_slot` - /// is bumped to `ceiling + 1` so steady-state proposals continue past - /// the repaired range (§4.4). + /// At most `cfg.bulk_prepare_window` slots recover concurrently. Once + /// the entire range is resolved and applied, reads may be admitted. + /// `next_slot` is advanced to `ceiling + 1` after the sweep. /// /// Cancellation: the `cancel` token is checked before each slot. On /// cancel the loop aborts without re-Accepting any further slots — the @@ -176,8 +177,6 @@ impl LeaderElection for PxGroup { *driver_guard = Some(handle); } - #[tracing::instrument(level = "info", skip_all, fields(s = self.log_store_id().unwrap_or(0), g = self.group_id, replica = self.local_replica().id, term))] - #[allow(clippy::too_many_lines)] async fn run_bulk_phase1( &self, term: u64, @@ -186,146 +185,14 @@ impl LeaderElection for PxGroup { cfg: PxElectionConfig, cancel: CancellationToken, ) { - let replica = self.local_replica(); - let group_id = self.group_id; - // Voting-only quorum -- see the matching fix/comment on - // `PxGroup::propose` in `group.rs`. - let quorum = self.quorum(); - - // Floor is the leader's OWN committed frontier — deliberately NOT maxed - // with `peer_contiguous_chosen_max`. After a restart a replica can win - // election while missing committed slots it never received (the value - // lived only as a `ChosenNotice` watermark, which carries no payload). - // Maxing the floor with the peers' commit point would SKIP exactly - // those slots, leaving the new leader serving a stale/old value for a - // committed key (e.g. a resurrected delete, or a missing put). Sweeping - // from the leader's own frontier forces Phase 1 to re-derive every - // higher slot from the quorum and adopt the real chosen value. This is - // safe because Phase 1 contacts a quorum: a committed value is held by - // at least one quorum member, so it is recovered rather than - // overwritten. - let _ = peer_contiguous_chosen_max; - let floor = replica.contiguous_chosen(); - let local_ceiling = replica.highest_seen_slot(); - let next_slot_minus_one = self.next_slot.load(Ordering::Acquire).saturating_sub(1); - let ceiling = local_ceiling - .max(next_slot_minus_one) - .max(peer_highest_seen_slot_max); - - debug!( - group_id, + self.recover_open_prefix( term, - floor, - ceiling, - local_contiguous_chosen = replica.contiguous_chosen(), - local_highest_seen_slot = local_ceiling, peer_contiguous_chosen_max, peer_highest_seen_slot_max, - "bulk phase 1 start" - ); - - if ceiling <= floor { - tokio::select! { - () = replica.await_apply_fence(floor) => {} - () = cancel.cancelled() => return, - } - self.leader_read_ready.store(true, Ordering::Release); - debug!(term, "bulk phase 1 skipped (empty range)"); - return; - } - - let mut slots_repaired = 0u64; - let mut all_resolved = true; - let window = cfg.bulk_prepare_window.max(1); - - for slot in (floor + 1)..=ceiling { - if cancel.is_cancelled() { - warn!( - group_id, - term, slot, slots_repaired, "bulk phase 1 cancelled (step down)" - ); - return; - } - if slots_repaired >= window { - tokio::task::yield_now().await; - slots_repaired = 0; - } - - // Issue Phase-1 Prepare at ballot (0, me) under term T with an - // empty payload so any adopted foreign value comes strictly - // from a remote's previously-Accepted entry; if none exist - // the entry stays a NoOp (we re-tag below). - let attempt = self - .run_prepare_phase(replica, slot, bytes::Bytes::new(), quorum, 0) - .await; - let mut entry = match attempt { - PrepareAttempt::Proceed { entry, .. } => entry, - PrepareAttempt::Retry { error, .. } | PrepareAttempt::Fail { error } => { - all_resolved = false; - warn!( - group_id, - term, - slot, - error = error.keyword(), - "bulk phase 1 prepare failed; will be retried by next leader" - ); - continue; - } - }; - - entry.term = term; - - match self.run_accept_phase(replica, &entry, quorum).await { - AcceptAttempt::Chosen => { - replica.learn_chosen(&entry, &[]).await; - self.fan_out_chosen_notice(&entry, group_id); - slots_repaired += 1; - } - AcceptAttempt::Retry { error, .. } | AcceptAttempt::Fail { error } => { - all_resolved = false; - warn!( - group_id, - term, - slot, - error = error.keyword(), - "bulk phase 1 accept failed; will be retried by next leader" - ); - } - } - } - - let next = ceiling.saturating_add(1); - self.next_slot.fetch_max(next, Ordering::AcqRel); - if !all_resolved { - warn!( - group_id, - term, ceiling, "bulk phase 1 incomplete; leader recovery remains closed" - ); - return; - } - tokio::select! { - () = replica.await_apply_fence(ceiling) => {} - () = cancel.cancelled() => return, - } - self.leader_read_ready.store(true, Ordering::Release); - // R65: advance `known_commit_slot` to the leader's own - // `contiguous_chosen` after the sweep. Leaders don't receive - // heartbeats or ChosenNotice, so without this the apply loop - // would never learn about slots the leader accepted as a follower - // (before winning the election) or slots resolved by the sweep. - // All slots up to `contiguous_chosen` are now confirmed chosen by - // the quorum that responded to Prepare — safe to apply. - let cc = replica.contiguous_chosen(); - replica.advance_known_commit_slot(cc); - replica.wake_apply_loop(); - debug!( - group_id, - term, - ceiling, - next_slot = next, - contiguous_chosen = cc, - "bulk phase 1 done" - ); + cfg, + cancel, + ) + .await; } } diff --git a/lib/crowdb-kv/src/cluster/group_election/recovery.rs b/lib/crowdb-kv/src/cluster/group_election/recovery.rs new file mode 100644 index 000000000..bcd3b56a8 --- /dev/null +++ b/lib/crowdb-kv/src/cluster/group_election/recovery.rs @@ -0,0 +1,113 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Bounded concurrent recovery of a new leader's open prefix. + +use crate::{ + cluster::group::{AcceptAttempt, PrepareAttempt, PxGroup}, + common::config::PxElectionConfig, +}; +use futures::{stream, StreamExt}; +use std::sync::atomic::Ordering; +use tokio_util::sync::CancellationToken; +use tracing::{debug, warn}; + +impl PxGroup { + #[tracing::instrument(level = "info", skip_all, fields(s = self.log_store_id().unwrap_or(0), g = self.group_id, replica = self.local_replica().id, term))] + pub(super) async fn recover_open_prefix( + &self, + term: u64, + peer_floor: u64, + peer_ceiling: u64, + cfg: PxElectionConfig, + cancel: CancellationToken, + ) { + let replica = self.local_replica(); + // Peers' committed frontiers carry no payload. Start at our own + // frontier so quorum Phase 1 recovers every missing chosen value. + let floor = replica.contiguous_chosen(); + let ceiling = replica + .highest_seen_slot() + .max(self.next_slot.load(Ordering::Acquire).saturating_sub(1)) + .max(peer_ceiling); + debug!(term, floor, ceiling, peer_floor, "bulk phase 1 start"); + if ceiling > floor { + let window = usize::try_from(cfg.bulk_prepare_window.max(1)).unwrap_or(usize::MAX); + let mut slots = stream::iter((floor + 1)..=ceiling) + .map(|slot| self.recover_prefix_slot(term, slot, &cancel)) + .buffer_unordered(window); + let mut resolved = true; + loop { + tokio::select! { + biased; + () = cancel.cancelled() => return, + result = slots.next() => match result { + Some(success) => resolved &= success, + None => break, + }, + } + } + self.next_slot + .fetch_max(ceiling.saturating_add(1), Ordering::AcqRel); + if !resolved { + warn!( + term, + ceiling, "bulk phase 1 incomplete; leader recovery remains closed" + ); + return; + } + } + tokio::select! { + biased; + () = cancel.cancelled() => return, + () = replica.await_apply_fence(ceiling.max(floor)) => {} + } + if replica.current_term_snapshot() != term || !replica.is_leader() { + return; + } + self.leader_read_ready.store(true, Ordering::Release); + replica.advance_known_commit_slot(replica.contiguous_chosen()); + replica.wake_apply_loop(); + debug!(term, ceiling, "bulk phase 1 done"); + } + + async fn recover_prefix_slot(&self, term: u64, slot: u64, cancel: &CancellationToken) -> bool { + let replica = self.local_replica(); + if cancel.is_cancelled() || replica.current_term_snapshot() != term || !replica.is_leader() { + return false; + } + let mut entry = match self + .run_prepare_phase(replica, slot, bytes::Bytes::new(), self.quorum(), 0) + .await + { + PrepareAttempt::Proceed { entry, .. } => entry, + PrepareAttempt::Retry { error, .. } | PrepareAttempt::Fail { error } => { + warn!( + slot, + error = error.keyword(), + "bulk phase 1 prepare failed; leader recovery remains closed" + ); + return false; + } + }; + if cancel.is_cancelled() || replica.current_term_snapshot() != term || !replica.is_leader() { + return false; + } + entry.term = term; + match self.run_accept_phase(replica, &entry, self.quorum()).await { + AcceptAttempt::Chosen => { + replica.learn_chosen(&entry, &[]).await; + self.fan_out_chosen_notice(&entry, self.group_id); + true + } + AcceptAttempt::Retry { error, .. } | AcceptAttempt::Fail { error } => { + warn!( + slot, + error = error.keyword(), + "bulk phase 1 accept failed; leader recovery remains closed" + ); + false + } + } + } +} diff --git a/lib/crowdb-kv/src/cluster/group_election_leader.rs b/lib/crowdb-kv/src/cluster/group_election_leader.rs index a4d0afba7..ca1d6586e 100644 --- a/lib/crowdb-kv/src/cluster/group_election_leader.rs +++ b/lib/crowdb-kv/src/cluster/group_election_leader.rs @@ -522,7 +522,10 @@ impl PxGroup { } // Keep at most one repair in flight. Prepare or // persistence IO must not occupy the heartbeat loop. - if gap_repair.as_ref().map_or(true, tokio::task::JoinHandle::is_finished) { + if self.leader_read_ready() + && recovery_task.as_ref().map_or(true, tokio::task::JoinHandle::is_finished) + && gap_repair.as_ref().map_or(true, tokio::task::JoinHandle::is_finished) + { gap_repair = self.start_gap_repair(&tenure_cancel); } } diff --git a/lib/crowdb-kv/src/cluster/group_fetchgap.rs b/lib/crowdb-kv/src/cluster/group_fetchgap.rs index cbe3b850b..068e345d5 100644 --- a/lib/crowdb-kv/src/cluster/group_fetchgap.rs +++ b/lib/crowdb-kv/src/cluster/group_fetchgap.rs @@ -4,17 +4,26 @@ #![allow(clippy::too_many_lines)] #![allow(clippy::cast_possible_truncation)] -use std::sync::atomic::Ordering; -use std::sync::Weak; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Weak, +}; use std::time::Duration; use tokio_util::sync::CancellationToken; -use tracing::{debug, warn}; +use tracing::debug; use crate::cluster::group::PxGroup; -use crate::paxos::roles::Acceptor; use crate::paxos::PxGroupId; +struct FetchGapInFlight(Arc); + +impl Drop for FetchGapInFlight { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } +} + /// R65: Follower-side `FetchGap` driver. Periodically checks the local /// replica's `gap_slots` set and sends `FetchGap` to the leader for each /// gap. On reply, overwrites the stale/missing value and wakes the apply @@ -25,6 +34,7 @@ pub(crate) async fn run_fetchgap_driver( cancel: CancellationToken, ) { let tick = Duration::from_millis(10); + let mut large_backlog = false; loop { if cancel.is_cancelled() { return; @@ -49,21 +59,17 @@ pub(crate) async fn run_fetchgap_driver( let Some(remote) = group.get_remote_replica(leader_id) else { continue; }; - // R65: snapshot fallback — if gap count exceeds the threshold, - // skip FetchGap and log a warning. A full automatic snapshot - // install for running replicas is a follow-up; for now the - // follower will eventually catch up via ChosenNotice + heartbeat. + // Running replicas have no atomic snapshot replacement yet. Keep + // draining large backlogs with the existing fixed inflight bound. let gap_count = group.local_replica().gap_slots.lock().len(); let threshold = group.config.election.catchup_snapshot_threshold; - if gap_count > threshold as usize { - warn!( + if gap_count > threshold as usize && !large_backlog { + debug!( group_id, - gap_count, - threshold, - "gap count exceeds snapshot threshold; FetchGap skipped (snapshot fallback not yet automatic)" + gap_count, threshold, "snapshot catch-up is unavailable; continuing bounded FetchGap" ); - continue; } + large_backlog = gap_count > threshold as usize; // Clone the crowdb-rpc transport + endpoint so the spawned task // can call `transport.send_fetch_gap()` directly. let rpc_transport = remote.rpc_transport().cloned(); @@ -81,6 +87,7 @@ pub(crate) async fn run_fetchgap_driver( let gap_slots = replica.gap_slots.clone(); let learner = replica.learner.clone(); let acceptor = replica.acceptor.clone(); + let wal = replica.wal().cloned(); let apply_notify = replica.apply_notify.clone(); let fetchgap_received = replica .replication_handles @@ -95,16 +102,17 @@ pub(crate) async fn run_fetchgap_driver( let rpc_transport_clone = rpc_transport.clone(); let rpc_endpoint_clone = rpc_endpoint.clone(); let rpc_timeout_clone = rpc_timeout; - let fetchgap_inflight_clone = fetchgap_inflight.clone(); + let inflight = FetchGapInFlight(fetchgap_inflight.clone()); let gap_slots_clone = gap_slots.clone(); let learner_clone = learner.clone(); let acceptor_clone = acceptor.clone(); + let wal_clone = wal.clone(); let apply_notify_clone = apply_notify.clone(); let fetchgap_received_clone = fetchgap_received.clone(); let fetchgap_failed_clone = fetchgap_failed.clone(); tokio::spawn(async move { + let _inflight = inflight; let Some(ref transport) = rpc_transport_clone else { - fetchgap_inflight_clone.fetch_sub(1, Ordering::AcqRel); if let Some(c) = &fetchgap_failed_clone { c.inc(); } @@ -132,7 +140,6 @@ pub(crate) async fn run_fetchgap_driver( )) }) .and_then(|r| r); - fetchgap_inflight_clone.fetch_sub(1, Ordering::AcqRel); match result { Ok(resp) => { if let Some(c) = &fetchgap_received_clone { @@ -153,7 +160,30 @@ pub(crate) async fn run_fetchgap_driver( term: resp.term, payload: resp.payload, }; - acceptor_clone.accept(&entry).await; + // Historical chosen values may belong to an older + // election term. Persist their original entry before + // exposing an applied frontier or snapshot coverage. + let persisted = if let Some(wal) = &wal_clone { + match wal + .append(&crate::wal::record::WALRecord::from_accepted(group_id, &entry)) + .await + { + Ok(_) => true, + Err(error) => { + tracing::error!(group_id, slot, %error, "FetchGap persistence failed; retaining gap for later recovery"); + false + } + } + } else { + true + }; + if !persisted || !acceptor_clone.install_chosen(&entry).await { + gap_slots_clone.lock().insert(slot); + if let Some(c) = &fetchgap_failed_clone { + c.inc(); + } + return; + } learner_clone.update_chosen_frontier(resp.slot, resp.term); apply_notify_clone.notify_one(); } diff --git a/lib/crowdb-kv/src/cluster/group_tests.rs b/lib/crowdb-kv/src/cluster/group_tests.rs index f57e9e1f8..31cdbe8e4 100644 --- a/lib/crowdb-kv/src/cluster/group_tests.rs +++ b/lib/crowdb-kv/src/cluster/group_tests.rs @@ -14,6 +14,14 @@ use crate::paxos::PxNodeId; /// repair step, and peer-applied injection — to integration tests under /// `tests/` without permanently widening the production public API. impl PxGroup { + /// Observe a missing chosen value without delivering its Accepted frame. + pub fn note_missing_chosen_for_tests(&self, slot: SlotIndex) { + let replica = self.local_replica(); + replica.record_gap(slot); + replica.note_chosen(slot, replica.current_term_snapshot()); + replica.wake_apply_loop(); + } + /// Whether a conditional mutation currently owns one key's admission. #[must_use] pub fn conditional_write_pending_for_tests(&self, key: &[u8]) -> bool { diff --git a/lib/crowdb-kv/src/cluster/local_replica_replay.rs b/lib/crowdb-kv/src/cluster/local_replica_replay.rs index 797318ce9..74eec8dfc 100644 --- a/lib/crowdb-kv/src/cluster/local_replica_replay.rs +++ b/lib/crowdb-kv/src/cluster/local_replica_replay.rs @@ -63,22 +63,25 @@ impl PxLocalReplica { let resume_from = engine.resume_from_slot(); let replica = Self::new_with_learner(id, role, PxLearner::with_engine(engine)); - // Pass 1: rebuild acceptor (promise + accept) state from the WAL. + // Accepted history and later promises are independent durable facts. + // FetchGap may learn an old chosen value after a higher promise; replay + // must retain that value and the higher fence regardless of record order. + for record in replay + .records + .iter() + .filter(|record| record.record_type == crate::wal::record::RecordType::Accepted) + { + let entry = record.to_log_entry().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "restore replay accepted missing log entry", + ) + })?; + let _ = replica.acceptor.accept(&entry).await; + } for record in &replay.records { - match record.record_type { - crate::wal::record::RecordType::Promised => { - let _ = replica.acceptor.prepare(record.slot, record.ballot).await; - } - crate::wal::record::RecordType::Accepted => { - let entry = record.to_log_entry().ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "restore replay accepted missing log entry", - ) - })?; - let _ = replica.acceptor.accept(&entry).await; - } - crate::wal::record::RecordType::VoteGranted => {} + if record.record_type == crate::wal::record::RecordType::Promised { + let _ = replica.acceptor.prepare(record.slot, record.ballot).await; } } @@ -124,25 +127,19 @@ impl PxLocalReplica { // legitimately touches. There is no safe way to "fall back" to // replaying it once the engine is past that floor. // - // Seed the frontier to `(resume_from, term-at-resume_from)` via - // `seed_resume_frontier` when the just-rebuilt acceptor has an - // accepted entry at that exact slot (the expected case: an engine - // can only ever have durably applied a slot that was itself - // accepted and WAL-logged, and Pass 1 rebuilds the *entire* WAL - // history). If it's missing (e.g. a WAL segment lost/GC'd after the - // engine already durably flushed that slot -- not expected, but not - // an invariant this restore path should trust blindly), leave the - // frontier at the fresh learner's default (`0`) rather than guess a - // term: under-reporting `contiguous_chosen`/`last_chosen_term` only - // costs more conservative heartbeat catch-up / safe-read bounds, - // never incorrectness, unlike attempting the skipped replay. + // A recovered engine proves the complete durable prefix even when + // WAL reclamation removed its boundary record. Preserve that prefix; + // requesting it again can never succeed once peers reclaimed it too. + // Term zero explicitly records unknown historical term. Election + // freshness uses the durable acceptor tip, not this chosen watermark. let highest = replica.acceptor.highest_seen_slot(); - let resume_from = resume_from.min(highest); let start_slot = if resume_from > 0 { - if let Some(entry) = replica.acceptor.accepted_at(resume_from) { - replica.learner.seed_resume_frontier(resume_from, entry.term); - } - resume_from + 1 + let term = replica + .acceptor + .accepted_at(resume_from) + .map_or(0, |entry| entry.term); + replica.learner.seed_resume_frontier(resume_from, term); + resume_from.saturating_add(1) } else { 1 }; diff --git a/lib/crowdb-kv/src/common/config.rs b/lib/crowdb-kv/src/common/config.rs index e918d1d55..064475d1b 100644 --- a/lib/crowdb-kv/src/common/config.rs +++ b/lib/crowdb-kv/src/common/config.rs @@ -259,12 +259,11 @@ pub struct PxElectionConfig { /// Maximum admissible clock skew across the cluster. Subtracted from the /// leader's `lease_read_until` to remain safe under skew. pub max_clock_skew_ms: u64, - /// Slots scanned per bulk-Phase-1 batch (new-leader open-prefix repair). + /// Maximum concurrent slots in new-leader open-prefix recovery. pub bulk_prepare_window: u64, - /// R65: gap count threshold for snapshot fallback. When a follower's - /// gap count exceeds this, `FetchGap` is skipped and a warning is logged - /// (full automatic snapshot install for running replicas is a - /// follow-up). Default matches `bulk_prepare_window` (1024). + /// Gap count at which snapshot catch-up would be useful. Atomic snapshot + /// replacement for running replicas is not implemented, so bounded + /// `FetchGap` continues above this diagnostic threshold. Default: 1024. pub catchup_snapshot_threshold: u64, /// Test-only override: when `true`, the election driver task is not /// spawned. Used by `testkit::cluster::start_cluster` to keep legacy M1/M2 diff --git a/lib/crowdb-kv/src/paxos/acceptor.rs b/lib/crowdb-kv/src/paxos/acceptor.rs index 96c3f9616..54eb9df54 100644 --- a/lib/crowdb-kv/src/paxos/acceptor.rs +++ b/lib/crowdb-kv/src/paxos/acceptor.rs @@ -110,6 +110,29 @@ impl PxAcceptor { self.slots.len() } + /// Retain a value proved chosen by the current recovered leader without + /// answering Accept or weakening a later promise. A higher accepted ballot + /// may already carry the same chosen value; conflicting evidence fails closed. + pub(crate) async fn install_chosen(&self, entry: &PxLogEntry) -> bool { + let _ = self.prepare(entry.slot, entry.ballot).await; + let Some(guard) = self.open_slot(entry.slot) else { + return false; + }; + let node = guard.value(); + self.bump_highest_seen(entry.slot); + loop { + let previous = node.accepted.load(Ordering::Acquire); + if let Some(accepted) = node.accepted() { + if accepted.ballot >= entry.ballot { + return accepted.payload == entry.payload; + } + } + if node.cas_accepted(previous, entry.clone()).is_ok() { + return true; + } + } + } + fn open_slot(&self, slot: SlotIndex) -> Option> { if slot < self.trim_slot() { return None; diff --git a/lib/crowdb-kv/tests/group_test.rs b/lib/crowdb-kv/tests/group_test.rs index 274c7090b..f4a7aa059 100644 --- a/lib/crowdb-kv/tests/group_test.rs +++ b/lib/crowdb-kv/tests/group_test.rs @@ -86,6 +86,12 @@ mod g4_learner_stream; #[path = "group_test/g5_recovery_test.rs"] mod g5_recovery; +#[path = "group_test/recovery_backlog_test.rs"] +mod recovery_backlog; + +#[path = "group_test/fetch_gap_durability_test.rs"] +mod fetch_gap_durability; + #[path = "group_test/g6_reconfig_test.rs"] mod g6_reconfig; diff --git a/lib/crowdb-kv/tests/group_test/fetch_gap_durability_test.rs b/lib/crowdb-kv/tests/group_test/fetch_gap_durability_test.rs new file mode 100644 index 000000000..50422cdcb --- /dev/null +++ b/lib/crowdb-kv/tests/group_test/fetch_gap_durability_test.rs @@ -0,0 +1,153 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::common::cluster::start_cluster; +use crowdb_kv::{ + cluster::{group::PxGroup, KvServer, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}, + paxos::roles::{Acceptor, Learner, PxBallot, PxLogEntry}, + wal::{replay::replay_group, IoBackend, MemBlockDevice, WalConfig, WalEngine}, +}; +use std::{path::PathBuf, sync::Arc, time::Duration}; + +#[tokio::test] +async fn fetched_historical_chosen_value_survives_a_follower_restart() { + restart_fetched_value(false).await; +} + +#[tokio::test] +async fn learned_historical_value_preserves_a_higher_promise_across_restart() { + restart_fetched_value(true).await; +} + +async fn restart_fetched_value(higher_promise: bool) { + let cluster = start_cluster(&[1, 2], 1).await; + let leader = cluster.leader().get_group(1).unwrap(); + let entry = PxLogEntry { + slot: 1, + ballot: PxBallot::new(1, 1), + term: 0, + payload: bytes::Bytes::from_static(b"\x01\x00\x00\x03\x00\x00\x00key\x05\x00\x00\x00value"), + }; + leader.local_replica().acceptor.accept(&entry).await; + leader.local_replica().learner.learn(entry.clone(), &[]).await; + let backend = Arc::new(IoBackend::MemBlock(MemBlockDevice::new())); + let disks = vec![PathBuf::from("/fetch-gap")]; + let wal = WalEngine::create( + backend.clone(), + WalConfig { + wal_disks: disks.clone(), + ..Default::default() + }, + 1, + ) + .await + .unwrap(); + let mut replica = PxLocalReplica::new(99, PxLocalReplicaRole::Follower); + replica.become_follower(7); + replica.set_believed_leader(1); + if higher_promise { + let stale = PxLogEntry { + ballot: PxBallot::new(0, 99), + payload: bytes::Bytes::from_static(b"\x01\x00\x00\x03\x00\x00\x00key\x05\x00\x00\x00stale"), + ..entry.clone() + }; + replica.acceptor.accept(&stale).await; + wal.append(&crowdb_kv::wal::WALRecord::from_accepted(1, &stale)) + .await + .unwrap(); + let promise = PxBallot::new(9, 99); + replica.acceptor.prepare(1, promise).await; + wal.append(&crowdb_kv::wal::WALRecord::from_promised(1, 7, 1, promise)) + .await + .unwrap(); + } + replica.set_wal(wal.clone()); + replica.persist_current_vote().await; + let mut group = PxGroup::new(1, replica); + group.set_remote_replicas(vec![PxRemoteReplica::new( + 1, + cluster.leader().listen_addr().unwrap().to_string(), + )]); + group + .get_remote_replica(1) + .unwrap() + .set_rpc_transport(cluster.leader().rpc_transport().unwrap()); + let group = Arc::new(group); + group.start_fetchgap_driver().await; + group.note_missing_chosen_for_tests(1); + tokio::time::timeout(Duration::from_secs(3), async { + while group.local_replica().contiguous_applied() != 1 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!( + group.local_replica().learner.engine_get(b"key").await, + Some((1, b"value".to_vec())) + ); + assert_eq!( + group.local_replica().promised_at(1).await, + Some(if higher_promise { + PxBallot::new(9, 99) + } else { + entry.ballot + }) + ); + group.shutdown(Duration::from_secs(3)).await; + wal.seal_all().await.unwrap(); + let replay = replay_group(&backend, &disks, 1).await.unwrap(); + assert!( + replay + .records + .iter() + .any(|record| record.to_log_entry() == Some(entry.clone())), + "FetchGap advanced the applied frontier without persisting the recovered entry" + ); + let restored = PxLocalReplica::restore_from_replay(99, PxLocalReplicaRole::Follower, &replay) + .await + .unwrap(); + assert_eq!(restored.contiguous_applied(), 1); + assert_eq!(restored.current_term_snapshot(), 7); + assert_eq!( + restored.learner.engine_get(b"key").await, + Some((1, b"value".to_vec())) + ); + assert_eq!(restored.accepted_at(1).await, Some(entry)); + if higher_promise { + assert_eq!(restored.promised_at(1).await, Some(PxBallot::new(9, 99))); + } + cluster.shutdown().await; +} + +#[tokio::test] +async fn fetch_gap_does_not_advertise_an_unproven_accepted_value_as_chosen() { + let cluster = start_cluster(&[1, 2], 1).await; + let leader = cluster.leader().get_group(1).unwrap(); + let entry = PxLogEntry { + slot: 1, + ballot: PxBallot::new(1, 1), + term: 0, + payload: bytes::Bytes::new(), + }; + leader.local_replica().acceptor.accept(&entry).await; + let transport = cluster.leader().rpc_transport().unwrap(); + let endpoint = cluster.leader().listen_addr().unwrap().to_string(); + let destination = crowdb_kv::rpc::ReplicaDestination { + endpoint: &endpoint, + replica_id: 1, + }; + assert!( + transport.send_fetch_gap(destination, 1, 1, 0, 1).await.is_err(), + "an accepted value without a chosen proof must not be advertised as chosen" + ); + leader.local_replica().learner.learn(entry.clone(), &[]).await; + let destination = crowdb_kv::rpc::ReplicaDestination { + endpoint: &endpoint, + replica_id: 1, + }; + let reply = transport.send_fetch_gap(destination, 1, 1, 0, 1).await.unwrap(); + assert_eq!(reply.slot, entry.slot); + assert_eq!(reply.payload, entry.payload); + cluster.shutdown().await; +} diff --git a/lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs b/lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs index 5401eaf69..6a678a14a 100644 --- a/lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs +++ b/lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs @@ -12,6 +12,7 @@ use std::time::{Duration, Instant}; use bytes::Bytes; +use crowdb_kv::cluster::group_election::LeaderElection; use crowdb_kv::cluster::replica::StepDownRequestPayload; use crowdb_kv::rpc::{KvGetRequest, KvSetRequest}; @@ -21,7 +22,13 @@ async fn wait_for_leader(cluster: &TestCluster, timeout: Duration) -> Option +// Licensed under the Apache License, Version 2.0. + +use crate::common::cluster::start_cluster; +use crowdb_kv::{ + cluster::group_election::LeaderElection, + common::config::PxElectionConfig, + paxos::roles::{Acceptor, Learner, PxBallot, PxLogEntry}, +}; +use std::time::Duration; +use tokio_util::sync::CancellationToken; + +#[tokio::test] +async fn bulk_recovery_fills_its_bounded_window_before_waiting_for_peer_io() { + let cluster = start_cluster(&[1, 2, 3], 1).await; + let group = cluster.leader().get_group(1).unwrap(); + let config = PxElectionConfig { + bulk_prepare_window: 4, + ..group.election_config() + }; + let recovery = group.run_bulk_phase1(0, 0, 12, config, CancellationToken::new()); + tokio::pin!(recovery); + // On this current-thread runtime the peers cannot answer until this poll + // yields. Slow IO at the first slot must leave all four slots dispatched. + for _ in 0..16 { + assert!(futures::poll!(&mut recovery).is_pending()); + if group.local_replica().highest_seen_slot() == 4 { + break; + } + } + assert_eq!(group.local_replica().highest_seen_slot(), 4); + tokio::time::timeout(Duration::from_secs(3), recovery) + .await + .expect("bounded recovery must finish"); + assert_eq!(group.local_replica().contiguous_applied(), 12); + assert!(group.leader_read_ready()); + cluster.shutdown().await; +} + +#[tokio::test] +async fn cancelling_a_pending_bulk_window_does_not_accept_its_prepared_slots() { + let cluster = start_cluster(&[1, 2, 3], 1).await; + let group = cluster.leader().get_group(1).unwrap(); + let config = PxElectionConfig { + bulk_prepare_window: 4, + ..group.election_config() + }; + let cancel = CancellationToken::new(); + let recovery = group.run_bulk_phase1(0, 0, 12, config, cancel.clone()); + tokio::pin!(recovery); + assert!(futures::poll!(&mut recovery).is_pending()); + cancel.cancel(); + recovery.await; + assert_eq!(group.local_replica().contiguous_applied(), 0); + for slot in 1..=12 { + assert!(group.local_replica().accepted_at(slot).await.is_none()); + } + cluster.shutdown().await; +} + +#[tokio::test] +async fn follower_fetches_a_backlog_above_the_unimplemented_snapshot_threshold() { + let cluster = start_cluster(&[1, 2, 3], 1).await; + let leader = cluster.leader().get_group(1).unwrap(); + let follower = cluster.followers()[0].get_group(1).unwrap(); + let last = leader.election_config().catchup_snapshot_threshold + 1; + for slot in 1..=last { + let entry = PxLogEntry { + slot, + ballot: PxBallot::new(1, 1), + term: 0, + payload: bytes::Bytes::new(), + }; + leader.local_replica().acceptor.accept(&entry).await; + leader.local_replica().learner.learn(entry, &[]).await; + follower.note_missing_chosen_for_tests(slot); + } + tokio::time::timeout(Duration::from_secs(3), async { + while follower.local_replica().contiguous_applied() != last { + tokio::task::yield_now().await; + } + }) + .await + .expect("snapshot threshold must not permanently disable bounded catch-up"); + for slot in 1..=last { + assert_eq!( + follower.local_replica().accepted_at(slot).await, + leader.local_replica().accepted_at(slot).await + ); + } + cluster.shutdown().await; +} diff --git a/lib/crowdb-kv/tests/rpc_migration_test.rs b/lib/crowdb-kv/tests/rpc_migration_test.rs index d54782206..171d2fb33 100644 --- a/lib/crowdb-kv/tests/rpc_migration_test.rs +++ b/lib/crowdb-kv/tests/rpc_migration_test.rs @@ -303,10 +303,17 @@ async fn crowdb_rpc_fetch_gap() { .send_prepare(1, ballot, 0, 1, 0) .await .expect("follower prepare"); - follower_remote + let accepted = follower_remote .send_accept(&entry, 1, 0) .await .expect("follower accept"); + assert!(matches!( + accepted, + crowdb_kv::paxos::roles::PxAcceptReply::Accepted { .. } + )); + // Both voters have accepted. Publish that quorum decision before serving + // the chosen value; an Accepted response alone is not a learner proof. + leader_replica.learn_chosen(&entry, &[]).await; // Use the follower's remote (points at leader) to fetch gap. let follower_group = cluster.follower.get_group(1).expect("follower group"); diff --git a/lib/crowdb-kv/tests/wal_test/replay_test.rs b/lib/crowdb-kv/tests/wal_test/replay_test.rs index bb0d527ab..dafaf7c79 100644 --- a/lib/crowdb-kv/tests/wal_test/replay_test.rs +++ b/lib/crowdb-kv/tests/wal_test/replay_test.rs @@ -382,16 +382,63 @@ async fn restore_from_replay_with_engine_resumes_from_last_applied_slot() { assert!(restored.accepted_at(3).await.is_some()); } -/// Defensive fallback: if the engine's resume floor doesn't line up with an -/// accepted entry in the WAL-rebuilt acceptor (not expected in practice -- -/// an engine can only durably apply a slot that was itself accepted and -/// WAL-logged -- but not a correctness invariant this restore path should -/// ever trust blindly), restore still skips straight to `resume_from + 1` -/// (never re-attempts the skipped prefix -- see the "no safe fallback" -/// rationale on `restore_from_replay_with_engine`) but leaves the frontier -/// at its conservative fresh-learner default instead of guessing a term. +#[tokio::test] +async fn durable_snapshot_prefix_survives_reclaimed_wal_boundary_records() { + let tmp = crowdb_test_harness::test_dirs::tempdir_in_test_data("snapshot-reclaimed-boundary"); + let options = CrowdbTreeConfig { + path: Some(tmp.path().to_string_lossy().into_owned()), + ..Default::default() + }; + let engine = CrowdbTreeEngine::open(&options).unwrap(); + for slot in 1..=3 { + engine + .apply( + slot, + &Batch::decode(&Bytes::from(encode_put_payload( + b"durable", + &[u8::try_from(slot).unwrap()], + ))), + ) + .into_ready() + .unwrap(); + } + engine.flush(); + assert_eq!(engine.persist_snapshot(), 3); + drop(engine); + let backend = sim_backend(); + let disks = vec![PathBuf::from("/reclaimed")]; + let wal = WalEngine::create(backend.clone(), test_config(&disks), 1) + .await + .unwrap(); + wal.seal_all().await.unwrap(); + let replay = replay_group(&backend, &disks, 1).await.unwrap(); + assert!(replay.records.is_empty()); + let restored = PxLocalReplica::restore_from_replay_with_engine( + 7, + PxLocalReplicaRole::Follower, + &replay, + Box::new(CrowdbTreeEngine::open(&options).unwrap()), + ) + .await + .unwrap(); + assert_eq!(restored.learner.engine_get(b"durable").await, Some((3, vec![3]))); + assert_eq!( + restored.contiguous_applied(), + 3, + "snapshot prefix must not become an unfillable WAL gap" + ); + assert_eq!(restored.contiguous_chosen(), 3); + assert_eq!( + restored.last_chosen_term(), + 0, + "a reclaimed boundary has an unknown historical term" + ); +} + +/// The durable engine owns its covered prefix even when the retained WAL +/// omits the boundary. Later entries must extend that prefix normally. #[tokio::test(flavor = "current_thread", start_paused = true)] -async fn restore_from_replay_with_engine_falls_back_when_resume_slot_has_no_accepted_entry() { +async fn restore_from_replay_with_engine_keeps_prefix_when_resume_slot_has_no_accepted_entry() { let backend = sim_backend(); let disks = vec![PathBuf::from("/wal")]; let config = test_config(&disks); @@ -414,22 +461,15 @@ async fn restore_from_replay_with_engine_falls_back_when_resume_slot_has_no_acce // accepted entry for. `last_applied_slot()` is itself a contiguous // watermark (crowdb-tree folds `received_slots_` forward from its own // frontier), so reaching floor 2 legitimately requires applying slot 1 - // too -- modeling an engine that durably has extra data at slot 2 with - // no independent WAL corroboration (e.g. a lost/truncated WAL record), - // rather than an outright-impossible-via-the-API state. + // too. The retained WAL may omit the already durable boundary after + // reclamation; it need not corroborate that complete prefix again. let engine = CrowdbTreeEngine::open(&CrowdbTreeConfig::default()).expect("open crowdb-tree engine"); engine - .apply( - 1, - &Batch::decode(&Bytes::from(encode_put_payload(b"phantom1", b"x"))), - ) + .apply(1, &Batch::decode(&Bytes::from(encode_put_payload(b"k1", b"v1")))) .into_ready() .unwrap(); engine - .apply( - 2, - &Batch::decode(&Bytes::from(encode_put_payload(b"phantom2", b"y"))), - ) + .apply(2, &Batch::decode(&Bytes::from(encode_put_payload(b"k2", b"v2")))) .into_ready() .unwrap(); engine.handle().flush().expect("flush"); @@ -448,21 +488,22 @@ async fn restore_from_replay_with_engine_falls_back_when_resume_slot_has_no_acce .await .unwrap(); - // Slots 1 and 2 are skipped outright (never re-attempted -- see the - // rationale above), so slot 1's WAL value is *not* recovered here: an - // honest, safe degradation for this assumed-impossible mismatch, not a - // silent correctness violation (nothing that was ever safely writable - // is lost or corrupted). Slot 3, past the skipped prefix, replays - // normally. - assert_eq!(restored.learner.engine_get(b"k1").await.map(|(_, v)| v), None); + // Slots 1 and 2 retain the engine's complete prefix. Slot 3 extends it + // normally without needing the reclaimed boundary record. + assert_eq!( + restored.learner.engine_get(b"k1").await.map(|(_, v)| v), + Some(b"v1".to_vec()) + ); + assert_eq!( + restored.learner.engine_get(b"k2").await.map(|(_, v)| v), + Some(b"v2".to_vec()) + ); assert_eq!( restored.learner.engine_get(b"k3").await.map(|(_, v)| v), Some(b"v3".to_vec()) ); - // No accepted entry at the resume floor (slot 2) to seed a term from, so - // the frontier stays at the fresh-learner default until slot 3's - // out-of-order `learn()` bumps the max-ever-seen high-water mark. - assert_eq!(restored.contiguous_chosen(), 0); + assert_eq!(restored.contiguous_chosen(), 3); + assert_eq!(restored.contiguous_applied(), 3); assert_eq!(restored.last_chosen_slot(), 3); assert_eq!(restored.last_chosen_term(), 1); } diff --git a/lib/crowdb-protocol/src/mgmt/node/deployment.rs b/lib/crowdb-protocol/src/mgmt/node/deployment.rs index c014c6275..32f6d460f 100644 --- a/lib/crowdb-protocol/src/mgmt/node/deployment.rs +++ b/lib/crowdb-protocol/src/mgmt/node/deployment.rs @@ -36,7 +36,7 @@ impl FormationPolicy { /// # Errors /// Rejects unsupported versions, malformed scopes and voter counts. pub fn validate(&self) -> Result<(), &'static str> { - if self.version != 1 || !matches!(self.initial_voters, 1 | 3) || !valid_cluster(&self.cluster) { + if self.version != 1 || !matches!(self.initial_voters, 1 | 3 | 5) || !valid_cluster(&self.cluster) { return Err("invalid node formation policy"); } Ok(()) @@ -138,8 +138,8 @@ impl NodeDeploymentPolicy { if self.version != 1 { return Err("unsupported node deployment policy version"); } - if !matches!(self.initial_voters, 1 | 3) { - return Err("initial voters must be one or three"); + if !matches!(self.initial_voters, 1 | 3 | 5) { + return Err("initial voters must be one, three or five"); } if !valid_cluster(&self.cluster) { return Err("cluster scope must contain 1 to 48 name characters"); diff --git a/lib/crowdb-protocol/src/mgmt/node/setup.rs b/lib/crowdb-protocol/src/mgmt/node/setup.rs index b9b8a26be..847c0de49 100644 --- a/lib/crowdb-protocol/src/mgmt/node/setup.rs +++ b/lib/crowdb-protocol/src/mgmt/node/setup.rs @@ -120,6 +120,16 @@ pub struct NodeSetupChoices { } impl NodeSetupChoices { + /// Whether the selected storage or services require ordinary metadata. + #[must_use] + pub fn requires_default_preparation(&self) -> bool { + !matches!(self.storage, SetupStorage::Diskless) + || self + .services + .iter() + .any(|service| service.enabled && service.kind != SetupServiceKind::PaxosKv) + } + /// Return the management and RPC listeners reserved for the bootstrap KV. /// /// The node launcher may shift these listeners for host-network containers diff --git a/lib/crowdb-protocol/tests/node_deployment_test.rs b/lib/crowdb-protocol/tests/node_deployment_test.rs index 84ed3ef8c..e2a70df6b 100644 --- a/lib/crowdb-protocol/tests/node_deployment_test.rs +++ b/lib/crowdb-protocol/tests/node_deployment_test.rs @@ -243,3 +243,29 @@ fn accepted_serialization_requires_explicit_version_and_complete_policy() { }; assert!(current.resolve(&future, None).is_err()); } + +#[test] +fn five_voter_policy_retains_the_accepted_formation_and_rejects_other_counts() { + let omitted = NodeDeploymentOverrides::default(); + let explicit = NodeDeploymentOverrides { + initial_voters: Some(5), + ..Default::default() + }; + let accepted = explicit.resolve(&omitted, None).unwrap(); + accepted.formation().validate().unwrap(); + assert_eq!(accepted.initial_voters, 5); + assert_eq!(omitted.resolve(&omitted, Some(&accepted)).unwrap(), accepted); + let changed = NodeDeploymentOverrides { + initial_voters: Some(3), + ..Default::default() + }; + assert!(changed.resolve(&omitted, Some(&accepted)).is_err()); + for count in [0, 2, 4, 6, 7, 255] { + let invalid = NodeDeploymentPolicy { + initial_voters: count, + ..Default::default() + }; + assert!(invalid.validate().is_err(), "count {count}"); + assert!(invalid.formation().validate().is_err(), "count {count}"); + } +} From cdf023d642da40f6b2db03c54b3f1a4fdc5fd049 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 31 Oct 2026 18:56:18 +0800 Subject: [PATCH 08/24] Reduce recovery memory and harden storage liveness --- .../src/iceberg/file_http/injection.rs | 14 +- .../src/catalog/recovery.rs | 57 ++ app/crowdb-chunk-kv-server/src/main.rs | 3 +- app/crowdb-chunk-kv-server/src/server.rs | 2 +- .../tests/catalog_test.rs | 3 + .../catalog_test/recovery_assignments.rs | 72 ++ .../src/background/domain_monitor/chunk_kv.rs | 10 +- .../domain_monitor/chunk_kv/liveness.rs | 36 + .../domain_monitor/diskdb_placement.rs | 139 +++- app/crowdb-kv-server/src/recovery/startup.rs | 13 +- .../tests/chunk_kv_balance_planning_test.rs | 2 + .../recovering_owner.rs | 79 ++ .../tests/diskdb_placement_test.rs | 48 ++ app/crowdb-kv-server/tests/startup_test.rs | 3 + .../tests/startup_test/reclamation.rs | 95 +++ container/crowdb-monitor/src/lib.rs | 5 +- container/crowdb-monitor/src/node.rs | 5 +- container/crowdb-monitor/src/node/runtime.rs | 5 +- .../src/node/runtime/recovery.rs | 3 + .../runtime/recovery/authority_refresh.rs | 35 + .../src/node/runtime/recovery/discovery.rs | 23 +- .../tests/member_discovery_test.rs | 25 + .../chunkds/design-crowdb-chunk-kv-server.md | 26 + .../chunkds/design-crowdb-chunk-stream.md | 37 +- ...sign-crowdb-chunkio-small-object-writer.md | 7 + doc/design/deploy/design-crowdb-deploy.md | 14 +- doc/design/diskdb/design-crowdb-diskdb.md | 5 + doc/design/kv/design-crowdb-kv-wal.md | 27 +- .../tree/design-crowdb-tree-chunk-storage.md | 20 + doc/design/tree/design-crowdb-tree-storage.md | 4 + doc/dev/physical-node-test-plan.md | 268 ++++++ doc/doc_index.md | 13 +- doc/working/track-two-physical-nodes.md | 775 ------------------ e2e-test/oci_reference_recovery.py | 9 + e2e-test/tests/deployment/kv_bootstrap.rs | 31 +- .../src/chunk/frame_observation/mirrored.rs | 2 + .../chunk/frame_observation/mirrored/order.rs | 47 ++ .../src/writer/small_pipeline.rs | 20 +- .../src/writer/small_pipeline/liveness.rs | 62 ++ .../tests/common/mirrored_frames.rs | 12 + .../tests/frame_observation_test/mirrored.rs | 89 ++ .../tests/small_object_test.rs | 7 +- .../small_object_test/staged_reservation.rs | 117 +++ lib/crowdb-chunk-kv/src/partition.rs | 191 +---- lib/crowdb-chunk-kv/src/partition/replay.rs | 236 ++++++ lib/crowdb-chunk-kv/src/partition/transfer.rs | 1 + lib/crowdb-chunk-kv/src/partition/tree.rs | 18 + .../tests/common/generated_journal.rs | 121 +++ .../tests/replay_memory_test.rs | 80 ++ .../tests/replay_tree_memory_test.rs | 58 ++ lib/crowdb-chunk-stream/src/memory.rs | 4 + .../src/production_chunk.rs | 21 +- .../src/production_chunk/allocation.rs | 4 +- .../src/production_chunk/growth.rs | 16 +- .../src/production_chunk/recovery/drain.rs | 66 +- lib/crowdb-chunk-stream/src/storage.rs | 5 + .../src/stream/capacity_plan.rs | 46 +- .../src/stream/successor.rs | 30 +- lib/crowdb-chunk-stream/src/stream/worker.rs | 2 + .../tests/capacity_plan_test.rs | 32 +- .../tests/common/initialization_window.rs | 81 ++ .../tests/production_chunk_test.rs | 15 +- .../production_chunk_test/capacity_refresh.rs | 77 ++ .../tests/production_chunk_test/growth.rs | 40 +- .../production_chunk_test/local_progress.rs | 2 +- .../production_chunk_test/preparation.rs | 71 +- .../production_chunk_test/recovery_drain.rs | 45 + .../tests/production_chunk_test/unwritten.rs | 13 +- .../src/semantic/observation.rs | 25 + lib/crowdb-kv/Cargo.toml | 8 + lib/crowdb-kv/src/wal/gc.rs | 18 +- lib/crowdb-kv/src/wal/index.rs | 35 + lib/crowdb-kv/src/wal/pipeline_writer.rs | 8 + lib/crowdb-kv/src/wal/record.rs | 3 + lib/crowdb-kv/src/wal/replay.rs | 31 +- lib/crowdb-kv/src/wal/replay/records.rs | 53 ++ lib/crowdb-kv/src/wal/segment.rs | 48 +- lib/crowdb-kv/src/wal/segment/read_buffer.rs | 50 ++ lib/crowdb-kv/src/wal/wal_engine.rs | 44 +- .../tests/replay_compaction_memory_test.rs | 82 ++ lib/crowdb-kv/tests/replay_memory_test.rs | 85 ++ lib/crowdb-kv/tests/wal_test.rs | 3 + lib/crowdb-kv/tests/wal_test/gc_test.rs | 119 +++ lib/crowdb-kv/tests/wal_test/index_test.rs | 2 + lib/crowdb-kv/tests/wal_test/record_test.rs | 9 + .../tests/wal_test/replay_compaction_test.rs | 116 +++ lib/crowdb-kv/tests/wal_test/segment_test.rs | 29 + .../crowdb-tree/maptable/buffer_pool.h | 19 +- .../include/crowdb-tree/snapshot/prepared.h | 9 +- .../backend/chunk/chunk_manifest_build.cpp | 8 +- .../src/backend/chunk/chunk_pack_pipeline.cpp | 18 +- .../src/backend/chunk/chunk_page_purpose.cpp | 26 +- .../src/backend/chunk/chunk_page_staging.cpp | 86 ++ .../src/backend/chunk/chunk_page_staging.h | 151 ++++ .../src/backend/chunk/chunk_page_store.cpp | 52 +- .../src/backend/chunk/chunk_page_store.h | 8 +- lib/crowdb-tree/src/maptable/buffer_pool.cpp | 3 +- lib/crowdb-tree/src/snapshot/persist.cpp | 339 +------- lib/crowdb-tree/src/snapshot/write.cpp | 361 ++++++++ .../integration/chunk_page_staging_test.cpp | 90 ++ .../integration/snapshot_memory_test.cpp | 274 +++++++ 101 files changed, 4126 insertions(+), 1630 deletions(-) create mode 100644 app/crowdb-chunk-kv-server/tests/catalog_test/recovery_assignments.rs create mode 100644 app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/liveness.rs create mode 100644 app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/recovering_owner.rs create mode 100644 app/crowdb-kv-server/tests/startup_test/reclamation.rs create mode 100644 container/crowdb-monitor/src/node/runtime/recovery/authority_refresh.rs create mode 100644 doc/dev/physical-node-test-plan.md delete mode 100644 doc/working/track-two-physical-nodes.md create mode 100644 lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored/order.rs create mode 100644 lib/crowdb-chunk-client/src/writer/small_pipeline/liveness.rs create mode 100644 lib/crowdb-chunk-client/tests/small_object_test/staged_reservation.rs create mode 100644 lib/crowdb-chunk-kv/src/partition/replay.rs create mode 100644 lib/crowdb-chunk-kv/tests/common/generated_journal.rs create mode 100644 lib/crowdb-chunk-kv/tests/replay_memory_test.rs create mode 100644 lib/crowdb-chunk-kv/tests/replay_tree_memory_test.rs create mode 100644 lib/crowdb-chunk-stream/tests/common/initialization_window.rs create mode 100644 lib/crowdb-chunk-stream/tests/production_chunk_test/capacity_refresh.rs create mode 100644 lib/crowdb-kv/src/wal/replay/records.rs create mode 100644 lib/crowdb-kv/src/wal/segment/read_buffer.rs create mode 100644 lib/crowdb-kv/tests/replay_compaction_memory_test.rs create mode 100644 lib/crowdb-kv/tests/replay_memory_test.rs create mode 100644 lib/crowdb-kv/tests/wal_test/replay_compaction_test.rs create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_page_staging.cpp create mode 100644 lib/crowdb-tree/src/backend/chunk/chunk_page_staging.h create mode 100644 lib/crowdb-tree/src/snapshot/write.cpp create mode 100644 lib/crowdb-tree/tests/integration/chunk_page_staging_test.cpp create mode 100644 lib/crowdb-tree/tests/integration/snapshot_memory_test.cpp diff --git a/app/crowdb-access-server/src/iceberg/file_http/injection.rs b/app/crowdb-access-server/src/iceberg/file_http/injection.rs index 08f122ee2..432be14ac 100644 --- a/app/crowdb-access-server/src/iceberg/file_http/injection.rs +++ b/app/crowdb-access-server/src/iceberg/file_http/injection.rs @@ -57,6 +57,8 @@ impl FileHttp { ) .await .map_err(|error| { + tracing::warn!(file = %owner.file, %error, + "native injection receive failed; unpublished blocks retained for reclamation"); body.failure() .map_or_else(|| admission_error(error), multipart::encoding_error) })?; @@ -64,11 +66,19 @@ impl FileHttp { .map_err(|_| FileS3ErrorCode::InternalError)? .seal_uploaded(owner, file_request.location.clone(), tree.clone()) .await - .map_err(multipart::seal_error)?; + .map_err(|error| { + tracing::warn!(file = %owner.file, %error, + "native injection seal failed; unpublished blocks retained for reclamation"); + multipart::seal_error(error) + })?; let file = FileRepository::new(self.store.clone()) .finalize_native_parquet(admission.context(), owner, &stem, tree, locations) .await - .map_err(catalog_error)?; + .map_err(|error| { + tracing::warn!(file = %owner.file, %error, + "native injection publication failed; candidate retained for reconciliation"); + catalog_error(error) + })?; let bytes = serde_json::to_vec(&serde_json::json!({"location": file.location.to_string()})) .map_err(|_| FileS3ErrorCode::InternalError)?; let length = bytes.len(); diff --git a/app/crowdb-chunk-kv-server/src/catalog/recovery.rs b/app/crowdb-chunk-kv-server/src/catalog/recovery.rs index 85a8feb60..f8ecbd0cf 100644 --- a/app/crowdb-chunk-kv-server/src/catalog/recovery.rs +++ b/app/crowdb-chunk-kv-server/src/catalog/recovery.rs @@ -8,6 +8,24 @@ use crowdb_protocol::chunk_kv::{ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage use std::{fmt::Display, future::Future, time::Instant}; impl ChunkKvRangeCatalogPublisher { + /// Recovers this instance's assignments against the current published head. + /// A successful replay remains usable when only remote assignments change; + /// every local entry must still match exactly, including its epoch and artifact. + /// # Errors + /// Returns catalog or artifact recovery failures under the supplied deadline. + pub async fn recover_current_for_instance( + &self, + instance_id: u64, + deadline: Instant, + recover: F, + ) -> Result, T)>, ChunkKvRangeCatalogError> + where + F: FnMut(Vec) -> R, + R: Future>, + { + self.recover_matching(deadline, Some(instance_id), recover).await + } + /// Recover a published catalog, discarding obsolete recovery results only /// when a newer authoritative generation proves publication advanced. /// @@ -17,6 +35,19 @@ impl ChunkKvRangeCatalogPublisher { pub async fn recover_current( &self, deadline: Instant, + recover: F, + ) -> Result, T)>, ChunkKvRangeCatalogError> + where + F: FnMut(Vec) -> R, + R: Future>, + { + self.recover_matching(deadline, None, recover).await + } + + async fn recover_matching( + &self, + deadline: Instant, + instance_id: Option, mut recover: F, ) -> Result, T)>, ChunkKvRangeCatalogError> where @@ -31,6 +62,17 @@ impl ChunkKvRangeCatalogPublisher { .as_ref() .is_some_and(|(next, _)| next.generation > head.generation) { + if result.is_ok() + && instance_id.is_some_and(|instance| { + let (_, next_pages) = latest.as_ref().expect("newer catalog exists"); + local_assignments_match(&pages, next_pages, instance) + }) + { + let (head, pages) = latest.expect("newer catalog exists"); + return result + .map(|recovered| Some((head, pages, recovered))) + .map_err(|error| ChunkKvRangeCatalogError::Unavailable(error.to_string())); + } if Instant::now() >= deadline { return Err(ChunkKvRangeCatalogError::GenerationConflict); } @@ -47,3 +89,18 @@ impl ChunkKvRangeCatalogPublisher { Ok(None) } } + +fn local_assignments_match( + before: &[ChunkKvRangeCatalogPage], + after: &[ChunkKvRangeCatalogPage], + instance_id: u64, +) -> bool { + let local = |entry: &&crowdb_protocol::chunk_kv::ChunkKvRangeCatalogEntry| { + crate::server::recoverable_local_entry(entry, instance_id) + }; + before + .iter() + .flat_map(|page| &page.entries) + .filter(local) + .eq(after.iter().flat_map(|page| &page.entries).filter(local)) +} diff --git a/app/crowdb-chunk-kv-server/src/main.rs b/app/crowdb-chunk-kv-server/src/main.rs index bd8edb6d2..7ef7f3ec7 100644 --- a/app/crowdb-chunk-kv-server/src/main.rs +++ b/app/crowdb-chunk-kv-server/src/main.rs @@ -219,7 +219,8 @@ async fn main() -> std::process::ExitCode { let recovery_service = &service; let instance_id = config.instance_id; match catalog - .recover_current( + .recover_current_for_instance( + instance_id, std::time::Instant::now() + std::time::Duration::from_secs(60), |pages| async move { recover_assigned_partitions(recovery_storage, recovery_service, &pages, instance_id).await diff --git a/app/crowdb-chunk-kv-server/src/server.rs b/app/crowdb-chunk-kv-server/src/server.rs index 957c8c332..e6e69884e 100644 --- a/app/crowdb-chunk-kv-server/src/server.rs +++ b/app/crowdb-chunk-kv-server/src/server.rs @@ -1391,7 +1391,7 @@ fn scan_page_bytes(page: &crowdb_chunk_kv::ScanPage) -> usize { }) } -fn recoverable_local_entry(entry: &ChunkKvRangeCatalogEntry, instance_id: u64) -> bool { +pub(crate) fn recoverable_local_entry(entry: &ChunkKvRangeCatalogEntry, instance_id: u64) -> bool { entry.owner.instance_id == instance_id && !matches!( entry.state, diff --git a/app/crowdb-chunk-kv-server/tests/catalog_test.rs b/app/crowdb-chunk-kv-server/tests/catalog_test.rs index ddd9a64fa..245211ac9 100644 --- a/app/crowdb-chunk-kv-server/tests/catalog_test.rs +++ b/app/crowdb-chunk-kv-server/tests/catalog_test.rs @@ -13,6 +13,9 @@ use crowdb_protocol::chunk_kv::{ }; use crowdb_protocol::chunk_stream::StreamName; +#[path = "catalog_test/recovery_assignments.rs"] +mod recovery_assignments; + fn page(generation: u64, owner_epoch: u64) -> ChunkKvRangeCatalogPage { let mut page = ChunkKvRangeCatalogPage { generation, diff --git a/app/crowdb-chunk-kv-server/tests/catalog_test/recovery_assignments.rs b/app/crowdb-chunk-kv-server/tests/catalog_test/recovery_assignments.rs new file mode 100644 index 000000000..6664d38b7 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/catalog_test/recovery_assignments.rs @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test] +async fn completed_replay_survives_remote_assignment_changes_after_original_deadline() { + let publisher = ChunkKvRangeCatalogPublisher::new(Arc::new(MemoryChunkKvRangeCatalogStore::default())); + let mut first = page(1, 1); + first.entries[0].range.end = Some(b"m".to_vec()); + let mut remote = first.entries[0].clone(); + remote.partition_id.low = 2; + remote.owner.instance_id = 9; + remote.owner.rpc_endpoint = "127.0.0.1:9901".into(); + remote.range.start = b"m".to_vec(); + remote.range.end = None; + remote.artifact.tree_id = 2; + remote.artifact.stream_name.low = 4; + first.entries.push(remote); + first.seal().unwrap(); + publisher + .publish(head(1, None, &first), vec![first.clone()]) + .await + .unwrap(); + let mut calls = 0; + let recovered = publisher + .recover_current_for_instance(8, std::time::Instant::now(), |pages| { + calls += 1; + let publisher = &publisher; + async move { + let mut next = pages[0].clone(); + next.generation = 2; + next.entries[1].owner_epoch = 2; + next.seal().unwrap(); + publisher + .publish(head(2, Some(1), &next), vec![next]) + .await + .unwrap(); + Ok::<_, &'static str>(pages[0].entries[0].owner_epoch) + } + }) + .await + .unwrap() + .unwrap(); + assert_eq!(calls, 1, "unchanged local artifacts must not replay twice"); + assert_eq!(recovered.0.generation, 2); + assert_eq!(recovered.2, 1); +} + +#[tokio::test] +async fn changed_local_epoch_never_reuses_completed_replay_after_deadline() { + let publisher = ChunkKvRangeCatalogPublisher::new(Arc::new(MemoryChunkKvRangeCatalogStore::default())); + let first = page(1, 1); + publisher + .publish(head(1, None, &first), vec![first]) + .await + .unwrap(); + let result = publisher + .recover_current_for_instance(8, std::time::Instant::now(), |_| { + let publisher = &publisher; + async move { + let next = page(2, 2); + publisher + .publish(head(2, Some(1), &next), vec![next]) + .await + .unwrap(); + Ok::<_, &'static str>(1) + } + }) + .await; + assert_eq!(result.unwrap_err(), ChunkKvRangeCatalogError::GenerationConflict); +} diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv.rs index 193b3ca19..7c757f980 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv.rs @@ -25,6 +25,7 @@ use super::{DomainMonitorDriver, DomainMonitorFuture}; mod balance; mod catalog; mod grants; +mod liveness; pub struct ChunkKvRangeMonitorDriver; @@ -59,8 +60,9 @@ impl DomainMonitorDriver for ChunkKvRangeMonitorDriver { if catalog::refresh_endpoints(control, descriptor).await? { return Ok(()); } - plan_dead_owner_transfer(control, descriptor).await?; - advance_dead_owner_exclusion(control, descriptor).await?; + let live_processes = liveness::live_processes(control).await?; + plan_dead_owner_transfer(control, descriptor, &live_processes).await?; + advance_dead_owner_exclusion(control, descriptor, &live_processes).await?; balance::plan(control, descriptor).await?; authorize_live_source_fences(control, descriptor).await?; publish_ready_transitions(control).await?; @@ -73,6 +75,7 @@ impl DomainMonitorDriver for ChunkKvRangeMonitorDriver { async fn plan_dead_owner_transfer( control: &Group0ControlPlane, descriptor: &DomainMonitorDescriptor, + live_processes: &HashSet, ) -> Result<(), String> { if descriptor.failure_policy != DomainFailurePolicy::AutomaticSharedStorage { return Ok(()); @@ -124,6 +127,7 @@ async fn plan_dead_owner_transfer( }); for entry in catalog.pages.iter().flat_map(|page| &page.entries) { if entry.state != ChunkKvRangeCatalogPartitionState::Serving + || live_processes.contains(&entry.owner.instance_id) || transitioning_partitions.contains(&entry.partition_id) || busy_owners.contains(&entry.owner.instance_id) || instances @@ -215,6 +219,7 @@ async fn plan_dead_owner_transfer( async fn advance_dead_owner_exclusion( control: &Group0ControlPlane, descriptor: &DomainMonitorDescriptor, + live_processes: &HashSet, ) -> Result<(), String> { let now_ms = wall_time_ms(); let dead_before = now_ms.saturating_sub(descriptor.dead_after_ms); @@ -224,6 +229,7 @@ async fn advance_dead_owner_exclusion( transition.phase, TransferPhase::TargetPrepared | TransferPhase::AwaitingFence ) || transition.release_proof.is_some() + || live_processes.contains(&transition.source.instance_id) || instances .get(&transition.source.instance_id) .is_some_and(|instance| instance.last_heartbeat_ms >= dead_before) diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/liveness.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/liveness.rs new file mode 100644 index 000000000..7b8590863 --- /dev/null +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/liveness.rs @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Independent process renewal remains visible while partitions replay. + +use super::{operation_error, wall_time_ms, Group0ControlPlane}; +use bytes::Bytes; +use crowdb_protocol::{ + chunk_stream::{StreamProcessLease, StreamPurpose}, + key::{StreamProcessLeaseKey, TextKey}, +}; +use std::collections::HashSet; + +pub(super) async fn live_processes(control: &Group0ControlPlane) -> Result, String> { + let rows = control + .scan_all_prefix(Bytes::from(StreamProcessLeaseKey::prefix_all()), 256) + .await + .map_err(|error| operation_error(&error))?; + let now = wall_time_ms(); + let mut live = HashSet::new(); + for row in rows { + let lease: StreamProcessLease = serde_json::from_slice(&row.value).map_err(|e| e.to_string())?; + lease.validate().map_err(|e| e.to_string())?; + let key = StreamProcessLeaseKey { + purpose: lease.identity.purpose, + instance_id: lease.identity.instance_id, + }; + if row.key.as_ref() != key.to_path().as_bytes() { + return Err("stream process lease differs from its key".into()); + } + if lease.identity.purpose == StreamPurpose::Wal && lease.expires_at_ms > now { + live.insert(lease.identity.instance_id); + } + } + Ok(live) +} diff --git a/app/crowdb-kv-server/src/background/domain_monitor/diskdb_placement.rs b/app/crowdb-kv-server/src/background/domain_monitor/diskdb_placement.rs index 293172501..e8660dd5c 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/diskdb_placement.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/diskdb_placement.rs @@ -12,6 +12,9 @@ use crowdb_protocol::common::{BindMapValue, InstanceValue, OwnerMapValue}; use crowdb_protocol::key::{BindMapKey, DiskGroupKey, InstanceKey, OwnerMapKey, RackKey, TextKey}; use std::collections::BTreeMap; +const OWNER_LEASE_MS: u64 = 3_600_000; +const OWNER_RENEW_MARGIN_MS: u64 = 60_000; + pub struct DiskdbPlacementMonitorDriver; impl DomainMonitorDriver for DiskdbPlacementMonitorDriver { @@ -44,29 +47,10 @@ async fn reconcile(control: &Group0ControlPlane, policy: &DomainMonitorDescripto .as_millis() .try_into() .unwrap_or(u64::MAX); - let instances = control - .scan_all_prefix(Bytes::from(InstanceKey::text_prefix_for_service("diskdb")), 256) - .await - .map_err(|e| e.to_string())?; - let mut heartbeats = BTreeMap::new(); - let mut loads = BTreeMap::new(); - for row in instances { - let instance: InstanceValue = decode(&row.value)?; - heartbeats.insert(instance.instance_id, instance.last_heartbeat_ms); - // Only the fenced server implementation publishes this capability. - let capable = control - .get(format!("/diskdb/ownership-capability/{}", instance.instance_id).as_bytes()) - .await - .map_err(|e| e.to_string())? - .value - .is_some(); - if capable - && !instance.rpc_endpoint.is_empty() - && now.saturating_sub(instance.last_heartbeat_ms) < policy.suspect_after_ms - { - loads.insert(instance.instance_id, 0usize); - } - } + let LiveInstances { + heartbeats, + mut loads, + } = live_instances(control, policy, now).await?; if loads.is_empty() { return Ok(()); } @@ -97,12 +81,9 @@ async fn reconcile(control: &Group0ControlPlane, policy: &DomainMonitorDescripto } groups.push((key, owner_key, owner.revision, value)); } + let mut renewed = false; for (group, key, revision, owner) in &groups { - let needs_owner = owner.as_ref().map_or(true, |owner| { - heartbeats - .get(&owner.instance_id) - .map_or(true, |last| now.saturating_sub(*last) >= policy.dead_after_ms) - }); + let needs_owner = owner_is_dead(owner.as_ref(), &heartbeats, now, policy.dead_after_ms); if needs_owner { if !has_data_binding(control, group).await? { continue; @@ -110,8 +91,15 @@ async fn reconcile(control: &Group0ControlPlane, policy: &DomainMonitorDescripto let target = least_loaded(&loads); assign(control, key, *revision, target, now).await?; *loads.get_mut(&target).expect("selected live target") += 1; + } else if renew_live_owner(control, key, *revision, owner.as_ref(), &loads, now).await? { + renewed = true; } } + // Renewal advances the map revision. Balance from a fresh snapshot on the + // next tick instead of attempting a move with that old generation. + if renewed { + return Ok(()); + } // Move at most one healthy group per tick. This bounds recovery work and // converges to a count difference of at most one without ping-pong. let target = least_loaded(&loads); @@ -132,6 +120,79 @@ async fn reconcile(control: &Group0ControlPlane, policy: &DomainMonitorDescripto Ok(()) } +fn owner_is_dead( + owner: Option<&OwnerMapValue>, + heartbeats: &BTreeMap, + now: u64, + dead_after_ms: u64, +) -> bool { + owner.map_or(true, |owner| { + heartbeats + .get(&owner.instance_id) + .map_or(true, |last| now.saturating_sub(*last) >= dead_after_ms) + }) +} + +async fn renew_live_owner( + control: &Group0ControlPlane, + key: &str, + revision: u64, + owner: Option<&OwnerMapValue>, + loads: &BTreeMap, + now: u64, +) -> Result { + let Some(owner) = owner.filter(|owner| { + loads.contains_key(&owner.instance_id) + && owner.lease_expiry_ms <= now.saturating_add(OWNER_RENEW_MARGIN_MS) + }) else { + return Ok(false); + }; + let mut value = owner.clone(); + value.lease_expiry_ms = now.saturating_add(OWNER_LEASE_MS); + publish_owner(control, key, revision, value).await?; + Ok(true) +} + +struct LiveInstances { + heartbeats: BTreeMap, + loads: BTreeMap, +} + +async fn live_instances( + control: &Group0ControlPlane, + policy: &DomainMonitorDescriptor, + now: u64, +) -> Result { + let instances = control + .scan_all_prefix(Bytes::from(InstanceKey::text_prefix_for_service("diskdb")), 256) + .await + .map_err(|error| error.to_string())?; + let mut result = LiveInstances { + heartbeats: BTreeMap::new(), + loads: BTreeMap::new(), + }; + for row in instances { + let instance: InstanceValue = decode(&row.value)?; + result + .heartbeats + .insert(instance.instance_id, instance.last_heartbeat_ms); + // Only the fenced server implementation publishes this capability. + let capable = control + .get(format!("/diskdb/ownership-capability/{}", instance.instance_id).as_bytes()) + .await + .map_err(|error| error.to_string())? + .value + .is_some(); + if capable + && !instance.rpc_endpoint.is_empty() + && now.saturating_sub(instance.last_heartbeat_ms) < policy.suspect_after_ms + { + result.loads.insert(instance.instance_id, 0); + } + } + Ok(result) +} + fn least_loaded(loads: &BTreeMap) -> u64 { *loads .iter() @@ -150,8 +211,23 @@ async fn assign( let value = OwnerMapValue { process_epoch: 0, instance_id: instance, - lease_expiry_ms: now.saturating_add(3_600_000), + lease_expiry_ms: now.saturating_add(OWNER_LEASE_MS), }; + publish_owner(control, key, revision, value).await?; + tracing::info!( + owner_key = key, + instance_id = instance, + "DiskGroup manager assigned" + ); + Ok(()) +} + +async fn publish_owner( + control: &Group0ControlPlane, + key: &str, + revision: u64, + value: OwnerMapValue, +) -> Result<(), String> { let placement = OwnerMapKey::from_path(key).map_err(|error| error.to_string())?; let rack_key = RackKey { rack_id: placement.rack_id, @@ -208,11 +284,6 @@ async fn assign( .await .map_err(|error| error.to_string())?; } - tracing::info!( - owner_key = key, - instance_id = instance, - "DiskGroup manager assigned" - ); Ok(()) } diff --git a/app/crowdb-kv-server/src/recovery/startup.rs b/app/crowdb-kv-server/src/recovery/startup.rs index 93e6756b4..932116dc6 100644 --- a/app/crowdb-kv-server/src/recovery/startup.rs +++ b/app/crowdb-kv-server/src/recovery/startup.rs @@ -185,9 +185,15 @@ pub async fn create_group_with_wal( wal_config.wal_record_format = crowdb_kv::wal::WalRecordFormat::TextLine; } wal_config.wal_skip_fsync = config.wal_skip_fsync; - let replay = replay_group(&wal_backend, &wal_config.wal_disks, group_id).await?; - let next_seg = replay.max_segment_id.saturating_add(1).max(1); - let wal = WalEngine::create_with_next_segment_id(wal_backend, wal_config, group_id, next_seg).await?; + let mut replay = replay_group(&wal_backend, &wal_config.wal_disks, group_id).await?; + let wal = WalEngine::create_from_replay( + wal_backend, + wal_config, + group_id, + replay.max_segment_id, + std::mem::take(&mut replay.index), + ) + .await?; let mut local_replica = { let engine = open_crowdb_tree_engine( @@ -198,6 +204,7 @@ pub async fn create_group_with_wal( &config.log_dir, ) .await?; + wal.set_snapshot_slot(engine.resume_from_slot()); PxLocalReplica::restore_from_replay_with_engine(replica_id, initial_role, &replay, engine).await? }; local_replica.set_wal(wal); diff --git a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs index 6cd9f43ec..d256bc2d7 100644 --- a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs +++ b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs @@ -5,6 +5,8 @@ mod balance_distribution; #[path = "common/balance_planning.rs"] mod balance_planning; +#[path = "chunk_kv_balance_planning_test/recovering_owner.rs"] +mod recovering_owner; use bytes::Bytes; use crowdb_kv::cluster::{group::PxGroup, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; diff --git a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/recovering_owner.rs b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/recovering_owner.rs new file mode 100644 index 000000000..3fb2585a2 --- /dev/null +++ b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/recovering_owner.rs @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_protocol::{ + chunk_stream::{StreamProcessIdentity, StreamProcessLease, StreamPurpose}, + common::InstanceValue, + key::{InstanceKey, StreamProcessLeaseKey}, +}; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[tokio::test] +async fn renewing_recovery_process_is_not_dead_when_discovery_is_stale() { + let fixture = TestBalancePlanning::new().await; + let path = InstanceKey { + service: "chunk-kv".into(), + instance_id: 1, + } + .to_path(); + let row = fixture.control.get(path.as_bytes()).await.unwrap(); + let mut instance: InstanceValue = serde_json::from_slice(row.value.as_deref().unwrap()).unwrap(); + instance.last_heartbeat_ms = 0; + fixture + .control + .put_batch(vec![( + Bytes::from(path), + Bytes::from(serde_json::to_vec(&instance).unwrap()), + )]) + .await + .unwrap(); + let now = u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis()).unwrap(); + let path = StreamProcessLeaseKey { + purpose: StreamPurpose::Wal, + instance_id: 1, + } + .to_path(); + let mut lease = StreamProcessLease { + identity: StreamProcessIdentity { + purpose: StreamPurpose::Wal, + instance_id: 1, + process_epoch: 2, + boot_nonce: [9; 16], + }, + expires_at_ms: now + 60_000, + prior_exclusion_deadline_ms: now - 1, + grant_sequence: 0, + grant_expires_at_ms: 0, + }; + lease.validate().unwrap(); + for expiry in [now + 60_000, now - 1] { + lease.expires_at_ms = expiry; + fixture + .control + .put_batch(vec![( + Bytes::from(path.clone()), + Bytes::from(serde_json::to_vec(&lease).unwrap()), + )]) + .await + .unwrap(); + fixture.tick().await; + let transfers = fixture + .control + .scan_all_prefix(Bytes::from(ChunkKvTransferKey::text_prefix_all()), 16) + .await + .unwrap(); + if expiry > now { + assert!( + transfers.is_empty(), + "a live recovery boot must retain its assigned WAL" + ); + } else { + assert_eq!( + transfers.len(), + 1, + "expired processes still require dead-owner failover" + ); + } + } +} diff --git a/app/crowdb-kv-server/tests/diskdb_placement_test.rs b/app/crowdb-kv-server/tests/diskdb_placement_test.rs index 6eddb6ece..495cc9ba3 100644 --- a/app/crowdb-kv-server/tests/diskdb_placement_test.rs +++ b/app/crowdb-kv-server/tests/diskdb_placement_test.rs @@ -160,6 +160,54 @@ async fn assigns_balances_and_replaces_dead_owners_without_changing_bindings() { .is_none()); } +#[tokio::test] +async fn renews_live_owner_leases_without_replacing_the_process_epoch_or_rewriting_fresh_leases() { + for remaining in [0, 1000] { + let (_, store) = registry_with_group_zero(PxLocalReplicaRole::Leader); + let control = Group0ControlPlane::acquire(&store).await.unwrap(); + instance(&control, 1, u64::MAX).await; + prepare_groups(&control).await; + let now = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap(); + let key = OwnerMapKey { + rack_id: 1, + node_id: 1, + disk_group_id: 1, + } + .to_path(); + put_json( + &control, + key.clone(), + &OwnerMapValue { + instance_id: 1, + process_epoch: 17, + lease_expiry_ms: if remaining == 0 { 1 } else { now + remaining }, + }, + ) + .await; + let initial = control.get(key.as_bytes()).await.unwrap().revision; + DiskdbPlacementMonitorDriver + .tick(&control, &descriptor()) + .await + .unwrap(); + let renewed: OwnerMapValue = get_json(&control, &key).await; + assert_eq!((renewed.instance_id, renewed.process_epoch), (1, 17)); + assert!(renewed.lease_expiry_ms > now + 60_000, "{renewed:?}"); + let revision = control.get(key.as_bytes()).await.unwrap().revision; + assert!(revision > initial); + DiskdbPlacementMonitorDriver + .tick(&control, &descriptor()) + .await + .unwrap(); + assert_eq!(control.get(key.as_bytes()).await.unwrap().revision, revision); + } +} + async fn prepare_groups(control: &Group0ControlPlane) { for dg in 1..=5 { put_json( diff --git a/app/crowdb-kv-server/tests/startup_test.rs b/app/crowdb-kv-server/tests/startup_test.rs index 133d4df8e..170380d1a 100644 --- a/app/crowdb-kv-server/tests/startup_test.rs +++ b/app/crowdb-kv-server/tests/startup_test.rs @@ -15,6 +15,9 @@ use crowdb_kv::wal::replay::replay_group; use crowdb_kv::wal::{IoBackend, WalEngine}; use crowdb_kv_server::recovery::startup::{create_group_with_wal, store_wal_root}; +#[path = "startup_test/reclamation.rs"] +mod reclamation; + fn encode_put_payload(key: &[u8], value: &[u8]) -> Vec { let mut buf = Vec::new(); buf.extend_from_slice(&1u16.to_le_bytes()); // 1 op (u16 LE) diff --git a/app/crowdb-kv-server/tests/startup_test/reclamation.rs b/app/crowdb-kv-server/tests/startup_test/reclamation.rs new file mode 100644 index 000000000..b999207c7 --- /dev/null +++ b/app/crowdb-kv-server/tests/startup_test/reclamation.rs @@ -0,0 +1,95 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_kv::wal::gc::run_gc_pass; + +#[tokio::test] +async fn restored_wal_keeps_old_segments_visible_to_safe_reclamation() { + let temp = crowdb_test_harness::test_dirs::tempdir_in_test_data("startup-wal-reclamation"); + let config = CrowDBConfig { + wal_root: temp.path().join("wal"), + config_root: temp.path().join("conf"), + data_root: temp.path().join("data"), + ..CrowDBConfig::for_tests() + }; + let backend = Arc::new(IoBackend::File); + let mut wal_config = WalConfig::with_root(store_wal_root(&config.wal_root, 0)); + wal_config.wal_segment_size = 256; + let wal = WalEngine::create(backend.clone(), wal_config, 1).await.unwrap(); + for slot in 1..=20 { + let entry = PxLogEntry { + slot, + ballot: PxBallot::new(3, 1), + term: 3, + payload: Bytes::from(encode_put_payload(b"key", &slot.to_le_bytes())), + }; + wal.append(&WALRecord::from_accepted(1, &entry)).await.unwrap(); + } + wal.seal_all().await.unwrap(); + drop(wal); + let group = create_group_with_wal( + 0, + 1, + 1, + PxLocalReplicaRole::Leader, + &config, + backend.clone(), + CrowdbTreeBackend::File, + ) + .await + .unwrap(); + let restored_wal = group.local_replica().wal().unwrap(); + assert_eq!( + restored_wal.index().slot_count(), + 20, + "startup discarded replayed slot locations" + ); + assert!(restored_wal.index().segments_snapshot().len() > 2); + assert_eq!( + run_gc_pass(restored_wal, 20).await.unwrap(), + 0, + "no durable snapshot yet" + ); + group.local_replica().learner.engine().flush(); + assert_eq!(group.local_replica().learner.engine().persist_snapshot(), 20); + drop(group); + let restarted = create_group_with_wal( + 0, + 1, + 1, + PxLocalReplicaRole::Leader, + &config, + backend.clone(), + CrowdbTreeBackend::File, + ) + .await + .unwrap(); + let restarted_wal = restarted.local_replica().wal().unwrap(); + assert_eq!( + restarted_wal.snapshot_slot(), + 20, + "restored durable floor must be immediately available" + ); + assert_eq!( + run_gc_pass(restarted_wal, 1).await.unwrap(), + 0, + "lagging learner still blocks GC" + ); + assert!( + run_gc_pass(restarted_wal, 20).await.unwrap() > 0, + "safe old segments must be reclaimable after restart" + ); + let replay = replay_group(&backend, &[store_wal_root(&config.wal_root, 0)], 1) + .await + .unwrap(); + assert!( + replay.records.len() < 20, + "reclaimed history must not return on the next restart" + ); + assert_eq!(replay.current_term, 3); + assert_eq!( + restarted.local_replica().learner.engine_get(b"key").await, + Some((20, 20_u64.to_le_bytes().to_vec())) + ); +} diff --git a/container/crowdb-monitor/src/lib.rs b/container/crowdb-monitor/src/lib.rs index 44e834dde..aa10172fd 100644 --- a/container/crowdb-monitor/src/lib.rs +++ b/container/crowdb-monitor/src/lib.rs @@ -46,7 +46,10 @@ pub use node::{ NodeManagementError, NodeRuntimeConfig, NODE_SERVICE_TYPE, }; #[cfg(feature = "test-util")] -pub use node::{member_recovery_destinations_for_tests, unique_member_endpoints_for_tests}; +pub use node::{ + member_recovery_destinations_for_tests, recovery_authority_work_for_tests, + unique_member_endpoints_for_tests, +}; #[cfg(feature = "test-util")] pub use preview::{preview_step_names_for_tests, run_preview_for_tests, PreviewError}; pub use probe::{ProbeError, ProbeExecutor}; diff --git a/container/crowdb-monitor/src/node.rs b/container/crowdb-monitor/src/node.rs index e43ac1144..3ea4de8b3 100644 --- a/container/crowdb-monitor/src/node.rs +++ b/container/crowdb-monitor/src/node.rs @@ -40,7 +40,10 @@ pub use runtime::{ }; pub use runtime::{control_node, run_node, run_node_with_inputs, NodeRuntimeConfig}; #[cfg(feature = "test-util")] -pub use runtime::{member_recovery_destinations_for_tests, unique_member_endpoints_for_tests}; +pub use runtime::{ + member_recovery_destinations_for_tests, recovery_authority_work_for_tests, + unique_member_endpoints_for_tests, +}; #[cfg(feature = "test-util")] pub use runtime::{publish_file_for_tests, resume_file_operations_for_tests}; #[cfg(feature = "test-util")] diff --git a/container/crowdb-monitor/src/node/runtime.rs b/container/crowdb-monitor/src/node/runtime.rs index d1dc30f05..94bd094a0 100644 --- a/container/crowdb-monitor/src/node/runtime.rs +++ b/container/crowdb-monitor/src/node/runtime.rs @@ -62,7 +62,10 @@ pub use recovery::{ accept_member_trust_for_tests, check_member_trust_for_tests, member_trust_ready_for_tests, }; #[cfg(feature = "test-util")] -pub use recovery::{member_recovery_destinations_for_tests, unique_member_endpoints_for_tests}; +pub use recovery::{ + member_recovery_destinations_for_tests, recovery_authority_work_for_tests, + unique_member_endpoints_for_tests, +}; type Result = std::result::Result>; diff --git a/container/crowdb-monitor/src/node/runtime/recovery.rs b/container/crowdb-monitor/src/node/runtime/recovery.rs index 8bbaec613..6e6cc423c 100644 --- a/container/crowdb-monitor/src/node/runtime/recovery.rs +++ b/container/crowdb-monitor/src/node/runtime/recovery.rs @@ -13,8 +13,11 @@ use ssh_key::PublicKey; use super::{durable, AcceptedNode, Result}; use crate::NodeIdentity; +mod authority_refresh; mod confirmed; mod discovery; +#[cfg(feature = "test-util")] +pub use authority_refresh::recovery_authority_work_for_tests; mod owners; mod publication; pub(super) use confirmed::{accept as accept_confirmed, prepare_admitted}; diff --git a/container/crowdb-monitor/src/node/runtime/recovery/authority_refresh.rs b/container/crowdb-monitor/src/node/runtime/recovery/authority_refresh.rs new file mode 100644 index 000000000..960d4e887 --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/recovery/authority_refresh.rs @@ -0,0 +1,35 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Shared deadline for recovered-route publication and optional contact refresh. + +use super::super::Result; +use std::future::Future; +use std::time::Duration; + +pub(super) async fn run( + publication: impl Future>, + confirmation: impl Future>, +) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(3); + // Recovered routes must reach current authority before optional reads + // through retained contacts can consume the shared deadline. + match tokio::time::timeout_at(deadline, publication).await { + Ok(Ok(())) => {} + Ok(Err(error)) => tracing::debug!(%error, "recovered endpoint publication awaits authority"), + Err(_) => tracing::debug!("recovered endpoint publication yielded at its deadline"), + } + match tokio::time::timeout_at(deadline, confirmation).await { + Ok(Ok(())) => {} + Ok(Err(error)) => tracing::debug!(%error, "confirmed recovery contacts await current authority"), + Err(_) => tracing::debug!("confirmed recovery contacts yielded at their deadline"), + } +} + +#[cfg(feature = "test-util")] +pub async fn recovery_authority_work_for_tests( + publication: impl Future>, + confirmation: impl Future>, +) { + run(publication, confirmation).await; +} diff --git a/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs b/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs index 2f54d2416..958f2b99e 100644 --- a/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs +++ b/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs @@ -52,14 +52,6 @@ async fn refresh(root: &Path, port: u16, web_port: u16) -> Result<()> { { return Err("member recovery is retired".into()); } - let authority_start = std::time::Instant::now(); - if let Err(error) = super::confirmed::refresh(root, &accepted).await { - tracing::debug!(%error, "confirmed recovery contacts await current authority"); - } - // Confirmation and publication share their existing three-second budget. - // A ten-second offline SSH proof must not stretch route refresh past the - // fifteen-second freshness fence while Group 0 is unavailable. - let publication_budget = Duration::from_secs(3).saturating_sub(authority_start.elapsed()); super::check(root, &accepted)?; let operation = PreparedBootstrap::load(&root.join("prepared-bootstrap.json"))?; let (trust, config) = super::confirmed::contacts(root, &operation)?; @@ -89,7 +81,7 @@ async fn refresh(root: &Path, port: u16, web_port: u16) -> Result<()> { let (members, conflicts) = unique(endpoints); let snapshot = MemberEndpoints { version: 1, - bootstrap: accepted.bootstrap, + bootstrap: accepted.bootstrap.clone(), observer_boot_id: local.boot_id, observed_at_unix_ms: SystemTime::now() .duration_since(UNIX_EPOCH)? @@ -99,16 +91,11 @@ async fn refresh(root: &Path, port: u16, web_port: u16) -> Result<()> { conflicts, }; durable::write(&root.join("member-endpoints.json"), &snapshot)?; - match tokio::time::timeout( - publication_budget, + Box::pin(super::authority_refresh::run( super::publication::reconcile(root, &operation, &snapshot, &config, web_port), - ) - .await - { - Ok(Ok(())) => {} - Ok(Err(error)) => tracing::debug!(%error, "recovered endpoint publication awaits authority"), - Err(_) => tracing::debug!("recovered endpoint publication yielded at its deadline"), - } + super::confirmed::refresh(root, &accepted), + )) + .await; Ok(()) } diff --git a/container/crowdb-monitor/tests/member_discovery_test.rs b/container/crowdb-monitor/tests/member_discovery_test.rs index 88155c7f9..b9b52e71d 100644 --- a/container/crowdb-monitor/tests/member_discovery_test.rs +++ b/container/crowdb-monitor/tests/member_discovery_test.rs @@ -4,6 +4,31 @@ #[path = "common/member_trust.rs"] mod trust; +#[tokio::test] +async fn stale_authority_contacts_cannot_starve_publication_through_recovered_routes() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::time::{Duration, Instant}; + let published = AtomicBool::new(false); + let start = Instant::now(); + crowdb_monitor::recovery_authority_work_for_tests( + async { + tokio::time::sleep(Duration::from_millis(1)).await; + published.store(true, Ordering::Release); + Ok(()) + }, + std::future::pending(), + ) + .await; + assert!( + published.load(Ordering::Acquire), + "old unavailable authority contacts consumed the publication budget" + ); + assert!( + start.elapsed() < Duration::from_secs(4), + "confirmation and publication retain one shared three-second budget" + ); +} + #[test] fn pinned_retained_contacts_are_probed_when_multicast_discovery_is_empty() { let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("member-recovery-contacts"); diff --git a/doc/design/chunkds/design-crowdb-chunk-kv-server.md b/doc/design/chunkds/design-crowdb-chunk-kv-server.md index 194601ac1..b1eb070ab 100644 --- a/doc/design/chunkds/design-crowdb-chunk-kv-server.md +++ b/doc/design/chunkds/design-crowdb-chunk-kv-server.md @@ -76,6 +76,32 @@ Normal service discovery filters expired registrations. Monitor observation uses the raw scan, preserving expired entries and heartbeat timestamps so absence cannot be confused with owner death. The default heartbeat is two seconds, `Suspect` begins at six seconds, and `Dead` recovery begins at ten. +Automatic dead-owner planning and lease exclusion also require the absence of +a live WAL process lease. Independent renewal starts before partition replay, +whereas discovery publication can wait for replay to finish. A renewing recovery +boot therefore retains its assignments despite stale discovery; it receives no +serving authority until normal recovery and grants complete. Expired processes +remain eligible for failover under the existing heartbeat and exclusion limits. + +When the catalog advances during initial replay, completed recovery remains +usable only if every local recoverable entry is identical in both validated +catalogs, including ownership epoch, range, state, transition, and artifact. +The service installs those handles with the newer head without replaying them +again. Remote assignment changes do not invalidate local recovery. Changed +local assignments still require recovery under the original reconciliation +deadline; failed replay is never accepted as completed recovery. + +WAL replay retains a SHA-256 fingerprint of each canonical complete record for +duplicate-sequence validation, rather than retaining its key and value payloads. +The fingerprint includes identity, epoch, request, recorded result, and operation. +Exact late duplicates remain valid; conflicting duplicates, sequence gaps, and +operation-digest mismatches remain corruption. Replay windows and retry-result +retention keep their existing independent bounds. +After each 16 MiB of estimated applied key/value and entry bytes, native replay drains its transient +memtables into the in-memory tree. This bounds overwritten versions retained +while normal maintenance is not yet running. It does not snapshot, publish a +checkpoint, advance the checkpoint replay offset, or trim WAL; a drain failure +still rejects recovery. ## 3. Serving Authority diff --git a/doc/design/chunkds/design-crowdb-chunk-stream.md b/doc/design/chunkds/design-crowdb-chunk-stream.md index 42664a0e3..786b92104 100644 --- a/doc/design/chunkds/design-crowdb-chunk-stream.md +++ b/doc/design/chunkds/design-crowdb-chunk-stream.md @@ -86,8 +86,11 @@ placement, growth, replacement, and their durable internal transitions. Each such transition updates chunk information independently. Stream metadata refers to stable chunk identities and logical mappings; internal chunk changes are transparent and never trigger Stream chunk record publication. -Recovery resolves current capacity and layout through the chunk layer instead -of trusting capacity cached in an older Stream checkpoint. +Recovery and read-only opening resolve current capacity and layout through the +chunk layer instead of trusting capacity cached in an older Stream checkpoint +or a retained adapter. Opening queries authoritative chunk metadata even when +the adapter has a cached layout; an unavailable query fails the open rather than +falling back to that cache. The checkpoint must still fit the current capacity. DiskIO writes carry a boolean sync option. For a synchronized request the server performs the ordered write, then the required flush, and returns one response @@ -284,8 +287,13 @@ strips incrementally; this remains internal to the chunk layer. Capacity preparation runs ahead of consumption. Measured write bandwidth and preparation duration size the initialized spare capacity with a two-interval -margin. At least one initialized Candidate is allocated and registered while -the Active chunk accepts appends. Normal writes use ready capacity; exceptional +margin, bounded between one frame and one physical strip. The physical allocator +rounds this target to strip boundaries without adding another growth multiplier. +As consumption approaches the hard chunk boundary, one initialized Candidate is allocated +and registered while the Active chunk accepts appends. Its initial capacity +follows observed batch consumption, not the configured maximum batch size. +Small cold streams retain their first strip without eagerly initializing spare +chunks. Normal writes use ready capacity; exceptional allocation/initialization delays apply backpressure instead of exposing unsafe capacity. @@ -298,8 +306,12 @@ leaves the one last referenced chunk to reconcile. An unattached Candidate has no acknowledged user data and can be reclaimed after owner exclusion. Different segment IO streams can drain concurrently, with at most eight drains -in flight. Every submitted drain must finish, including after a peer fails; -physical sealing requires all drains to succeed and current writer authority. +in flight across the complete layout, including during tail recovery. Strip +boundaries do not serialize independent drain requests or consume the prefix +scan's remaining owner deadline. Every submitted drain must finish, including +after a peer fails; physical sealing requires all drains to succeed and current +writer authority. Recovery retains its original captured deadline and requests +durable mirror repair only after all submitted drains have completed. ### 5.2 Tail reconstruction @@ -314,6 +326,10 @@ after validating the entire group. It then adds one compact location. Before publishing a fresh attachment or admitting writes into newly allocated capacity, zero and synchronize each new physical strip on every required mirror. Initialization uses one submission per replica, without automatic retries. An +initialization pass admits at most two strips at once; their required mirrors +still synchronize together. This bounds zeroing bursts while ordinary writes +use the initialized prefix. Every submitted initialization finishes before the +pass returns, and capacity is exposed only after every strip succeeds. An uncertain result leaves that capacity unavailable to the writer; the owner cannot repeat initialization after later data writes. A replacement strip receives its complete retained prefix and a zero suffix with the same single-attempt contract @@ -350,7 +366,14 @@ complete prefix; incomplete groups do not enter it. A failed observation cannot subsequently be finalized as a healthy shorter prefix. After owner exclusion, observe the current available mirrors concurrently -within a bounded IO budget. Use the shortest complete-append prefix among healthy +within a bounded IO budget. When retained DiskIO topology covers every segment, +order each strip's replicas by rack and node for consistent observation lanes +across strips. Allocation order alone can alternate racks, making every lane +depend on a slow cross-rack link. These route observations only order existing +segments: incomplete hints preserve original ordering, no replica is added or +omitted, and the original authoritative layout remains unchanged. Continue +checking every completed lane's checksum and complete-prefix agreement under +the same deadline. Use the shortest complete-append prefix among healthy mirrors that finish observation. A mirror that fails or times out during the read is unavailable, not a zero-length or partially observed prefix. Corrupt replicas are not valid shorter candidates; use healthy copies or report an diff --git a/doc/design/chunkio/design-crowdb-chunkio-small-object-writer.md b/doc/design/chunkio/design-crowdb-chunkio-small-object-writer.md index 1b93e502c..c01fe2e96 100644 --- a/doc/design/chunkio/design-crowdb-chunkio-small-object-writer.md +++ b/doc/design/chunkio/design-crowdb-chunkio-small-object-writer.md @@ -154,6 +154,13 @@ Resource allocation releases the existing chunk lifecycle guard; publication reacquires it and revalidates ownership and sequence. No extra persistent high-water record is required. +Lease renewal refreshes the authoritative readable layout while retaining the +writer's exact staged, unconsumed strip as a local write view. This also applies +to an empty prepared replacement. Renewal does not add that strip to published +capacity or advance the readable cursor. Consume still precedes the mirror +write, and Confirm still follows its durability. A changed chunk identity, writer epoch, or staged strip +fails the writer instead of selecting a later reservation and leaving a gap. + When automatic conversion is enabled and at least eight strips remain, one special reservation allocates eight mirror sets with the configured copy count and four parity segments from a joint placement plan. Each completed diff --git a/doc/design/deploy/design-crowdb-deploy.md b/doc/design/deploy/design-crowdb-deploy.md index 425f8a3ad..879bc273b 100644 --- a/doc/design/deploy/design-crowdb-deploy.md +++ b/doc/design/deploy/design-crowdb-deploy.md @@ -434,11 +434,15 @@ Recovery proceeds independently of Group-0 availability: conditional new service operations; later stop/delete commands win. Public client URIs are independent of these internal addresses. -The contact worker checks current authority before probing peers so retirement -still takes precedence. Confirmation and later address publication share a -three-second authority budget; SSH proofs retain their ten-second bound. A -failed authority lookup must not add another full publication wait and expire -otherwise valid routes at their fifteen-second freshness fence. +The contact worker checks retained retirement and accepted identity before +probing peers. After authenticating recovered routes, address publication +checks current authority through those routes before optional confirmation +refreshes retained contacts. Publication and confirmation share a three-second +authority budget; SSH proofs retain their ten-second bound. Unreachable old +authority contacts must not consume the budget before recovered addresses can +be published, nor add another full wait that expires otherwise valid routes at +their fifteen-second freshness fence. Current authority still fences removed +members and conditional service operations before publication. This permits all members to restart with changed IPs at once. Neither stable DNS names nor fixed IPs nor a complete peer-address list is required. Multicast must diff --git a/doc/design/diskdb/design-crowdb-diskdb.md b/doc/design/diskdb/design-crowdb-diskdb.md index 8b8ee84d5..2dc994d4e 100644 --- a/doc/design/diskdb/design-crowdb-diskdb.md +++ b/doc/design/diskdb/design-crowdb-diskdb.md @@ -367,6 +367,11 @@ disk-group's bound data group. `diskdb-ownership` monitor assigns bound disk-groups to live, fencing-capable DiskDB instances. Missing or dead owners are replaced; count balancing moves at most one healthy group per tick. Updates compare the owner record revision. + Before a live capable owner's one-hour placement lease expires, the monitor + renews it through the same revision-checked publication path without changing + the instance or process epoch. The DiskDB observes the new map generation and + reacquires its data-group fence. Fresh leases are not rewritten each tick; + renewals defer count balancing until the next snapshot. - **Binding map** (`/hw/dg_bind/...`) — the console recommends the ordinary KV group with the fewest bound DiskGroups cluster-wide and allows an explicit override at creation and publishes the binding, hardware record, and node membership atomically. Missing selections, Group 0, and unknown diff --git a/doc/design/kv/design-crowdb-kv-wal.md b/doc/design/kv/design-crowdb-kv-wal.md index 8ecb3261b..3308aa8ad 100644 --- a/doc/design/kv/design-crowdb-kv-wal.md +++ b/doc/design/kv/design-crowdb-kv-wal.md @@ -360,6 +360,9 @@ The split matters: replay/restore are purely *local* (this node's WAL), but `Acc tail from a crash mid-write); later segments are still processed. 3. Rebuild per-`(group, slot)` acceptor state keeping the **highest-ballot** `Promised` / `Accepted` per slot (later/higher-ballot records win, Paxos rule). + Retain promises and accepted values independently while scanning; release + superseded payloads immediately. Segment locations, counts and election + state still account for every verified record. 4. `current_term` = max `term` across all records. 5. `voted_for` = the node from the latest `VoteGranted` whose `term == current_term`. This is election safety state, not just debug metadata: after crash, the node must not grant a second vote in the same term. Output: `ReplayResult { records, max_segment_id, current_term, voted_for }`. @@ -401,6 +404,13 @@ outright; see `../tree/design-crowdb-tree-engine.md`). `replay_group` → create `WalEngine` and resume `next_segment_id = max_segment_id + 1` → `restore_from_replay` → `set_wal` → wrap in `PxGroup`. +- Seed the live pipeline index with the verified replay index before starting + writers. Its slot locations and segment metadata cover retained history as + well as future appends, so GC can reclaim pre-restart segments. +- Initialize the WAL's local snapshot watermark from the freshly opened + engine's durable resume floor. Peer progress and the reclamation certificate + remain independent gates; local recovery does not authorize prefix removal. + The group also records a conservative local slot tip: ``` @@ -466,9 +476,12 @@ the node fails out of the group and rebuilds from peers via snapshot install. ### 6.7 Replay performance -Replay is sequential disk reads, parallelized across disks; WAL sizes stay small -because of GC (§7), so replays are typically sub-second. The dominant cost is -state machine snapshot load (if any), not WAL read. +Binary replay uses a 1 MiB read-ahead buffer for frame markers, format probes +and complete frames. A larger frame receives enough space for its own length. +Every frame still passes its normal CRC and format checks. Retained payloads +own their bytes so replacing an accepted record can release it independently +of neighboring records in the input buffer. Memory scales with the retained +highest-ballot state and index, rather than the number of replaced versions. --- @@ -493,6 +506,14 @@ contract is in [slot retention](design-crowdb-kv-slot.md#11-interaction-with-sna GC runs at **segment granularity**, not record granularity. A whole segment is unlinked from disk when *every* record in it has `slot < gc_slot`. This is cheap (one `unlink()` per segment) and avoids any in-place rewriting. +- Slot-based eligibility does not retire election state. Retain a segment + proving the highest term and the segment containing the latest vote at the + highest recorded vote term, even when their slot records are below the GC + watermark. A mixed vote/data segment follows the same rule. +- Derive these election fields from every verified record during replay and + update them as live segments are written. They are in-memory index metadata; + WAL record and segment formats remain unchanged. + ### 7.3 GC trigger GC is triggered by: diff --git a/doc/design/tree/design-crowdb-tree-chunk-storage.md b/doc/design/tree/design-crowdb-tree-chunk-storage.md index cf24f22ff..c81fb9c2c 100644 --- a/doc/design/tree/design-crowdb-tree-chunk-storage.md +++ b/doc/design/tree/design-crowdb-tree-chunk-storage.md @@ -57,6 +57,26 @@ and a `ChunkPageRef` containing chunk identity, physical offset, logical length, and frame revision. Frame footers own payload integrity and expected chunk identity; high-cardinality reference entries do not duplicate their checksums. +Pending writes form a sparse byte overlay on one captured immutable manifest. +Starting a write does not materialize the published logical image or its holes. +Partial writes capture only their intersected published packs, retaining the +verified untouched bytes even if those mirrors become unavailable before commit. +Later overlapping writes replace earlier bytes; overlay fragments are bounded +to 64 KiB so replacing a small range cannot copy an arbitrarily large fragment. +Each fragment retains its full logical coverage while storing trailing zeros +implicitly. Zero padding therefore still overwrites base bytes and participates +in coverage checks without retaining an allocation-unit buffer per padded page. +Publication assembles one pack from verified base bytes and the overlay at a +time. Unchanged packs still undergo frame verification before reuse. Both +publication paths discard the overlay after commit, releasing its buffers. +The tree releases each encoded snapshot page as soon as its store write +completes, including asynchronous completion. It retains the page's publication +identity through the final anchor barrier, without retaining a second copy of +all bytes already accepted into the backend overlay. +Snapshot preparation retains logical encoded page bytes. It reserves aligned +addresses immediately but adds zero padding to the current page only at write +submission, avoiding an additional allocation-unit padding copy of every page. + Mapping-table slots do not store physical chunk offsets. Their tagged 64-bit page-reference words name a logical location whose covering pack is resolved through immutable segmented reference tables. Untagged words remain valid for diff --git a/doc/design/tree/design-crowdb-tree-storage.md b/doc/design/tree/design-crowdb-tree-storage.md index d0c3b52a8..e5cbc9058 100644 --- a/doc/design/tree/design-crowdb-tree-storage.md +++ b/doc/design/tree/design-crowdb-tree-storage.md @@ -439,6 +439,10 @@ alike. This is the "4 GiB / 8 GiB btree cache" knob (`capacity_bytes` default metadata under the pool mutex. The page table (`pid -> frame index`) is an open-addressing array, not `std::unordered_map`, to keep the hot path branch-light and cache-friendly. +- **Unused capacity.** Arena allocation preserves the configured frame count + without initializing every unused frame. Fresh and anonymous frames are + zeroed when acquired. A read miss loads and validates the complete frame + before publishing it, preserving initialization and stable frame addresses. - **Pinning and eviction.** A reader pins the leaf/inner frames along its descent, reads via the view, then releases. Pinned frames are never evicted. Eviction uses a **CLOCK (second-chance)** sweep: skip pinned and diff --git a/doc/dev/physical-node-test-plan.md b/doc/dev/physical-node-test-plan.md new file mode 100644 index 000000000..7eb0bb78a --- /dev/null +++ b/doc/dev/physical-node-test-plan.md @@ -0,0 +1,268 @@ + + + +# Physical Node Test Plan + +Validate five-node deployment, data correctness, sustained load, and recovery +across two physical hosts. This is a reusable plan. Store each run's results +separately; previous passes do not establish that a new run passed. + +Upstream: [deployment and recovery](../design/deploy/design-crowdb-deploy.md), +[OCI build and runtime](../../container/oci-image/README.md), +[container acceptance](../../e2e-test/README.md), and +[KV recovery](../design/kv/design-crowdb-kv-slot.md). + +## 1. Default environment and boundaries + +- Physical host A: `192.168.31.129`. +- Physical host B: `192.168.31.184`. +- Verify host identity, current interface, and address before each run. + A LAN IP locates a host but does not establish resource ownership. +- Use existing SSH configuration and Ed25519 authentication; verify + noninteractive `BatchMode`. Never record login usernames, passwords, private + keys, access tokens, or application secrets in this plan or committed reports. + Supply credentials through local SSH configuration/private inputs; redact logs. +- Use five fixed voters: A1/A2/A3 on A, B1/B2 on B. Port offsets are `0/1/2` + on A and `0/1` on B. Each node has a separate persistent volume and identity. + These are two physical host/rack failure domains, not five hosts. +- Use host networking, hard limits of `1 CPU / 2048 MiB` per container, and + one Tokio worker per service. Record cgroup limits and any deviations. +- Simulate host failure by stopping this run's containers on that host; record + the scope. Do not change shared firewalls, interfaces, routes, kernel settings, + or the Docker daemon. +- A reduced three-node run must explicitly declare A1/A2/B1 and three voters. + It cannot substitute for five-node coverage. + +## 2. Execution contract + +- Run every build, test, and executable through `pixi run`, including remotely. +- Create a unique run ID. Store scripts, source snapshots, receipts, clients, + logs, and results in `.crowdb-runtime/artifacts/physical-nodes//`. + Inventory volume ownership separately. Do not clean shared runtime directories + during an active run. Check both hosts' free disk space and memory first. +- No repository task currently implements this entire plan. Save a harness + using current public interfaces, its source, and actual commands. Do not depend + on earlier `/tmp` files, hardcoded node UUIDs, or deleted artifacts. Do not + present this plan as an existing one-command Pixi task. +- Finish shared builds before consumers; never build concurrently into the same + output directory. Freeze source during OCI builds. Record commit, dirty diff, + and relevant source identities. +- Reuse images only when service, dependency, and source inputs match. Import + the same verified OCI on both hosts; compare execution configuration and + ordered rootfs layers. Distinguish OCI manifest digests from Docker image IDs. +- Label resources with run ownership; verify it before faults, recreation, and + cleanup. +- Execute phases in order. Capture the first failure and stop dependent work. + After repair, rerun affected cases and resume unfinished work; retain valid + passes whose inputs have not changed. +- Do not increase timeouts, weaken assertions, or add caller retries to pass. + Poll only defined readiness/recovery conditions. Never replay timed-out + mutations; audit uncertain writes through read-only calls. +- Use `PASS / FAIL / NOT RUN`. An untriggered fault is a coverage gap even if a + later request succeeds. Any failed or unrun required case prevents all-pass. + +## 3. Deployment and baseline data + +- [ ] **P00 Environment, authentication, and image**: verify SSH key access, + clocks, Docker/cgroups, free space, management interfaces, cross-host listener + reachability, and existing resource owners. Save redacted inventory, source + identity, build receipts, and same-image import verification on both hosts. +- [ ] **P01 Five-node discovery and initialization**: form five voters through + normal startup policy, discovery, key admission, and production storage + preparation. Verify five distinct discovery UUIDs/node IDs, accepted + `initial_voters = 5`, fixed membership, and real listener addresses. Apply + offsets exactly once; check same-host conflicts and cross-host loopback. + Currently `run-node.py --initial-voters` accepts only `1/3`; use the supported + explicit node-policy/controlled container path for five voters and verify + Monitor's accepted result. Do not invent a working `--initial-voters 5` command. +- [ ] **P02 Whole-cluster readiness**: verify seven managed services per node, + 35 total, including counts, live processes, health, and dependencies. All five + Web endpoints must report Ready. Monitor observations must postdate this + container start and be no older than three seconds; old snapshots are invalid. + Measure age when receiving the snapshot and record that receipt time. Verify + current DiskDB ownership leases; live owners must renew expiring leases + without changing their process epoch. +- [ ] **P03 Baseline data and allocation**: check five-replica mappings for + Group 0, metadata Group 1, and explicit data Group 3. Wait for current leaders/ + leases before single-attempt operations. Verify exact KV single/batch writes; + deterministic S3 objects of at least 96 KiB with byte/hash checks; Iceberg + namespace/table/property commits and metadata locations; and native ChunkDB + allocation/query of the same ID. Cross-read through all five endpoints. Use + `crowdb-tpc-loader` for generated Iceberg and standalone S3 data, plus generated + Dataset samples with inline fields, multipart payloads, and complete readback. + Record nodes, groups, objects, and acknowledgments without application secrets. + +## 4. Write scale and 30 minutes of sustained load + +- [ ] **P04 Short 512-record reproduction**: use native ChunkKV defaults. + Write 32 batches of 16 records, each 64 KiB: 512 records/32 MiB across five + owners. Read the complete catalog first; generate run-unique keys within + actual ranges and verify routed owners. Exclude reserved S3, Iceberg, and + Dataset metadata prefixes; appending a test suffix to a protocol key still + corrupts that protocol's scan namespace. Select safe ranges covering all five + owners, or fail preparation. Uniform first bytes do not establish + coverage. Test the old key shape separately and report its coverage. + Preserve the default five-second deadline per batch. Verify every mutation + acknowledgment and full readback; save catalog split/cutover state, latency, + and results. Audit failed/uncertain writes read-only without replay. +- [ ] **P05 Larger working set**: after P04 passes, seed a fresh namespace with + at least 4096 deterministic 64 KiB records, 256 MiB total, across five Serving + owners. Budget both mirrors, metadata, WAL, reclamation, and artifacts; retain + at least 30% free test-media capacity. Expand only owned test media or fail + preparation if insufficient; do not silently shrink the working set. Record + key counts, acknowledged bytes, and verification digests, not just API success. +- [ ] **P06 Sustained workload**: run **30 continuous minutes** on P05's data, + excluding builds, deployment, and idle waiting. Mix native ChunkKV inserts, + overwrites, deletes, and readback across both hosts and all five owners. + Concurrently exercise S3 reads/writes/deletes, Iceberg commit/load/cleanup, and + Dataset sample reads/publication. Declare concurrency, ratios, pacing, and + working-set limits before starting. Continuously produce successful operations + while reusing/reclaiming bounded data; a fixed load followed by waiting does + not qualify. +- [ ] **P07 Soak data verification**: journal each mutation's operation ID, + key/version, expected length/hash, acknowledgment state, and timestamp. + Serialize overwrites per key or use an explicit version contract to prevent + verifier races. Continuously sample acknowledged data; finish with a complete + audit of live keys and expected deletions. Journal timeout/connection-loss as + `OutcomeUnknown` and audit it, rather than declaring it uncommitted. Any + mismatch, request failure, OOM, or unexpected process replacement fails the + normal load phase. +- [ ] **P08 Load observations**: sample throughput, success/failure/unknown + counts, p50/p95/p99/max latency, CPU, RSS/cgroup memory, free space, service + PIDs, catalog, and reclamation at least every five seconds. Report acknowledged + operations/bytes, actual active duration, and resource changes. Live containers + alone do not prove a pass. Keep different load-stage latencies separate. + +## 5. Faults and recovery using the soak data + +Before each case, save identities, PIDs, catalog, data inventory, and replica +status. After recovery, verify P03's baseline, all P07 data, and fresh +acknowledged writes from the current case. + +- [ ] **P09 Retained-volume recreation**: recreate all five containers with + original volumes, startup policies, and accepted identities. Verify unchanged + UUIDs/node IDs, 35 recovered services, readback through every endpoint, and + all replicas caught up. Repeat affected recovery after production image fixes. +- [ ] **P10 Unresponsive child process**: send `SIGSTOP` only to one verified + ChunkKV child. Monitor must recover it with a new PID within 60 seconds; the + other 34 service PIDs must remain unchanged. Verify old data and fresh + single-attempt operations. Clear injected faults on failure paths too. +- [ ] **P11 Cross-host KV lifecycle**: from a Web endpoint with an available + remote leader, stop/restart its local follower. Separately stop the current + leader. Verify Stop receipts, actual termination, continued remote management, + a new PID, and no collateral restarts. A completed Stop receipt must remain + readable after the operation stops the former Group-0 leader. +- [ ] **P12 Both B nodes offline**: stop B1/B2; A retains three-voter quorum. + Verify single-attempt KV writes/reads and catalog takeover onto surviving + Serving owners within 180 seconds. While B stays offline, submit fresh S3 + writes and Iceberg property commits through A, then cross-read. Use new + witnesses per fault to prove new writes occurred. Restore B, then recreate + B1/B2 individually with retained volumes. Check original identities, all data + through five endpoints, and all five replicas caught up. +- [ ] **P13 All three A nodes offline**: stop A1/A2/A3; B loses quorum. + Observe current unavailability within 30 seconds; writes must not acknowledge + success. Audit unknown outcomes after recovery without asserting absence. + Restore A; verify every service, five-replica checkpoints, and complete data, + not just a new leader. +- [ ] **P14 Allocation-only safe retry**: inject bounded Group-1 leadership + step-down while preserving KV processes and the Group-0 lease. Within three + seconds, verify the leader accepted at least one step-down. Observe explicit + allocation-level `Unavailable` safe retry. After release, complete same-ID + allocation/query within the original five-retry/ten-second limit. Record + actual retry evidence and original limits; success alone is insufficient. + Use the production client; instrumentation must not change branches or policy. +- [ ] **P15 Large backlog and full cold restart**: use normal acknowledged + writes to leave a controlled offline KV replica more than 1024 slots behind. + Record group, start/end checkpoints, and actual gap. Restore it and verify + bounded-concurrency FetchGap/leader recovery progress, then cold-start all + five retained-volume nodes. Fix acknowledged targets for Groups 0/1/3; + require all five replicas per group to reach them with healthy engines and + exact data. Do not move targets with ongoing writes. Report actual coverage + of reclaimed WAL, empty sealed WAL, high promise/old chosen, and other recovery + boundaries; component regressions do not prove physical coverage. +- [ ] **P16 Final recovery gate**: require all catalog entries Serving, all five + endpoints Ready, and 35 healthy services. The next sample must show the same + normal PIDs. Audit all data; complete a fresh KV batch and fresh ChunkDB + allocation/query. Fully scan owned DiskDB occupancy read-only; verify no + cross-chunk overlap among live mirrored segments and no scan truncation. + +Measure recovery budgets independently: catalog takeover 180 seconds, service +recovery 180 seconds, and fixed-checkpoint catch-up 180 seconds per group. +Record each condition's own start and elapsed time. Whole-phase duration cannot +prove individual deadlines. Preserve shorter production-client limits. + +## 6. Failure evidence, repairs, and cleanup + +- [ ] **P17 Failure capture**: print failing cases, operations, and complete + errors. Use `CROWDB_E2E_PRINT_SERVICE_ERRORS=1` while debugging. Save client + output, service logs, current inspect data, exit/OOM state, health freshness, + KV roles/leases/checkpoints, catalog/transfers, DiskIO availability, and disk + usage. Export allowlisted/redacted fields without credentials. +- [ ] **P18 Repair verification**: locate the first divergence and root cause. + Add focused regressions and run affected component tests plus fmt/clippy. + Rebuild/verify images when production inputs change. Rerun affected cases + under the same resources and limits. After repairing P04, complete P05–P08; + 512 records alone do not finish testing. Repairs or harness restarts during + the soak require a complete new 30-minute run after the fix. +- [ ] **P19 Freeze and cleanup**: stop owned containers before freezing full + service logs. Delete only inventoried containers, volumes, and transfer files + with verified ownership. Check owned ports are closed on both hosts, no owned + resources remain, and SSH key access still works. Retain run artifacts. + Do not prune the daemon or clean other tasks' resources. Capture failures + before cleanup. + +## 7. Deliverables and the next run + +Save these artifacts and link them from the final summary: + +- `run.json`: run ID, source/image identities, both IPs, node/port/resource + mapping, ownership inventory, actual commands, and harness source locations. + No login credentials. +- `results.json`: P00–P19 states, times, limits, errors, evidence paths, and + unrun reasons. Preserve first failures separately from repaired results. +- `data-ledger.jsonl`, `metrics.jsonl`, and `checkpoints.json`: acknowledged/ + uncertain outcomes, load metrics, fixed targets, and actual catch-up. + Segment large files if needed. +- `cleanup.json` and `summary.md`: cleanup verification, actual passing + coverage, unresolved defects, and next starting point. Distinguish "run ended" + from "all required tests passed." CI does not replace physical results. +- `loader/`: loader source identity, generation/injection/verification commands + and reports, Iceberg tables/snapshots, S3 inventory, immutable Dataset snapshots, + and sample verification summaries. Confirm implementation and acceptance of + all three targets before use; mark unimplemented paths `NOT RUN`. + +Open issue requiring retest (2026-10-31): five containers writing +512 records × 64 KiB reached the default five-second deadline during +split/cutover. The previous run acknowledged and recovered only 294 records, +not a full load. Reproduce, repair, and fully pass P04; then seed at least +4096 records and run 30 continuous minutes. Remove this open item only after +successful repair and retest, preserving results in its run directory. + +Suggested request for a future run: + +> Execute the full five-node test in `doc/dev/physical-node-test-plan.md` with +> the two LAN hosts and existing SSH keys. Run continuously for 30 minutes. +> Resolve open issues first, capture and repair failures, then resume. +> Do not reduce coverage or relax limits. + +## 8. Implementation and regression entry points + +- Startup/images: `container/oci-image/{run-node,runtime,artifact,transfer}.py`, + `container/single-node-container/entrypoint.sh`, and + `container/crowdb-monitor/src/node/startup/policy.rs`. +- Formation/observations: `lib/crowdb-protocol/src/mgmt/node/`, + `app/crowdb-web/src/node/`, + `lib/crowdb-console-shared/src/ops/cluster/readiness/`, and + `container/crowdb-monitor/src/node/runtime/`. +- Data/recovery: `lib/crowdb-{kv,kv-client,chunk-kv-client,chunkdb-client}/`, + `app/crowdb-chunkdb/src/topology/`, and + `lib/crowdb-chunk-stream/src/production_chunk/`. +- Component regressions: + `lib/crowdb-kv/tests/group_test/{recovery_backlog,fetch_gap_durability}_test.rs`, + `lib/crowdb-chunk-stream/tests/production_chunk_test/finalized_empty.rs`, + `container/crowdb-monitor/tests/{member_discovery,node_service_receipt}_test.rs`, + and `app/crowdb-chunkdb/tests/{allocation_retry,topology_availability}_test.rs`. +- Container/protocol regressions: `e2e-test/tests/deployment/`, + `e2e-test/tests/console/`, `e2e-test/tests/access_iceberg/`, and corresponding + S3/native SDK acceptance entries. Select tests for actual changes and use + `pixi run`. Distinguish component, container, and real cross-host coverage. diff --git a/doc/doc_index.md b/doc/doc_index.md index 232ef5c87..3de30bc2e 100644 --- a/doc/doc_index.md +++ b/doc/doc_index.md @@ -47,12 +47,13 @@ Temporary plans live under `doc/working/`; flow analyses live under ## Dev Environment (`doc/dev/`) -| Doc | When to read | -| ---------------------------- | ---------------------------------------------------------------------- | -| `doc/dev/crash_debugging.md` | Core collection, host setup, GDB, and exact-build symbols. | -| `doc/dev/env_setup.md` | Benchmark commands, sentinels, prerequisites, and perf-counter setup. | -| `doc/dev/hyper_fork.md` | Hyper fork branches, submodule, build, sync, validation, and recovery. | -| `tools/README.md` | Tool directories, Pixi task entry points, CI checks, and suite timing. | +| Doc | When to read | +| ------------------------------------ | -------------------------------------------------------------------------------- | +| `doc/dev/crash_debugging.md` | Core collection, host setup, GDB, and exact-build symbols. | +| `doc/dev/env_setup.md` | Benchmark commands, sentinels, prerequisites, and perf-counter setup. | +| `doc/dev/hyper_fork.md` | Hyper fork branches, submodule, build, sync, validation, and recovery. | +| `doc/dev/physical-node-test-plan.md` | Repeatable two-host, five-container physical tests and 30-minute sustained load. | +| `tools/README.md` | Tool directories, Pixi task entry points, CI checks, and suite timing. | ## Component Acceptance diff --git a/doc/working/track-two-physical-nodes.md b/doc/working/track-two-physical-nodes.md deleted file mode 100644 index c65a80654..000000000 --- a/doc/working/track-two-physical-nodes.md +++ /dev/null @@ -1,775 +0,0 @@ - - - -# Two Physical Node Test Track - -Run ID: `two-physical-nodes-20261030T010205Z`. -Latest fixture: `physical-20261030-r17`; result: **five-container recovery matrix -passed across two physical hosts**, including all five replicas' acknowledged -Group-0/1/3 checkpoints. All owned physical fixtures are cleaned up and key-only -SSH works. First publication is `25059cf7d`; the follow-up CI repairs pass local -verification and are prepared for the authorized second publication. Historical -CI runs remain failed; the next published commit requires its own remote result. - -The separate 512-record/default-five-second capacity probe failed; 294 confirmed -records are exact recovery witnesses, not a capacity pass. User disk cleanup -deleted the earlier local archives, including attempt 15 forensic files. Later -sections describe that loss and the surviving/reconstructed evidence explicitly; -historical archive descriptions below refer to their state before that cleanup. - -## Attempt 12 placement and image (historical) - -- Host A: `u24-amd`, `192.168.31.129`, two logical voters, offsets `0/1`. -- Host B: `cj7960`, `192.168.31.184`, one logical voter, offset `0`. -- Cluster ID: `475d2a73-19f4-421e-8ccf-39693ef368c2`. -- Host networking, separate retained volumes, one CPU and 2 GiB per node. -- Final OCI digest: - `sha256:105cd3d3fccad16b1e907e3ba7a7f8cad327319a9f678a4815cdd09921a9b080`. -- Build revision: `5d03757b75a9b1867eebbcab9346e84d153bdcf2`; source snapshot: - `b1a845605cd9f41136a67e81a91f51119912639b`. -- Both hosts imported the same archive. Execution configuration and ordered - rootfs layers were verified, including across Docker 29/28 import formats. - The receipt records the production sources at build time; the allocation - retry tests and this report were added afterward. - -## Attempt 12 verified observations (historical) - -- [x] Host inventory, direct reachability, and distinct host-network listeners. -- [x] Host-B `cj` key-only SSH using `/home/cj/.ssh/id_ed25519` with - `BatchMode=yes` and `IdentitiesOnly=yes`; public key installed without - removing existing authorized keys. SSH directory/file modes verified. -- [x] All discovery candidates admitted with `ssh_password = null`. -- [x] Three-voter Group 0 active across the two physical hosts. -- [x] All seven services healthy on every node after production disk setup. -- [x] KV put/get/batch, 96 KiB S3 object exact-byte reads, Iceberg namespace, - table and property round trip, and initial small mirrored allocation. -- [x] All three containers recreated from retained volumes with the corrected - image, original identities retained, and no password-file environment. -- [x] Managed Chunk-KV listener fault: `SIGSTOP` on the owned child, bounded - monitor replacement with a different PID, all services healthy, then - KV/S3/Iceberg reads through all three endpoints. -- [x] With host B stopped, a Group-3 KV write and read succeeded through host A. -- [x] After host B returned, existing KV/S3/Iceberg data read correctly from - every endpoint. All seven services had stable PIDs across the next health - pass; a final native KV batch also passed. -- [x] Focused RPC tests prove a known allocation `Unavailable` retries with the - same chunk ID, while a lost allocation reply returns `OutcomeUnknown` and - submits exactly once. These are separate from physical fault coverage. -- [x] Full final service logs frozen after container stop, inspect/status and - policy snapshots retained, and exact owned containers/volumes removed. - All owned endpoints closed; the host SSH key remains installed. - -## Attempt 12 failures and coverage limits (historical) - -These failures are retained as first-divergence evidence. Attempt 16 below -passed the repaired storage failover, fresh allocation, physical safe-retry -and expected majority-host-loss cases. - -- [ ] **Host-B storage availability:** the first S3 read while B was stopped - returned `ServiceUnavailable` with caller retries disabled. A subsequent - observation-only wait also failed: after the existing 180-second bound, - catalog generation 14 still assigned B's range to owner 3. A persisted - `TargetPreparing` transfer from owner 2 to owner 1 reserved both surviving - owners. That explains why the current planner had no eligible target; the - reason preparation remained unfinished required diagnosis. Returning B - restored corpus reads. Availability during B's absence did not pass. -- [ ] **Final allocation:** a fresh native allocation after restored health - failed with `diskdb allocate failed for disk_group 2: diskdb server does not - own the requested resource: disk-group not owned`. Listener health and old - data reads therefore do not establish restored allocation readiness. -- [ ] **Physical allocation-only retry:** a bounded pause of the two follower - KV processes was followed by successful allocation and query, but the - instrumented client observed no allocation-level safe retry. The physical - case does not prove that specific path. Retry limits and caller deadlines - were unchanged. Pausing only one of three voters cannot remove quorum. -- [ ] **Final-image two-voter host-loss repeat:** not run after the preceding - recovery failure. The expected quorum-loss observation passed in attempt 11 - on its earlier image and is retained as historical evidence only. - -## Repairs validated during continuation - -- Admission now publishes the target's accepted, offset-resolved setup choices; - omitted choices previously published identical KV endpoints for every node. -- Standard DiskDB policy uses independent main/HTTP/RPC blocks - `11000/11100/11200`, each shifted once. Adjacent offset nodes previously - collided because the RPC field was interpreted as a legacy three-port base. - Explicit listeners preserve legacy custom-base deployment behavior. -- Access startup probes every catalog owner before another process launch; - a healthy owner previously masked another owner replaying its WAL and - consumed the restart budget. Tests cover redirects, multiple healthy owners - and an unavailable second owner. The physical listener fault then passed. - -## Evidence - -Root: `.crowdb-runtime/artifacts/two-physical-nodes-20261030T010205Z/`. - -- `attempt10-local-fixed/`: local offsets `0/1/2`, active Group 0; no storage - readiness claim for that diskless fixture. -- `attempt11-final-physical/`: earlier mixed-placement KV/quorum/recreation - results, expected two-voter loss, and cleanup. Its storage phase failed. -- `attempt12/result.json`: historical case classification and next unrun work. -- `attempt12/catalog-all-owners-fix-receipt.json`: final image receipt. -- `attempt12/baseline-catalog-all-owners-fix/`, `listener-fixed/`: - corrected-image baseline and successful managed recovery evidence. -- `attempt12/remote-stop-{catalog,transfer,split}.json`, - `matrix-failure-remote/`: failover diagnostics and durable transitions. -- `attempt12/native-final-allocation.json`, `final-allocation-failure/`: - final allocation error and complete frozen service logs. -- `attempt12/corpus-read.json`, `final-health-stable.json`, `cleanup.json`: - restored reads, stable process observations, and verified cleanup. -- Reproduction scripts are retained in `attempt12/`; client-only retry logging - lived under `/tmp` and did not change production client sources. - -## Local verification - -- Passed: full `crowdb-protocol` and `crowdb-web` Rust tests. -- Passed: monitor policy/dependencies/readiness, 15 tests; ChunkDB allocation - and dynamic retry, five tests. -- Passed: clippy for protocol, monitor, Web and ChunkDB with tests and - `-D warnings`; Rust fmt and `git diff --check`. -- Earlier unchanged launcher verification: 134 OCI Python tests and shell - syntax passed. The final production image build/import passed. -- Default `ssh -o BatchMode=yes cj@192.168.31.184` also succeeded without an - explicit identity flag after fixture cleanup. - -## Attempt 13 diagnostics (historical) - -- User requested continued repair and full physical validation before inspecting - the eight linked CI jobs. -- Corrected stale DiskDB ownership rejection routing and released-source - Chunk-KV grant/WAL recovery; focused regressions and affected clippy pass. -- OCI digest: `sha256:3b21576d0bb18aec223bdb4cd4916b6400c76d3b1cf74a883afa90eb588d78f7`; - source snapshot `8d0e21fb044c27b8dd0e9bcb98cf4de1f808b7fd`. -- New owned fixture `physical-20261030-r13` reached active three-voter Group 0 - and seven healthy services on every node. Baseline KV/S3/Iceberg reads pass. - Retained-volume recreation and the remaining matrix are in progress. -- The corpus harness now waits for a newly created data group's elected leader - before its first write; the initial immediate write encountered election - startup. This adds a setup assertion, not a retry of a data operation. -- Evidence is retained in `attempt13/`; attempt-12 failures remain preserved. - -- Attempt 13 retained-volume recreation and listener recovery passed. The - isolated quorum case exposed a stale KV route to a removed replica: the - missing-local-group response did not trigger discovery. Regression first - failed, then passed for point reads/writes, conditional writes and scans; - the 65-test KV store suite also passed. Missing-group responses now carry - the existing typed NotLeader status, with legacy point-response fallback. -- The temporary two-voter metadata fixture retired its removed replica's local - creation record; it is not reused for the later three-voter recovery cases. - Its logs, membership errors and cleanup evidence remain in `attempt13/`. - -## Attempt 14 diagnostics - -- The isolated Group-1 outage also affected DiskDB metadata and therefore - produced terminal disk-allocation errors before an allocation-only retry. - These are failed observations, not retry coverage. The removed replica is - retired and the invalid two-voter production fixture is not reused. -- Recovery publication incorrectly applied the observer's local KV management - and RPC ports to every member. Binding and service receipts showed duplicated - Host-A seeds and an incorrect offset on Host B. A mixed-port/same-host/IPv6 - regression reproduced this, then passed after retaining each member's ports. -- All 13 monitor member-trust tests and monitor clippy pass. Attempt-14 owned - containers, volumes and transfer archives were removed; frozen logs remain. -- User authorized the full physical and CI repair commit/push, followed by a - separate five-container test on these two hosts and another commit/push. - -## Attempt 15 continuation - -- OCI `sha256:5a9495fc919858e5c4aebc707c35115dce769aeb3646dba88d83c489660a7e43`, - source snapshot `41c9b797a28641a2408d3fa3562805d754f25a7e`, passed baseline, - all-node retained-volume recreation, and managed listener recovery. -- Every binding retained the same correct three mixed-port management seeds. -- Allocation-only retry passed with the original five-retry/10-second client - limits: concurrent Group-1-only step-downs left all three voters and Group 0 - live, produced a known `Unavailable` response, and then allocation/query - succeeded for the same chunk identity. This validates transient leadership - unavailability; it is not a claim of physically pausing two KV processes. -- Host-B-stop catalog recovery failed within the existing 180-second bound. - A target rejected a sealed, physically empty WAL chunk whose strips had - legitimately been returned by `trim_sealed_strips`. The authoritative chunk - had zero capacity, sealed length and acknowledged cursor. Recovery incorrectly - required at least one mirror strip. B was restored in the fixture's finally - block, and complete diagnostics remain under `attempt15/matrix-failure-remote`. -- A regression reproduces the empty sealed tail and checks active, nonempty, - wrong-purpose and stale-snapshot rejection. The 47-test production chunk - adapter suite passes after handling only this canonical empty sealed state. - The existing physical fixture will be upgraded with its volumes preserved. - -- Upgrading the retained fixture to OCI - `sha256:0f9326c2f4b08b8e069983b9348efad7dea88a2e275392dde0c6c7d189828ae9` - resolved the previously stuck empty-tail transfer; all services and corpus - reads recovered, and listener recovery passed again. -- The next Host-B-stop run exposed missing repair admission on a failed old-WAL - drain. The referenced active chunk remained at writer epoch 4, with no new - epoch-5 allocation: failure occurred before successor allocation. The - recovery path returned on an offline mirror without feeding the existing - durable repair queue. The fix drains every mirror, conditionally marks failed - segments unavailable only when a drained survivor exists, and keeps recovery - unavailable until the replacement layout itself drains successfully. It does - not mark transport failures as corruption or omit failed drains from success. -- The ChunkDB crash-gap regression confirms unavailable layout records are - admitted by its repair scanner. Focused recovery tests cover lost metadata - replies and the absence of any drained survivor. - -- All 49 production adapter tests pass, including durable failure-report reply - loss and all-mirror drain failure. The existing recovery IO deadline remains - unchanged. The repaired physical matrix remains the next required gate. - -## Attempt 15: reclaimed journal exposed duplicate physical allocation - -- A1 Chunk-KV recovery rejected tree `4789280886995677209`: both mirrors of - chunk `216631886962935262:3939770405816213384` contained zero bytes in the - first 262284 bytes, including the frame at offset 65606. -- Its sealed tree metadata still referenced A2 disk unit 318 and A1 disk unit - 321. Current DiskDB busy records assigned both units to newer WAL chunks. - Allocation timestamps and chunk records are retained in - `attempt15/allocation-records.log` and `overlapping-chunks.log`; both raw - mirror reads and complete service logs are retained with the fixture. -- A deterministic restart regression reproduced the upstream fault: a durable - KV engine retained a busy record while earlier WAL segments were reclaimed; - the restored acceptor still had trim watermark zero before maintenance. - `JournalScan` returned success with empty history, so DiskDB recovered an - incomplete occupancy bitmap and could reuse a live extent. -- `JournalScan` now requires every slot in the requested applied interval, - including NoOps, before filtering by key. Missing history returns the existing - typed GC-gap error, selecting DiskDB's full-scan fallback. The regression was - observed failing before the fix and passing afterward. The full KV store - suite passed 66 tests. A separate DiskDB integration regression verifies the - old busy unit remains reserved and is excluded from subsequent allocations. -- This does not repair overwritten mirror contents. Attempt 15 remains a - failed durability case; preserve its evidence and repeat the matrix with - fresh owned volumes after rebuilding. No data-loss case is counted as a pass. -- The next fixture explicitly accepts ChunkDB dynamic ownership policy. - Attempt 15's fixed ownership policy cannot establish availability while a - ChunkDB owner is stopped; changing ownership mode requires a fresh accepted - deployment rather than silently changing the existing cluster. - -- The existing DiskDB recovery suite passed all eight tests, and KV/DiskDB - clippy with tests and denied warnings passed. The failed fixture is stopped; - its volumes remain intact, with frozen metadata/WAL and service-log archives - for all three nodes under `attempt15/allocation-overlap-frozen/`. - -## Attempt 16: explicit policy file and nonzero port offset - -- The fresh dynamic-ownership fixture stopped at A2 startup: `deployment inputs - differ from accepted node policy`. The preflight had sealed shifted service - ports, but the following monitor invocation compared the unshifted file - inputs against that accepted record. No Group-0 bootstrap or data corpus had - started. A1/A2 volumes are retained for retry with the same accepted inputs. -- Port resolution now compares explicit file/argument listeners in the accepted - port space and hands the complete resolved policy to preflight/runtime once. - Repeated acceptance preserves policy bytes; a changed listener is rejected. -- The new repeated-file-acceptance regression failed before repair. All 20 - policy, CLI-input and management-input tests pass, as does monitor clippy - with tests and denied warnings. `pixi run tree-lint` also completed with exit - zero; its complete diagnostic output is retained with verification logs. -- First-start failure logs and the prior image receipt remain under - `attempt16/before-policy-file-offset-fix/`; subsequent bootstrap has not yet - been verified on the corrected image. - -- Corrected attempt-16 image: OCI - `sha256:faab2bd15b6abe37598caa9a78e8e6d75d38490559f2ae97144d05e1846aa8fc`, - source snapshot `59bca7e66bddd0454c2a3121b57dd6219264e3c2`. Both hosts - imported matching configuration/rootfs layers. All three execution records - explicitly contain `CROWDB_CHUNKDB_OWNERSHIP_POLICY=dynamic`. -- All three nodes initialized and reached seven healthy services. KV batch/data, - S3 exact-byte payload and Iceberg metadata reads passed through every node. -- All containers recreated with their retained volumes and identities; all - services and three-endpoint data reads passed again. Native scans found no - cross-chunk physical overlap among 134 live mirrored segments before - recreation or 238 afterward. -- Managed Chunk-KV `SIGSTOP` recovery completed inside the existing 60-second - bound with a changed PID, followed by all-service health and exact corpus - reads on every endpoint. Host-B stop/recovery is the next active case. - -- Host B stopped: Group-3 KV quorum write/read passed, all catalog entries - returned to surviving owners within the original bound, and one-shot S3 and - Iceberg corpus reads passed through both A endpoints while B stayed stopped. - B then returned and was recreated from its volume: all services and exact - reads passed on all three endpoints. This supersedes the earlier failed - remote-availability observations for this corrected dynamic-policy image. - -- The first host-A loss case confirmed quorum loss and rejection of a write. - Its recovery assertion then failed with S3 connection refused because the - harness read a retained pre-stop `monitor.json` as current readiness. - Failure capture showed A1/A2's new snapshots had no Access entry yet, while - the earlier readiness observations had already accepted the old seven-service - snapshot. The same processes later became ready without another intervention. -- The physical harness now requires status timestamps to follow the current - Docker container start and be no older than the existing three-second status - freshness contract. Failure capture still retains raw stale evidence. The - original 180-second recovery bound and one-shot corpus reads are unchanged; - repeat this host-loss case with the corrected assertion before advancing. - -- Repeating the host-A two-voter loss with current-status checks passed: - Group 0 became unavailable, a write could not complete during quorum loss, - and all seven services plus exact KV/S3/Iceberg reads recovered at all three - endpoints. The timed-out write has an unknown eventual outcome; this case - asserts unavailable write completion during the outage, not nonexistence of - its key after quorum returns. -- Group-1-only rapid leadership step-downs reproduced the explicit native - allocation `Unavailable` retry (`retries exhausted after 11 attempts, last - error: not leader`). Releasing the fault completed same-ID allocation/query - within the unchanged client bounds, with all three metadata voters retained. - All service checks and three-endpoint corpus reads passed afterward. - -- Final attempt-16 gate passed: every catalog partition Serving, all seven - services per node healthy with stable PIDs across the next health pass, a - fresh native KV batch and fresh allocation/query, plus exact corpus reads - through all three endpoints. A final native metadata scan found no - cross-chunk overlap among 2432 live mirrored segments. -- Attempt-16 and the quarantined attempt-15 owned containers/volumes were then - removed after freezing final service logs. Exact run labels were checked; - all owned endpoints closed, transfer/policy files were removed, and the host - SSH key was retained. Cleanup receipts are stored in each attempt directory. - -## Requested CI continuation - -- All eight requested job logs are retained under `ci/run-41003611925/`. - The original run used commit `39440d023d7fb162dd61c377e8eb17eefa8c4c9a`. -- AccessTests: retained Monitor startup/resume/cleanup fixtures now seal accepted - policy before identity effects. Exact failures, full Monitor tests, component - Access tests, official SDK tests, Dataset SDK boundary/lint/tests and Python - installed-consumer checks pass. Python coverage includes 21 consumer cases - and the real HTTP Split SDK round trip. Records: - `ci/ci-dataset-sdk-results.json`, `ci/ci-access-tests.log`, - `ci/ci-access-sdk.log`. -- Console/deployment bootstrap: diskless fixtures incorrectly disabled mandatory - Paxos KV, causing node exit followed by `No route to host`. Shared fixtures - now retain explicit Group-0 KV listeners and disable ordinary storage setup. -- Monitor restart decisions now use liveness. Normal ChunkKV transfer readiness - withdrawal previously caused responsive children to be replaced. The - regression preserves responsive PIDs beyond the failure threshold and still - requires replacement for a genuinely unhealthy listener. Published status, - startup dependencies and admission retain readiness checks. -- KV-only diskless setup must not reserve default Group 1. Dataset browse - reproduced the race (`200` instead of absent-group `404`), and the setup - regression reproduced `Running` instead of `Ready`. Preparation now excludes - this mandatory prerequisite when storage is diskless. All 12 setup tests, - full Monitor tests and clippy pass. -- S3: all 45 original cases pass with two slots and the original resource and - deadline budgets, including official SDK injected cleanup, storage outages, - process recovery and protocol parity. Artifact: - `container-e2e/run-76afe2e0dfe24ac6a583002643fddd3d`; log: `ci/s3-final.log`. - The original Go cleanup failure is unclassified: its artifact omitted the - underlying error. Focused and full executions pass; safe operation/error-code - diagnostics now preserve evidence if it recurs. -- Console browser: all 27 cases pass, including original failed replica - navigation and logical creation. Artifact: - `container-e2e/run-810e73b204924f97bd774df1c148dbc0`; - log: `ci/console-browser-final.log`. -- Native data: all 14 original cases pass across the five Dataset cases, - six-node round trips and ordered remainder. Original data assertions, - balancing, process-preservation checks and deadlines remain. Records: - `ci/remaining-sequence.json`, `ci/native-data-remainder-cases.json`. -- The candidate review helper now expects mandatory Paxos KV checked and the - other five services unchecked; all fixed choices remain read-only. Six-node - verification passes. Bridge default-policy assertions now use the accepted - DiskDB RPC port 11200 and one 16-GiB zone. No visible UI behavior changed. -- Complete bridge admission passes SSH proof, candidate review, foreign-cluster - rejection, restart, service lifecycle, minority authority gating and default - setup. It verifies retained 16-GiB disk identity after Monitor restart, - bitmap/capacity properties and no OOM/PID exhaustion. Artifact: - `container-e2e/run-29a8a32454d945c795c825e79084525b`; - log: `ci/bridge-admission-contract-final.log`. -- Inspection passes the complete original production balancing/browser case - in 329 seconds. Artifact: - `container-e2e/run-4380b4daaa054bd1a528ac73ffafbb01`; - log: `ci/console-inspection-final.log`. -- Native operations exposed a remote-leader routing defect. Stopping a local - follower left the remote leader healthy with a valid lease, but restart - returned 502 after requests to `127.0.0.1:10100`. HTTP topology ingestion now - resolves wildcard store listeners against the reporting management origin - before caching. Explicit peer endpoints remain intact. The regression fails - before the fix and passes afterward, covering IPv4/IPv6 and the cached - Group-0 leader. Full Console shared, Web and Monitor tests/clippy pass. - The exact lifecycle case passes in 8.8 seconds with original assertions: - `container-e2e/run-4315916e556445948d33173a097cfede`; - log: `ci/ci-diskdb-kv-origin-fixed.log`. -- Reference launch preflight exposed stale container protocol mappings. Fresh - Docker/Pixi/CLI/Compose launches now publish S3 16000, Iceberg 17000 and - Dataset 18000 while retaining host-side port inputs. Version-2 launch - receipts select canonical targets; version-1 receipts keep their immutable - ports/image through resume and retained-volume recreation. Compose provides - explicit legacy target overrides. Image exposed ports and examples agree. - All 136 OCI tests, 14 reference-fixture tests, six CLI lifecycle tests and - full CLI tests/clippy pass. The rebuilt image receipt is retained as - `ci/protocol-ports-image-receipt.json`. -- On the rebuilt image, complete native operations passes all 19 original - cases in 530 seconds, including typed restarts, retained bytes under load, - catalog navigation, ownership, multipart and explicit binding repair. - Artifact: `container-e2e/run-912e419cf5774ea197ec626fce31dbee`; - log: `ci/ci-console-native-operations-closing.log`. -- Complete transition passes production split, journal inspection and final - balanced Serving placement across three owners in 281 seconds. Artifact: - `container-e2e/run-a06b30aecc5e4ba8953d59df60b463a5`; - log: `ci/ci-console-transition-closing.log`. -- All six deployment-reference cases pass on the final image: Docker, Compose - and editable UI for one/three nodes, plus real Pixi shortcuts and confirmed - nonvoter cold recovery on new names/IPs. Original voter IDs, retained data, - credentials and identities are preserved. Artifact: - `container-e2e/run-554eaff75f0b4beb83d0dfdb59e3bb45`; - log: `ci/ci-deployment-reference-closing.log`. -- Final-image Console browser again passes all 27 original cases in 291 - seconds. Artifact: `container-e2e/run-5d6bcf681bda4e88998e34dd06bb9e47`; - log: `ci/ci-console-browser-closing.log`. -- Final-image native data passes all 14 original cases in one complete run, - including all five Dataset cases, six-node protocol round trips, topology, - retry, geometry, hardware and maintenance. Artifact: - `container-e2e/run-c831fca5df334574a14035c11c89ae54`; - log: `ci/ci-console-native-data-closing.log`. -- Final-image inspection passes the complete original case in 317 seconds, - including production balancing, real page layout and Iceberg inspection. - Artifact: `container-e2e/run-eac3d6766dab43809a0eee22414435e0`; - log: `ci/ci-console-inspection-closing.log`. -- All eight requested CI job selections now pass locally. Ordered closing - commands, image identity and complete results are in `ci/closing-sequence.json`; - per-job classification is in `ci/requested-job-results.json`. Remote CI for - the new commits is observed after publication. -- Quality: affected all-target clippy with warnings denied, workspace Rust - formatting, launcher shell syntax and C++ tree-lint pass. No C++ source - changed. Both bridge fixtures completed resource cleanup; their six owned - 16-GiB temporary payload files were retired after matching inode/size receipts. - Logs, metadata, disk headers and exact retirement records remain under - `ci/retired-bridge-disks-*`. Attempt-15 forensic archives remain intact. - -## Authorized publication and five-container follow-up - -- The first coherent commit/push includes the passed physical matrix, all eight - requested local CI selections and relevant quality gates. -- After that push, a fresh five-container cluster will span the same two - physical hosts (three containers on A, two on B). Preserve immutable image - identity, key-only admission, exact data witnesses, retained-volume recovery, - both host-loss cases and owned-resource cleanup. -- Verify repairs and commit/push the five-node changes and evidence summary - a second time. The execution plan remains active until then. - -## Five-container physical continuation - -- First publication completed: commit - `25059cf7d9a5f08d61479f4749659aa6f3fe119f` pushed to `task-fix-build`. - PR #55 CI run `41128190276` completed: 22 jobs passed; five follow-up - failures are recorded below and remain part of this task. -- Selected five initial voters across two physical hosts: A offsets `0/1/2`, - B offsets `0/1`, with a separate volume and seven services per container. -- New regressions reproduced three entry restrictions: accepted voter count, - default preparation membership and bound-member conflict handling. Policy now - accepts only `1/3/5`; preparation maps every admitted voter to its own initial - storage binding; all multi-node scopes retain the existing formation fences. - Policy, all nine preparation tests and all nine formation tests pass. -- Full affected tests/clippy and a fresh image precede physical deployment. - The physical matrix will include 512 exact 64-KiB ChunkKV records, all five - serving owners, cross-host KV/S3/Iceberg reads, retained-volume recreation, - unresponsive-child recovery, remote-leader KV stop/start, both host-loss cases, - bounded allocation retry and a final no-overlapping-allocation scan. - -- Full affected Protocol, Console shared, Web and Monitor tests, all-target - clippy and workspace formatting pass. One Monitor CLI test retained the old - invalid-voter error text; it now asserts the `1/3/5` contract and retains all - no-identity/no-storage side-effect checks. -- The first remote CI run also exposed `g3_leader_change::leader_change_simulation`: - the second one-shot write returned failure. Its original artifact omitted the - response, so the precise original reply remains unknown. The focused case, - a complete 131-case group run and twenty logged repetitions pass before the - helper repair. A deterministic regression then proves that the fixture's - role-only wait can return before a current quorum lease exists. The helper - now waits within the same three-second bound for current proposing tenure - and quorum lease; writes remain one-shot and report the complete response on - failure. The three affected cases, complete `test-core` gate and KV clippy - pass. Production Paxos logic is unchanged. Evidence: - `ci/run-41128190276/core-verification.json`, `ci-leader-ready-{red,green}.log`. - - -### Attempt 17: five voters on two physical hosts - -- Immutable image `0ef175e40bef7a152fda9abff2929f41df3830d39bedf555f04de53776dcab51` - is verified on both hosts. All five Group-0/default-metadata/manual-data voters, - 35 service health observations and KV/S3/Iceberg witnesses pass initially. -- The additional 512-record capacity case failed its original five-second batch - deadline during split/cutover. A full read-only audit found records 0–293 exact - (19,267,584 bytes), with 294–511 absent. No timed-out mutation was replayed. - Preserve this failed capacity case separately from the 294-record recovery - witness. No process OOM or PID replacement occurred during the capacity failure. -- All-five retained-volume recreation passes with exact identities and witness - bytes. SIGSTOP ChunkKV is replaced within 48.3 seconds; the other 34 PIDs and - all witnesses are retained. Cross-host local-KV stop/start passes after fixing - the harness to accept an omitted stopped-service row as well as a null PID. -- Stopping both B containers preserves the manual-data KV quorum write, but - catalog recovery fails its original 180-second bound. WAL recovery drains - schedule repair, then new active WAL initialization still selects stopped - physical IO targets. This remains a failed case until repaired and rerun. -- Whole-cluster readiness also exposes a fixed-9090 lookup and a `1/3` reducer - restriction. Repair accepted Web-listener publication and five-voter reduction; - verify the original volume set after upgrading the image. -- First-push CI exposed four additional groups: KV-only readiness collection, - stale CLI Access ports, journal selection during normal owner transfer, and GC - foreground overlap. Keep the original assertions/deadlines and verify each - group before final publication. Browser trace and GC operation timestamps are - retained under `ci/run-41128190276/`. - -- Repaired-image round 1 preserved the original volumes but exposed finalized - empty WAL tails: the current Stream record still referenced a zero-boundary - chunk that the ChunkDB finalizer had legitimately sealed, reclaimed and - tombstoned. Exact records under `upgrade-round1/` show `Deleted`, a seal time, - zero capacity/cursors and no remaining strips. Recovery previously accepted - only `Sealed` for that empty case. Both a cold adapter and complete writer - reopen regression reproduce the error, then pass with exact finalization - proof. Reopen preserves the tombstone and allocates a fresh chunk; eleven - invalid/unproven states remain rejected. Full Stream and ChunkKV server tests - pass; quality gates and the second image upgrade precede resumed faults. -- The same round confirms the independent cluster-observation repair: all five - UUIDs resolve to the accepted per-host `9090/9091/9092` listeners, with no - invalid-voter or foreign-member reasons. Remaining service reasons correctly - report the blocked WAL recovery. See `upgrade-round1/partial-cluster-status.json`. - -- Round 2 image `cde2868ade7b3437c27f01c67c7eaf407a02c15b53b874e580c04d0611d1b1ea` - restores all original volumes, 35 services, all five cluster-readiness views - and exact witnesses. B-host loss now passes Catalog takeover, original data, - and one-shot S3/Iceberg writes. Both B containers recreate with retained - identities and all five nodes verify the outage writes and 294 exact records. -- The subsequent majority-host loss correctly rejects writes, but restoration - fails the original 180-second bound. KV role/lease health returns while new - leaders' applied frontiers remain behind. `upgrade-round2/` preserves the - exact failure, current per-replica health and complete logs. They show serial - bulk Phase 1 competing with background repair, plus follower FetchGap stopping - permanently above the 1024-gap snapshot threshold despite having no automatic - running-replica snapshot replacement. Two regressions reproduce these paths. - Bounded concurrent recovery, mutually exclusive initial/background repair and - continued 16-request FetchGap pass three regressions, including cancellation - before Accept. Full core gates and a rebuilt image precede the resumed matrix. - -- The complete `test-core` gate, KV all-target clippy and workspace formatting - pass after the backlog repair. Evidence: `ci/run-41128190276/backlog-gates.json`. - -- During round 3 the user deleted `.crowdb-runtime` for disk cleanup. Historical - artifact archives, including the earlier frozen corruption records, are no - longer available. Surviving `/tmp` logs/scripts were copied back into the same - runtime artifact root, as requested; current state and complete service logs - were recaptured from the five original Docker volumes. The cleanup also - interrupted artifact publication after the already-failed restoration wait. - Do not treat reconstructed expectations or new captures as recovered archives. -- Round 3 image `64454951913004b00d5d8dad480e8ddeb65088ce34cb427b64c3d0269ec48a71` - again passes original-volume upgrade and B-host loss/recreation. Majority loss - rejects writes, but restoration still fails the original 180-second bound. - Further regressions show FetchGap failing to persist recovered entries and - restart discarding a valid durable snapshot prefix when its WAL boundary is - absent. Both failures reproduce before their fixes, then focused restart and - durable-tree regressions pass. Recovery also needs to probe retained pinned - contacts when multicast is absent; an empty-discovery regression reproduces - the missing probes. All route publication continues to require pinned SSH - identity/bootstrap proof. Complete gates and another image precede resumption. - -- After the durable catch-up and retained-contact fixes, complete `test-core`, - Monitor tests, both crates' all-target clippy and workspace formatting pass. - Confirmation/publication share their existing three-second authority budget, - keeping offline SSH proof refresh below the fifteen-second route fence. - Evidence: `ci/run-41128190276/durable-gates.json`. - -- Round 4 image `2989a3a0dc7729126f005330ee58f1e701a87d3696164ce750a259d874755f03` - restores all 35 services, five cluster-readiness views and exact witnesses. - The stronger per-replica check then fails its 180-second bound: B replicas - remain at Group-0 applied slots 77005 and 34771 while A advances beyond 140000. - The matrix stops before new fault injection. Current receipts, checkpoints and - complete logs are retained under attempt 17's `round4/upgrade-failure/`. -- A real-RPC regression reproduces historical chosen replay being rejected by - a higher promise; another proves FetchGap incorrectly exposed accepted-only - values. Both fail before the repair. Learning now preserves higher promises, - WAL replay retains accepted history independently of promise record order, - and the source requires a recovered contiguous chosen proof. Focused tests - pass; complete core gates and another retained-volume upgrade remain pending. -- An actual zero-ballot probe confirms both blocked boundary slots retain a - higher promise (`(1,4)` / `(1,5)`) than the source's historical chosen ballot - `(0,3)`. Both probes are rejected without changing those promises. The - retained receipt, probe source and complete response are under `round4/`. -- The repaired FetchGap cases preserve exact nonempty values and higher - promises through WAL restart. The existing RPC migration fixture now - publishes its actual two-voter quorum decision before fetching; response - assertions remain intact. Complete core coverage passes through its retained - prefix and resumed remainder; affected all-target clippy and formatting pass. - Evidence: `ci/run-41128190276/promise-resumed-gates.json` and - `ci-fetch-gap-promise-{red,green}.log`. -- Round 5 image `51d0a67638ee9102664710eb95711dae3d201d066c09e44bb8a3aa1ae7e9996d` - restores all services, five readiness views and exact data. All five replicas - reach the fixed Group-0/1/3 checkpoints within the original 180-second bound; - the Group-0 check completes in 110.3 seconds after both B replicas had been - tens of thousands of slots behind. Per-second checkpoint observations and - immutable receipts accompany the retained-volume upgrade. Host faults follow. -- Review found that 64 retained address hints could consume the whole probe - budget and exclude a discovered changed IP. A 64-member regression fails - before prioritizing fresh hints, then both discovery regressions and Monitor - clippy pass. The five-member physical fixture probes the same complete set - in either order; the next CI image includes this budget-boundary repair. -- Round 5 B-host loss/recreation passes in 338.6 seconds for the whole - sequence: surviving quorum write/read, Catalog takeover within 180 seconds, - new unique acknowledged KV/S3/Iceberg witnesses, exact original reads, - retained identity recreation of both B containers and all-five reads. - Majority-host loss, allocation fault, final audit and cleanup remain pending. -- Majority-host loss/restoration passes in 175.3 seconds for the whole sequence: - the two surviving voters reject the write, then all 35 services, all-five - replica checkpoints and exact witnesses recover within their original bounds. -- The first allocation fault observation failed a harness race: its wait ended - on the first follower's no-op reply before the leader's accepted reply arrived. - It now waits for actual acceptance within the same three-second bound and - persists every reply. The resumed case observes explicit allocation-only safe - retry, completes same-ID allocation/query in 3.12 seconds, keeps every KV PID - and verifies Group-0 lease availability during the fault. Earlier passes are - retained; final fresh-outage writes, allocation audit and cleanup follow. -- The resumed final gate passes: a second unique outage witness, all-five exact - reads, stable service PIDs, fresh native batch/allocation and all Group-0/1/3 - applied checkpoints. The final scan finds 1774 live mirrored segments with - no cross-chunk physical overlap. All five labeled containers/volumes are - removed, all 85 owned endpoints are closed, and key-only host SSH still works. - Final frozen service logs were archived after stopping each container, with - no concurrent-write tar caveat. The separate 512-record capacity deadline - failure remains a measured limit; 294 exact records are the recovery witness. - Evidence: `attempt17/repaired-matrix-sequence.json`, `final-closed/`, - `final-all-replicas-caught-up.json`, `physical-five-occupancy.log`, `cleanup.json`. -- Complete storage tests and formatting pass after the physical matrix. The - current isolated Journal browser case passes in 3268 ms; its original ordered - native-operations group then stops at Shell's KV-stop command. Three earlier - cases pass and are retained. Whole-workspace clippy and all 277 framework - tests pass before the newly diagnosed stop-receipt repair. -- Shell's trace shows Stop requested at `00:45:35.108Z`, KV exiting about - 0.1 seconds later, and the identical SSH request timing out after another - three-second authority read. The target had been Group-0 leader; its durable - execution record already says complete. The isolated Shell case passes in - 4.3 seconds, then a real-authority/owned-child regression reproduces receipt - failure after authority loss. Exact completed KV-stop receipts now carry - bootstrap proof and return without re-execution; legacy, foreign, incomplete, - retired and live-replacement states still require current authority. - -### Stop-receipt and settled-Journal verification - -- Full Monitor tests, affected Monitor/E2E clippy, the 277 fixture tests, - bundled native-browser type checks, workspace fmt and required `tree-lint` - pass. The C++ gate exited zero with existing warnings in unchanged sources. -- The release image containing the completed-stop receipt fix built in 139.7 s; - Docker execution identity is - `sha256:fea066994ab45918a04335fe93b00044aebb3fe73b958278806536a90aa05cba`. -- Exact Shell recovery passes: Playwright 6.5 s, initial election 3385 ms; - the stopped leader's duplicate operation returns its immutable receipt. -- Exact Journal recovery passes: placement settles in 502 ms before the browser, - Playwright 4.6 s, owner restart response 2290 ms. The measured pre-change - browser baseline was 3.268 s; the new time is below twice that baseline. - All navigation, identity, owner-loss/restart, stale-cursor and cleanup assertions - retain their original deadlines. -- Evidence: `ci/run-41128190276/receipt-fixture-gates.json` and its named logs. - The initial direct-source TypeScript invocation lacked the serializer snapshot - supplied by `bundle_client`; the passing check uses that same snapshot layout. -- Continue the native-operations selection in its original relative order: - retain unchanged binding/CRUD passes and rerun the KV-stop lifecycle case, - failed Shell case and all fifteen previously unrun cases (17 remaining). - -### Complete native operations and explicit KV-only witness setup - -- The native-operations closing run actually executed all original 19 cases, - despite the launcher-only remainder filter described above. All 19 passed; - `native-operations-complete-coverage.json` validates the exact original case - set against `results.json`. `native-resume-orchestration-note.json` preserves - the planned/actual selection difference. The resume helper now filters both - launcher and executor selections; no assertions or resources changed. -- The deployment-services closing selection stopped at - `deployment::kv_bootstrap::monitor_creates_real_kv_groups_then_validates_after_restart`: - `/stores/0/groups/1/ready` returned 404 (`group 1 not found in store 0`). Seven - earlier/in-flight cases passed and remain valid. -- This KV-only fixture must explicitly request its ordinary witness Group 1: - `/system/prepare` accepts bootstrap ownership; it does not create that group. - Both KV-only witness tests now assert no default-preparation record and no - existing Group 1, create the group once through the production management API, - and await its leader under the existing management four-second setup bound. - Group-0/1 identity, readiness, foreign-bootstrap rejection, exact data and - retained initialization checks remain. Restart does not recreate the group. -- Repair sequence: affected E2E clippy/fmt, both exact KV-only cases, then the - 24 unrun deployment cases in original relative order. Combined with the seven - valid passes and two focused cases, this preserves all 33 deployment cases. - -### Sustained terminal-writer fault versus automatic DiskIO recovery - -- Console API stopped at `console::cli_container::terminal_chunk_writer_recovery`. - The correct health endpoints returned fresh `ready: true` after the nominal - forty-second pause. Six independent cases passed, including the formerly - failing plain Docker attachment; retain them for the resumed selection. -- Monitor lifecycle logs prove the fault healed before the thirty-second I/O - deadline: DiskIO PIDs 1414/594/631 were replaced by 2344/1522/1515 around - 19–27 seconds after fault submission. The healthy responses were not stale. -- The injector now holds each verified owned Monitor while pausing DiskIO, - verifies all six original processes remain stopped after forty seconds, - and resumes DiskIO before Monitor. Production recovery policy, original fault - duration, health assertions, terminal-replay proof and restart/data checks - remain intact. No host-wide settings change. -- An `ExitStack` restores every paused process even if another resume fails; - client reaping still runs on failure. Two cleanup regressions and all 279 - fixture tests pass. The focused terminal-writer test is running. -- Deployment's complete 33-case union and Native Operations' exact 19-case set - are verified by the named `*-complete-coverage.json` records. - -### Replace supervisor suspension with an owned network fault - -- Holding Monitor also suspends its child-log pumps. That attempted injector - produced health-listener curl timeouts while all six PIDs remained stopped; - it is a failed observation, not a pass. Preserve `terminal-fault-held-focused.log` - and `run-fd6b4633ab9c4f0e981d5a62da2387fa` for that first divergence. -- Use a forty-second packet drop on each accepted DiskIO TCP listener instead. - The helper joins only an authenticated owned CLI bridge container's network - namespace, receives only NET_ADMIN, and mounts a fingerprinted host netfilter - tool snapshot read-only. It never joins the host namespace or edits host rules. - Existing Monitor, log pumping and health samplers continue during the outage; - automatic DiskIO replacement cannot heal the blocked listener early. -- A disposable private-namespace probe passed add/check/delete of the exact - DROP rule using the current release image. ExitStack cleanup handles each - jump, rule and chain independently, including partial setup and a failed - removal; focused unit coverage also rejects host-network targets before any - helper execution. The original health/currentness, terminal writer restart, - unchanged Access PIDs, exact data and operation-fencing assertions remain. - -### Bound the sustained GC load at its original cadence - -- Native Operations 19/19, deployment-services 33/33 and Console API 92/92 - complete coverage is verified. GC cleanup's six cases passed; convergence - failed at 96/128 files after otherwise monotonic revisions. -- The convergence failure is an explained process exit: the sole Chunk-KV - container has `OOMKilled=true`, exit 137, with its unchanged 512 MiB limit. - Four zero-delay SDK workers created 45,828 table cycles before that exit, - versus the original minimum 48. Keep the frozen state and counters in - `gc-zero-delay-oom-proof.json`; do not count this overload as a passing run. -- Retain the original 500 ms inter-table interval while running until cleanup - completes. Stagger four workers by quarter-cycle offsets so 250 ms deletion - observations do not repeatedly alias all workers' gaps. Sleeps stay outside - latency measurements; the minimum counts, real deletion overlap, 64 small - writes and original test deadlines remain. The separate zero-delay foreground - probe keeps its existing zero-delay/yield behavior. -- Repeat convergence and the restart matrix with that shared pressure change. - The five other cleanup cases retain valid passes because their load inputs - are unchanged. Preserve the complete failed long-run evidence. - -### Closing verification and publication - -- Both paced GC repetitions pass: restart matrix 132.54 s; convergence 479.44 s - including fixture lifecycle, with 471.46 s client execution. The convergence - task reaches `Complete`, deletes all 128 source files and verifies their 1554 - logical bytes. The original 64 foreground writes, minimum SDK operation counts - and measured overlap with actual deletion intervals all pass. The failed - zero-delay overload remains separate evidence. -- Complete affected container coverage is 151 distinct cases: Native Operations - 19, deployment-services 33, Console API 92, GC cleanup 6 and GC convergence 1. - Their `*-complete-coverage.json` records identify the actual result files and - distinguish retained unchanged passes from repeated changed cases. The two - shared GC pressure consumers were both repeated after restoring the cadence. -- Core, storage, Protocol, Web, Monitor, Console Shared, KV Client, ChunkDB and - Chunk Stream verification pass, along with workspace fmt, whole-workspace and - subsequent affected-crate clippy, native-browser type checks, all 280 fixture - tests and required C++ tree lint. The final post-pressure E2E clippy/fmt and - framework checks pass. No production changes followed the final release-image - build; later changes are tests, fixture tooling, CI prerequisites and docs. -- Rechecked run `41003611925`: eight failed jobs, repaired and locally verified - before the first publication. Rechecked run `41128190276`: five failed jobs, - repaired and verified by the follow-up coverage above. The historical job - conclusions are retained; local passes do not change those remote conclusions. -- The user renewed commit/push authorization after temporarily deferring push. - The completed execution plan is removed for this publication. Cleanup-tool - changes belonging to the user's concurrent work are excluded from the commit. -- Physical scope remains five containers across two hosts, with complete - recovery/cleanup results. The separate 512-record/default-five-second capacity - failure is unresolved as a capacity limit; the 294 acknowledged records are - verified recovery witnesses only. Earlier archives deleted during user disk - cleanup cannot be recovered or claimed as retained. diff --git a/e2e-test/oci_reference_recovery.py b/e2e-test/oci_reference_recovery.py index 715685956..537b8f65a 100644 --- a/e2e-test/oci_reference_recovery.py +++ b/e2e-test/oci_reference_recovery.py @@ -129,6 +129,15 @@ def replace_addresses(cluster, before): def verify(cluster, cluster_id, witness, credentials): assert cluster.implementation == 'docker' and cluster.node_count == 3 + # This case submits one explicit later-member admission. Stop automatic + # admission before exposing that candidate, rather than racing two callers. + for index in range(3): + disabled = admission_request(origin(cluster, index), '/api/node/auto-join', + {'enabled': False}) + assert disabled['enabled'] is False + wait(lambda index=index: request(origin(cluster, index), '/api/node/auto-join'), + lambda value: value['observation']['phase'] == 'disabled', + 'automatic admission quiesced before manual admission', seconds=10) cluster.expected_names.append(cluster.run_id + '-node4') create_nonvoter(cluster) cluster.command('start', cluster.expected_names[3]) diff --git a/e2e-test/tests/deployment/kv_bootstrap.rs b/e2e-test/tests/deployment/kv_bootstrap.rs index 6dc8b00ac..295064038 100644 --- a/e2e-test/tests/deployment/kv_bootstrap.rs +++ b/e2e-test/tests/deployment/kv_bootstrap.rs @@ -36,6 +36,7 @@ async fn monitor_creates_real_kv_groups_then_validates_after_restart() { if write { create_witness_group().await; } + wait_for_witness_group().await; for group in [0, 1] { let ready = request( &http, @@ -76,7 +77,6 @@ async fn monitor_creates_real_kv_groups_then_validates_after_restart() { pub(super) async fn create_witness_group() { use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; - use std::time::{Duration, Instant}; let origin = format!("http://{}:10000", crate::common::management::addresses(1)[0]); let kv = CrowdbKvClient::new(ClientConfig::new(vec![origin.clone()])); @@ -113,17 +113,28 @@ pub(super) async fn create_witness_group() { 201, ) .await; + wait_for_witness_group().await; +} + +async fn wait_for_witness_group() { + use std::time::{Duration, Instant}; + + let origin = format!("http://{}:10000", crate::common::management::addresses(1)[0]); + let http = client(); let deadline = Instant::now() + Duration::from_secs(4); loop { - let ready = request( - &http, - &origin, - Method::GET, - "/stores/0/groups/1/ready", - Value::Null, - 200, - ) - .await; + let response = http + .get(format!("{origin}/stores/0/groups/1/ready")) + .send() + .await + .expect("witness group readiness request"); + let status = response.status(); + let ready: Value = response.json().await.expect("witness group readiness JSON"); + assert!( + matches!(status.as_u16(), 200 | 503), + "unexpected witness readiness HTTP {status}: {ready}" + ); + assert_eq!(status.as_u16() == 200, ready["ready"] == true, "{ready}"); if ready["ready"] == true { assert_eq!(ready["leader_id"], 1); assert_eq!(ready["voting_replicas"], 1); diff --git a/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored.rs b/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored.rs index 22fb4183a..db33f5218 100644 --- a/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored.rs +++ b/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored.rs @@ -17,6 +17,7 @@ use crate::{DiskWriter, STREAM_CHUNK_BYTES}; mod read; use read::MirroredWindowReader; +mod order; struct MirroredLayout { chunk: Arc, @@ -59,6 +60,7 @@ impl MirroredFrameReaders { return Err(invalid_layout("chunk purpose, owner, epoch or state is invalid")); } let (capacity, replicas) = validate_layout(&chunk, chunk_id)?; + let chunk = order::by_rack(chunk, disk_io.as_ref()); let layout = Arc::new(MirroredLayout { chunk, disk_io, diff --git a/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored/order.rs b/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored/order.rs new file mode 100644 index 000000000..3d26a93fe --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored/order.rs @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Keep each observation lane in a consistent failure domain when possible. + +use crate::DiskWriter; +use crowdb_protocol::chunkdb::rpc::{Chunk, Strip}; +use std::{collections::HashMap, sync::Arc}; + +pub(super) fn by_rack(mut chunk: Arc, disk_io: &dyn DiskWriter) -> Arc { + let Some(topology) = disk_io.disk_topology() else { + return chunk; + }; + let placements: HashMap<_, _> = topology + .disks() + .map(|disk| { + ( + (disk.disk_id.high, disk.disk_id.low), + (disk.rack_id, disk.node_id, disk.disk_group_id), + ) + }) + .collect(); + // Retained routing is an ordering hint, never authority to add or omit a + // replica. Incomplete hints preserve the supplied layout's original order. + let complete = chunk.strips.iter().all(|strip| match &strip.strip { + Some(Strip::MirrorStrip(mirror)) => mirror.segments.iter().all(|segment| { + segment + .disk_id + .is_some_and(|id| placements.contains_key(&(id.high, id.low))) + }), + _ => false, + }); + if !complete { + return chunk; + } + for strip in &mut Arc::make_mut(&mut chunk).strips { + if let Some(Strip::MirrorStrip(mirror)) = &mut strip.strip { + mirror.segments.sort_by_key(|segment| { + segment + .disk_id + .and_then(|id| placements.get(&(id.high, id.low))) + .copied() + }); + } + } + chunk +} diff --git a/lib/crowdb-chunk-client/src/writer/small_pipeline.rs b/lib/crowdb-chunk-client/src/writer/small_pipeline.rs index b0fb92b64..24d91c855 100644 --- a/lib/crowdb-chunk-client/src/writer/small_pipeline.rs +++ b/lib/crowdb-chunk-client/src/writer/small_pipeline.rs @@ -76,6 +76,7 @@ impl ManagedPipeline { } mod batch; +mod liveness; mod prefetch; mod worker; @@ -659,25 +660,6 @@ impl OwnedChunk { self.policy.chunk_capacity.saturating_sub(self.cursor) } - async fn renew_liveness(&mut self) -> Result<()> { - self.flush_pending_advance().await?; - let chunk_id = self - .chunk - .id - .ok_or_else(|| IoError::AllocationFailed("shared chunk missing id".into()))?; - self.chunk = advance_chunk( - Arc::clone(&self.allocator), - chunk_id, - self.writer_epoch, - self.chunk.modify_ts, - self.cursor, - None, - u64::try_from(self.policy.writer_lease.as_millis()).unwrap_or(u64::MAX), - ) - .await?; - Ok(()) - } - fn current_strip(&self) -> Result<&crowdb_protocol::chunkdb::rpc::ChunkStrip> { self.chunk .strips diff --git a/lib/crowdb-chunk-client/src/writer/small_pipeline/liveness.rs b/lib/crowdb-chunk-client/src/writer/small_pipeline/liveness.rs new file mode 100644 index 000000000..f1125f1e1 --- /dev/null +++ b/lib/crowdb-chunk-client/src/writer/small_pipeline/liveness.rs @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{advance_chunk, Arc, ChunkState, IoError, OwnedChunk, Result}; + +impl OwnedChunk { + pub(super) async fn renew_liveness(&mut self) -> Result<()> { + let staged = self + .staged_reservation + .map(|(_, _, sequence)| { + self.chunk + .strips + .iter() + .find(|strip| strip.strip_sequence == sequence) + .cloned() + .ok_or_else(|| IoError::MetadataConflict("staged reservation strip is missing".into())) + }) + .transpose()?; + self.flush_pending_advance().await?; + let chunk_id = self + .chunk + .id + .ok_or_else(|| IoError::AllocationFailed("shared chunk missing id".into()))?; + let mut refreshed = advance_chunk( + Arc::clone(&self.allocator), + chunk_id, + self.writer_epoch, + self.chunk.modify_ts, + self.cursor, + None, + u64::try_from(self.policy.writer_lease.as_millis()).unwrap_or(u64::MAX), + ) + .await?; + if refreshed.id != Some(chunk_id) + || refreshed.writer_epoch != self.writer_epoch + || refreshed.state != ChunkState::Active as i32 + { + return Err(IoError::MetadataConflict( + "shared chunk ownership changed during lease renewal".into(), + )); + } + if let Some(staged) = staged { + // Unconsumed reservations are absent from the readable layout. + // Retain this local write view without advancing published capacity. + match refreshed + .strips + .iter() + .find(|strip| strip.strip_sequence == staged.strip_sequence) + { + Some(current) if current != &staged => { + return Err(IoError::MetadataConflict( + "staged reservation changed during lease renewal".into(), + )); + } + Some(_) => {} + None => refreshed.strips.push(staged), + } + } + self.chunk = refreshed; + Ok(()) + } +} diff --git a/lib/crowdb-chunk-client/tests/common/mirrored_frames.rs b/lib/crowdb-chunk-client/tests/common/mirrored_frames.rs index 2af4fa8b9..e11906deb 100644 --- a/lib/crowdb-chunk-client/tests/common/mirrored_frames.rs +++ b/lib/crowdb-chunk-client/tests/common/mirrored_frames.rs @@ -64,6 +64,8 @@ impl TestMirroredFrames { calls: AtomicUsize::new(0), short: AtomicBool::new(false), delay_ms: AtomicU64::new(0), + topology: None, + slow_disk: None, }); Self { chunk, @@ -79,10 +81,15 @@ pub struct TestFrameDisks { pub calls: AtomicUsize, pub short: AtomicBool, pub delay_ms: AtomicU64, + pub topology: Option, + pub slow_disk: Option<(u64, std::time::Duration)>, } #[async_trait] impl DiskWriter for TestFrameDisks { + fn disk_topology(&self) -> Option { + self.topology.clone() + } async fn write(&self, _: &Segment, _: u64, _: Bytes) -> crowdb_chunk_client::Result<()> { Err(IoError::WriteFailed("read-only observation fixture".into())) } @@ -104,6 +111,11 @@ impl DiskWriter for TestFrameDisks { ) -> crowdb_chunk_client::Result { self.calls.fetch_add(1, Ordering::Relaxed); let delay = self.delay_ms.load(Ordering::Relaxed); + if let Some((disk, duration)) = self.slow_disk { + if segment.disk_id.is_some_and(|id| id.high == disk) { + tokio::time::sleep(duration).await; + } + } if delay > 0 { tokio::time::sleep(std::time::Duration::from_millis(delay)).await; } diff --git a/lib/crowdb-chunk-client/tests/frame_observation_test/mirrored.rs b/lib/crowdb-chunk-client/tests/frame_observation_test/mirrored.rs index ae512c44f..b9e454e4b 100644 --- a/lib/crowdb-chunk-client/tests/frame_observation_test/mirrored.rs +++ b/lib/crowdb-chunk-client/tests/frame_observation_test/mirrored.rs @@ -8,6 +8,66 @@ use std::sync::atomic::Ordering; use super::*; use crate::mirrored_fixture::TestMirroredFrames as Layout; +#[tokio::test(start_paused = true)] +async fn alternating_mirror_order_cannot_force_every_observation_across_a_slow_rack() { + use crowdb_diskio_client::{DiskId, DiskTopologyView, TestDiskRoute}; + + let mut layout = Layout::new(); + for (index, strip) in layout.chunk.strips.iter_mut().enumerate() { + if index % 2 == 1 { + let Some(Strip::MirrorStrip(mirrors)) = &mut strip.strip else { + panic!("mirror layout"); + }; + mirrors.segments.reverse(); + } + } + let topology = DiskTopologyView::from_routes_for_tests( + (1..=2) + .map(|disk| TestDiskRoute { + disk_id: DiskId::new(disk, 1), + rack_id: 3 - disk, + node_id: disk, + disk_group_id: disk, + instance_id: disk, + endpoint: format!("127.0.0.1:{}", 9 + disk), + }) + .collect(), + ); + let disks = Arc::get_mut(&mut layout.disks).unwrap(); + disks.topology = Some(topology); + disks.slow_disk = Some((1, Duration::from_millis(60))); + disks.delay_ms.store(1, Ordering::Relaxed); + let original = Arc::new(layout.chunk); + let deadline = Instant::now() + Duration::from_millis(100); + let readers = + MirroredFrameReaders::new(original.clone(), Layout::NAME, layout.disks.clone(), deadline).unwrap(); + let selected = select_frame_prefix( + readers.readers(), + readers.scan_config(0, 0, 0, 64 * 1024 * 1024).unwrap(), + 65536, + deadline, + ) + .await + .unwrap(); + assert_eq!(selected.healthy_replicas(), 1); + assert_eq!(selected.observation().summary().physical_end, layout.body_length); + let mut sink = TestExtentSink::default(); + replay_frame_prefix( + readers.readers()[selected.selected_index()].as_ref(), + selected.observation(), + deadline, + &mut sink, + ) + .await + .unwrap(); + assert_eq!(sink.records.len(), 1); + assert!(Instant::now() < deadline); + let Some(Strip::MirrorStrip(mirrors)) = &original.strips[1].strip else { + panic!("mirror layout"); + }; + assert_eq!(mirrors.segments[0].disk_id.unwrap().high, 2); +} + #[tokio::test] async fn pinned_layout_reads_grouped_data_beyond_old_cursor_across_frames_and_strips() { let layout = Layout::new(); @@ -41,6 +101,35 @@ async fn pinned_layout_reads_grouped_data_beyond_old_cursor_across_frames_and_st assert_eq!(sink.records[0].logical_length, 100_000); } +#[tokio::test(start_paused = true)] +async fn incomplete_topology_hints_preserve_even_the_unmapped_replica() { + use crowdb_diskio_client::{DiskId, DiskTopologyView, TestDiskRoute}; + + let mut layout = Layout::new(); + let disks = Arc::get_mut(&mut layout.disks).unwrap(); + disks.topology = Some(DiskTopologyView::from_routes_for_tests(vec![TestDiskRoute { + disk_id: DiskId::new(2, 1), + rack_id: 1, + node_id: 2, + disk_group_id: 2, + instance_id: 2, + endpoint: "127.0.0.1:11".into(), + }])); + disks.slow_disk = Some((1, Duration::from_millis(40))); + let deadline = Instant::now() + Duration::from_millis(20); + let readers = MirroredFrameReaders::new( + Arc::new(layout.chunk), + Layout::NAME, + layout.disks.clone(), + deadline, + ) + .unwrap(); + assert_eq!(readers.readers().len(), 2); + assert!(readers.readers()[0].read_window(0, 512).await.is_err()); + assert_eq!(Instant::now(), deadline); + assert_eq!(layout.disks.calls.load(Ordering::Relaxed), 1); +} + #[tokio::test] async fn unavailable_replica_is_excluded_without_reading_or_mutating_its_layout() { let mut layout = Layout::new(); diff --git a/lib/crowdb-chunk-client/tests/small_object_test.rs b/lib/crowdb-chunk-client/tests/small_object_test.rs index 96fa371ed..0896d45b9 100644 --- a/lib/crowdb-chunk-client/tests/small_object_test.rs +++ b/lib/crowdb-chunk-client/tests/small_object_test.rs @@ -33,6 +33,9 @@ mod durable; #[path = "common/small_native.rs"] mod native; +#[path = "small_object_test/staged_reservation.rs"] +mod staged_reservation; + fn frame_bytes(payload_bytes: usize) -> u64 { u64::try_from(payload_bytes).unwrap() + 35 } @@ -109,7 +112,7 @@ impl ChunkAllocator for MockAllocator { high: u64::try_from(req.chunk_type).unwrap() << 56, low, }); - let strips: Vec<_> = (0..req.strip_count.max(1)) + let strips: Vec<_> = (0..req.strip_count) .map(|sequence| make_strip(chunk_id, sequence, req.copy_count.max(1))) .collect(); let chunk = Chunk { @@ -126,7 +129,7 @@ impl ChunkAllocator for MockAllocator { acknowledged_cursor: 0, closed_strip_sequence: None, writer_lease_deadline_ms: req.writer_lease_ms, - next_strip_sequence: req.strip_count.max(1), + next_strip_sequence: req.strip_count, cleanup_intents: vec![], last_strip_replacement: None, owner_key: req.owner_key, diff --git a/lib/crowdb-chunk-client/tests/small_object_test/staged_reservation.rs b/lib/crowdb-chunk-client/tests/small_object_test/staged_reservation.rs new file mode 100644 index 000000000..15202c44c --- /dev/null +++ b/lib/crowdb-chunk-client/tests/small_object_test/staged_reservation.rs @@ -0,0 +1,117 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test(start_paused = true)] +async fn lease_renewal_preserves_the_unpublished_first_strip() { + let mut configured = policy(); + configured.conversion_enabled = true; + configured.memory_budget = 64 * 1024 * 1024; + configured.chunk_capacity = 2 * 1024 * 1024; + configured.small_strip_prefetch_count = 2; + let renewal = configured.writer_lease / 2; + let (client, allocator, disk) = client(configured); + let mut writer = client.prepare_small_write(16).await.unwrap(); + for _ in 0..1000 { + if allocator.reserve_calls.load(Ordering::Relaxed) > 0 { + break; + } + tokio::task::yield_now().await; + } + assert_eq!(allocator.reserve_calls.load(Ordering::Relaxed), 1); + assert_eq!( + allocator + .state + .lock() + .unwrap() + .chunks + .values() + .next() + .unwrap() + .capacity, + 0 + ); + tokio::time::sleep(renewal + Duration::from_millis(1)).await; + for _ in 0..1000 { + if allocator.snapshot().2 > 0 { + break; + } + tokio::task::yield_now().await; + } + assert!( + allocator.snapshot().2 > 0, + "the empty writer must renew before submission" + ); + { + let state = allocator.state.lock().unwrap(); + let published = state.chunks.values().next().unwrap(); + assert_eq!(published.capacity, 0); + assert!( + published.strips.is_empty(), + "renewal must not publish an unwritten strip" + ); + } + writer + .on_data(Bytes::from_static(b"0123456789abcdef")) + .await + .unwrap(); + let locations = writer.finish_durable().await.unwrap(); + assert_eq!(locations.len(), 1); + assert_eq!(locations[0].offset, 0); + for (_, _, image) in disk.writes.lock().unwrap().iter() { + let frame = parse_frame(image, locations[0].chunk_id.unwrap()).unwrap(); + assert_eq!(frame.payload, b"0123456789abcdef"); + } + assert_eq!(disk.calls(), 3); + client.shutdown_small_writes().await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn an_idle_prepared_replacement_still_writes_at_zero_after_renewal() { + let mut configured = policy(); + configured.conversion_enabled = true; + configured.memory_budget = 64 * 1024 * 1024; + configured.chunk_capacity = 2 * 1024 * 1024; + configured.small_strip_prefetch_count = 2; + let renewal = configured.writer_lease / 2; + let (client, allocator, _) = client(configured); + let payload = Bytes::from(vec![0x5a; MAX_SMALL]); + let mut writer = client.prepare_small_write(MAX_SMALL).await.unwrap(); + writer.on_data(payload.clone()).await.unwrap(); + let original = writer.finish_durable().await.unwrap()[0].chunk_id; + for _ in 0..1000 { + if allocator.snapshot().0 == 2 { + break; + } + tokio::task::yield_now().await; + } + assert_eq!( + allocator.snapshot().0, + 2, + "a replacement must be prepared before renewal" + ); + tokio::time::sleep(renewal + Duration::from_millis(1)).await; + { + let state = allocator.state.lock().unwrap(); + assert!(state.advances >= 2, "current and prepared chunks must both renew"); + let replacement = state.chunks.get(&(original.unwrap().high, 2)).unwrap(); + assert!(replacement.modify_ts >= 3, "the prepared replacement must renew"); + assert_eq!(replacement.capacity, 0); + assert!(replacement.strips.is_empty()); + } + for _ in 1..32 { + let mut writer = client.prepare_small_write(MAX_SMALL).await.unwrap(); + writer.on_data(payload.clone()).await.unwrap(); + assert_eq!(writer.finish_durable().await.unwrap()[0].chunk_id, original); + } + let mut writer = client.prepare_small_write(16).await.unwrap(); + writer + .on_data(Bytes::from_static(b"replacement-data")) + .await + .unwrap(); + let location = writer.finish_durable().await.unwrap().remove(0); + assert_ne!(location.chunk_id, original); + assert_eq!(location.offset, 0); + client.shutdown_small_writes().await.unwrap(); +} diff --git a/lib/crowdb-chunk-kv/src/partition.rs b/lib/crowdb-chunk-kv/src/partition.rs index a9b3af1b1..67f77e6a2 100644 --- a/lib/crowdb-chunk-kv/src/partition.rs +++ b/lib/crowdb-chunk-kv/src/partition.rs @@ -13,6 +13,7 @@ mod ingress; mod journal; mod maintenance; mod observation; +mod replay; mod scan; mod split; mod transfer; @@ -24,6 +25,7 @@ pub use frontier::JournalFrontier; pub use ingress::SplitIngress; pub use journal::{PartitionJournal, StreamPartitionJournal}; pub use observation::TreeObservation; +use replay::{replay_suffix, validate_replay_record, ReplayState}; pub use split::{PreparedSplit, PreparedSplitWriter, SplitHandoffStore, SplitWriterTarget}; pub use tree::{CrowdbPartitionTree, PartitionTree}; use worker_sender::WorkerSender; @@ -228,15 +230,6 @@ struct RecoverySeed { recovered: bool, } -struct ReplayState { - seed: RecoverySeed, - checkpoint_applied_seq: u64, - stream_name: StreamName, - retained_results: usize, - replayed: HashMap, - last_new_sequence: Option, -} - struct SplitTransition { plan: SplitPlan, artifact: Option, @@ -1900,6 +1893,7 @@ async fn replay_child_overlay( let mut replay = ReplayState { seed, checkpoint_applied_seq: cutover_seq, + applied_bytes_since_drain: 0, stream_name: journal.stream_name(), retained_results, replayed: HashMap::new(), @@ -1935,185 +1929,6 @@ async fn replay_child_overlay( Ok(replay.seed) } -async fn replay_suffix( - partition_id: PartitionId, - ownership_epoch: u64, - range: &PartitionRange, - checkpoint: &Checkpoint, - retained_results: usize, - tree: &dyn PartitionTree, - journal: &dyn PartitionJournal, -) -> Result { - const WINDOW_BYTES: usize = 1024 * 1024; - let tail = journal.tail(); - if checkpoint.replay_offset > tail { - return Err(ChunkKvError::JournalCorruption( - "checkpoint replay offset exceeds journal tail".into(), - )); - } - let mut replay = ReplayState::new(checkpoint, retained_results); - let mut read_offset = checkpoint.replay_offset; - let mut frame_offset = read_offset; - let mut buffered = BytesMut::new(); - while read_offset < tail { - let bytes = journal.read_window(read_offset, WINDOW_BYTES).await?; - if bytes.is_empty() { - return Err(ChunkKvError::JournalCorruption( - "journal returned no bytes before durable tail".into(), - )); - } - read_offset = read_offset - .checked_add(bytes.len() as u64) - .ok_or_else(|| ChunkKvError::JournalCorruption("journal replay offset overflows".into()))?; - buffered.extend_from_slice(&bytes); - loop { - let decoded = decode_frame(&buffered)?; - let FrameDecode::Complete(decoded) = decoded else { - break; - }; - validate_replay_record(partition_id, ownership_epoch, &decoded.record)?; - let belongs = range.contains(decoded.record.operation.key()); - replay - .process(tree, frame_offset, decoded.record, belongs) - .await?; - let consumed = decoded.bytes_consumed; - buffered.advance(consumed); - frame_offset = frame_offset - .checked_add(consumed as u64) - .ok_or_else(|| ChunkKvError::JournalCorruption("journal frame offset overflows".into()))?; - } - if buffered.len() > crate::MAX_FRAME_BYTES { - return Err(ChunkKvError::JournalCorruption( - "incomplete frame exceeds maximum size".into(), - )); - } - } - if !buffered.is_empty() { - return Err(ChunkKvError::IncompleteFrame); - } - Ok(replay.seed) -} - -impl ReplayState { - fn new(checkpoint: &Checkpoint, retained_results: usize) -> Self { - Self { - seed: RecoverySeed { - applied_seq: checkpoint.applied_seq, - applied_position: checkpoint.replay_offset, - retry_replay_offset: checkpoint.replay_offset, - results: HashMap::new(), - result_order: std::collections::VecDeque::new(), - expired_floor: HashMap::new(), - recovered: true, - }, - checkpoint_applied_seq: checkpoint.applied_seq, - stream_name: checkpoint.stream_name, - retained_results, - replayed: HashMap::new(), - last_new_sequence: None, - } - } - - async fn process( - &mut self, - tree: &dyn PartitionTree, - frame_offset: u64, - record: WalRecord, - belongs_to_partition: bool, - ) -> Result<()> { - if let Some(previous) = self.replayed.get(&record.mutation_seq) { - if previous != &record { - return Err(ChunkKvError::JournalCorruption( - "conflicting duplicate mutation sequence".into(), - )); - } - return Ok(()); - } - if let Some(last) = self.last_new_sequence { - let expected = last.checked_add(1).ok_or_else(|| { - ChunkKvError::JournalCorruption("journal mutation sequence overflows".into()) - })?; - if expected != record.mutation_seq { - return Err(ChunkKvError::JournalCorruption(format!( - "journal mutation sequence has a gap: expected {expected}, got {} at offset {frame_offset}", - record.mutation_seq - ))); - } - } - if record.mutation_seq > self.checkpoint_applied_seq { - let expected = self - .seed - .applied_seq - .checked_add(1) - .ok_or_else(|| ChunkKvError::JournalCorruption("mutation sequence overflows".into()))?; - if record.mutation_seq != expected { - return Err(ChunkKvError::JournalCorruption(format!( - "journal mutation sequence has a gap after checkpoint {}: expected {expected}, got {} at offset {frame_offset}", - self.checkpoint_applied_seq, record.mutation_seq - ))); - } - if belongs_to_partition && record.result.applied() { - tree.apply(record.mutation_seq, &record.operation) - .await - .map_err(|error| { - ChunkKvError::Internal(format!( - "replay tree apply failed at mutation sequence {}: {error}", - record.mutation_seq - )) - })?; - } else { - tree.advance_noop(record.mutation_seq).await.map_err(|error| { - ChunkKvError::Internal(format!( - "replay tree no-op failed at mutation sequence {}: {error}", - record.mutation_seq - )) - })?; - } - self.seed.applied_seq = record.mutation_seq; - } - self.seed.applied_position = frame_offset; - if belongs_to_partition { - let response = MutationResponse { - mutation_seq: record.mutation_seq, - result: record.result.clone(), - journal_position: JournalPosition { - stream_name: self.stream_name, - offset: frame_offset, - }, - }; - retain_recovered( - &mut self.seed, - self.retained_results, - record.request_id, - record.operation_digest, - response, - )?; - } - self.last_new_sequence = Some(record.mutation_seq); - self.replayed.insert(record.mutation_seq, record); - Ok(()) - } -} - -fn validate_replay_record(partition_id: PartitionId, ownership_epoch: u64, record: &WalRecord) -> Result<()> { - if record.partition_id != partition_id || record.ownership_epoch > ownership_epoch { - return Err(ChunkKvError::JournalCorruption( - "WAL record partition or epoch is invalid".into(), - )); - } - if record.operation_digest != canonical_operation_digest(&record.operation) { - return Err(ChunkKvError::JournalCorruption( - "WAL operation digest mismatch".into(), - )); - } - match record.result { - MutationResult::Applied { revision } if revision != record.mutation_seq => Err( - ChunkKvError::JournalCorruption("applied revision differs from mutation sequence".into()), - ), - _ => Ok(()), - } -} - fn retain_recovered( seed: &mut RecoverySeed, retained_results: usize, diff --git a/lib/crowdb-chunk-kv/src/partition/replay.rs b/lib/crowdb-chunk-kv/src/partition/replay.rs new file mode 100644 index 000000000..ace9e3ac0 --- /dev/null +++ b/lib/crowdb-chunk-kv/src/partition/replay.rs @@ -0,0 +1,236 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Streaming WAL replay with compact, complete-record duplicate identities. + +use super::{ + canonical_operation_digest, decode_frame, retain_recovered, Buf, BytesMut, Checkpoint, ChunkKvError, + FrameDecode, HashMap, JournalPosition, MutationResponse, MutationResult, PartitionId, PartitionJournal, + PartitionRange, PartitionTree, RecoverySeed, Result, StreamName, WalRecord, +}; +use sha2::{Digest, Sha256}; +use std::io::Write; + +pub(super) struct ReplayState { + pub(super) seed: RecoverySeed, + pub(super) checkpoint_applied_seq: u64, + pub(super) stream_name: StreamName, + pub(super) retained_results: usize, + pub(super) replayed: HashMap, + pub(super) last_new_sequence: Option, + pub(super) applied_bytes_since_drain: usize, +} + +pub(super) async fn replay_suffix( + partition_id: PartitionId, + ownership_epoch: u64, + range: &PartitionRange, + checkpoint: &Checkpoint, + retained_results: usize, + tree: &dyn PartitionTree, + journal: &dyn PartitionJournal, +) -> Result { + const WINDOW_BYTES: usize = 1024 * 1024; + let tail = journal.tail(); + if checkpoint.replay_offset > tail { + return Err(ChunkKvError::JournalCorruption( + "checkpoint replay offset exceeds journal tail".into(), + )); + } + let mut replay = ReplayState::new(checkpoint, retained_results); + let mut read_offset = checkpoint.replay_offset; + let mut frame_offset = read_offset; + let mut buffered = BytesMut::new(); + while read_offset < tail { + let bytes = journal.read_window(read_offset, WINDOW_BYTES).await?; + if bytes.is_empty() { + return Err(ChunkKvError::JournalCorruption( + "journal returned no bytes before durable tail".into(), + )); + } + read_offset = read_offset + .checked_add(bytes.len() as u64) + .ok_or_else(|| ChunkKvError::JournalCorruption("journal replay offset overflows".into()))?; + buffered.extend_from_slice(&bytes); + loop { + let decoded = decode_frame(&buffered)?; + let FrameDecode::Complete(decoded) = decoded else { + break; + }; + validate_replay_record(partition_id, ownership_epoch, &decoded.record)?; + let belongs = range.contains(decoded.record.operation.key()); + replay + .process(tree, frame_offset, decoded.record, belongs) + .await?; + let consumed = decoded.bytes_consumed; + buffered.advance(consumed); + frame_offset = frame_offset + .checked_add(consumed as u64) + .ok_or_else(|| ChunkKvError::JournalCorruption("journal frame offset overflows".into()))?; + } + if buffered.len() > crate::MAX_FRAME_BYTES { + return Err(ChunkKvError::JournalCorruption( + "incomplete frame exceeds maximum size".into(), + )); + } + } + if !buffered.is_empty() { + return Err(ChunkKvError::IncompleteFrame); + } + Ok(replay.seed) +} + +impl ReplayState { + pub(super) fn new(checkpoint: &Checkpoint, retained_results: usize) -> Self { + Self { + seed: RecoverySeed { + applied_seq: checkpoint.applied_seq, + applied_position: checkpoint.replay_offset, + retry_replay_offset: checkpoint.replay_offset, + results: HashMap::new(), + result_order: std::collections::VecDeque::new(), + expired_floor: HashMap::new(), + recovered: true, + }, + checkpoint_applied_seq: checkpoint.applied_seq, + stream_name: checkpoint.stream_name, + retained_results, + replayed: HashMap::new(), + last_new_sequence: None, + applied_bytes_since_drain: 0, + } + } + + pub(super) async fn process( + &mut self, + tree: &dyn PartitionTree, + frame_offset: u64, + record: WalRecord, + belongs_to_partition: bool, + ) -> Result<()> { + let fingerprint = record_fingerprint(&record)?; + if let Some(previous) = self.replayed.get(&record.mutation_seq) { + if previous != &fingerprint { + return Err(ChunkKvError::JournalCorruption( + "conflicting duplicate mutation sequence".into(), + )); + } + return Ok(()); + } + if let Some(last) = self.last_new_sequence { + let expected = last.checked_add(1).ok_or_else(|| { + ChunkKvError::JournalCorruption("journal mutation sequence overflows".into()) + })?; + if expected != record.mutation_seq { + return Err(ChunkKvError::JournalCorruption(format!( + "journal mutation sequence has a gap: expected {expected}, got {} at offset {frame_offset}", + record.mutation_seq + ))); + } + } + if record.mutation_seq > self.checkpoint_applied_seq { + let expected = self + .seed + .applied_seq + .checked_add(1) + .ok_or_else(|| ChunkKvError::JournalCorruption("mutation sequence overflows".into()))?; + if record.mutation_seq != expected { + return Err(ChunkKvError::JournalCorruption(format!( + "journal mutation sequence has a gap after checkpoint {}: expected {expected}, got {} at offset {frame_offset}", + self.checkpoint_applied_seq, record.mutation_seq + ))); + } + if belongs_to_partition && record.result.applied() { + tree.apply(record.mutation_seq, &record.operation) + .await + .map_err(|error| { + ChunkKvError::Internal(format!( + "replay tree apply failed at mutation sequence {}: {error}", + record.mutation_seq + )) + })?; + self.applied_bytes_since_drain = self + .applied_bytes_since_drain + // Include arena/index overhead for small values and deletes. + .saturating_add(256) + .saturating_add(record.operation.key().len()) + .saturating_add(record.operation.successful_value().map_or(0, <[u8]>::len)); + if self.applied_bytes_since_drain >= 16 * 1024 * 1024 { + tree.drain_replay().await?; + self.applied_bytes_since_drain = 0; + } + } else { + tree.advance_noop(record.mutation_seq).await.map_err(|error| { + ChunkKvError::Internal(format!( + "replay tree no-op failed at mutation sequence {}: {error}", + record.mutation_seq + )) + })?; + } + self.seed.applied_seq = record.mutation_seq; + } + self.seed.applied_position = frame_offset; + if belongs_to_partition { + let response = MutationResponse { + mutation_seq: record.mutation_seq, + result: record.result.clone(), + journal_position: JournalPosition { + stream_name: self.stream_name, + offset: frame_offset, + }, + }; + retain_recovered( + &mut self.seed, + self.retained_results, + record.request_id, + record.operation_digest, + response, + )?; + } + self.last_new_sequence = Some(record.mutation_seq); + self.replayed.insert(record.mutation_seq, fingerprint); + Ok(()) + } +} + +pub(super) fn validate_replay_record( + partition_id: PartitionId, + ownership_epoch: u64, + record: &WalRecord, +) -> Result<()> { + if record.partition_id != partition_id || record.ownership_epoch > ownership_epoch { + return Err(ChunkKvError::JournalCorruption( + "WAL record partition or epoch is invalid".into(), + )); + } + if record.operation_digest != canonical_operation_digest(&record.operation) { + return Err(ChunkKvError::JournalCorruption( + "WAL operation digest mismatch".into(), + )); + } + match record.result { + MutationResult::Applied { revision } if revision != record.mutation_seq => Err( + ChunkKvError::JournalCorruption("applied revision differs from mutation sequence".into()), + ), + _ => Ok(()), + } +} + +fn record_fingerprint(record: &WalRecord) -> Result<[u8; 32]> { + struct HashWriter(Sha256); + impl Write for HashWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.update(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut writer = HashWriter(Sha256::new()); + writer.0.update(b"crowdb-chunk-kv-replay-record-v1"); + bincode::serialize_into(&mut writer, record).map_err(|error| { + ChunkKvError::JournalCorruption(format!("WAL record fingerprint failed: {error}")) + })?; + Ok(writer.0.finalize().into()) +} diff --git a/lib/crowdb-chunk-kv/src/partition/transfer.rs b/lib/crowdb-chunk-kv/src/partition/transfer.rs index 24a695951..989a00730 100644 --- a/lib/crowdb-chunk-kv/src/partition/transfer.rs +++ b/lib/crowdb-chunk-kv/src/partition/transfer.rs @@ -229,6 +229,7 @@ async fn catch_up_inner(state: &mut super::WorkerState, request: &TransferCatchU let mut replay = super::ReplayState { seed, checkpoint_applied_seq: current.applied_seq, + applied_bytes_since_drain: 0, stream_name: request.parent_journal.stream_name(), retained_results: state.config.retained_results, replayed: HashMap::new(), diff --git a/lib/crowdb-chunk-kv/src/partition/tree.rs b/lib/crowdb-chunk-kv/src/partition/tree.rs index f507dde5b..c12de5d29 100644 --- a/lib/crowdb-chunk-kv/src/partition/tree.rs +++ b/lib/crowdb-chunk-kv/src/partition/tree.rs @@ -47,6 +47,12 @@ pub trait PartitionTree: Send + Sync { ) -> Result<(Vec, bool)>; async fn apply(&self, mutation_seq: u64, operation: &MutationOperation) -> Result<()>; async fn advance_noop(&self, mutation_seq: u64) -> Result<()>; + /// Drains replay's transient buffers without publishing a checkpoint or trimming WAL. + /// # Errors + /// Returns an in-memory tree drain or storage-read failure. + async fn drain_replay(&self) -> Result<()> { + Ok(()) + } async fn checkpoint(&self, wal_replay_offset: u64) -> Result<(u64, u64)>; async fn checkpoint_snapshot( &self, @@ -418,6 +424,18 @@ impl PartitionTree for CrowdbPartitionTree { Ok(()) } + async fn drain_replay(&self) -> Result<()> { + let tree = std::sync::Arc::clone(&self.tree); + let runtime = tokio::runtime::Handle::current(); + tokio::task::spawn_blocking(move || { + let writer = crowdb_tree_ffi::AsyncCrowdbtree::from_shared(tree); + runtime.block_on(writer.flush()) + }) + .await + .map_err(|error| ChunkKvError::TreeUnavailable(error.to_string()))? + .map_err(map_tree_apply_error) + } + async fn checkpoint(&self, wal_replay_offset: u64) -> Result<(u64, u64)> { if let Some(page_store) = self.config.as_ref().and_then(|config| config.page_store.as_ref()) { page_store diff --git a/lib/crowdb-chunk-kv/tests/common/generated_journal.rs b/lib/crowdb-chunk-kv/tests/common/generated_journal.rs new file mode 100644 index 000000000..13bb72d01 --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/common/generated_journal.rs @@ -0,0 +1,121 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_chunk_kv::{ + canonical_operation_digest, encode_frame, Checkpoint, ChunkKvError, JournalPosition, MutationOperation, + MutationResult, PartitionId, PartitionJournal, RequestId, Result, WalRecord, +}; +use crowdb_chunk_stream::StreamName; + +pub struct TestGeneratedJournal { + count: u64, + frame_bytes: usize, + duplicate: Option, + request_conflict: bool, +} + +impl TestGeneratedJournal { + pub fn new(count: u64, duplicate: Option, request_conflict: bool) -> Self { + Self { + count, + frame_bytes: encode_frame(&Self::record(1, false)).unwrap().len(), + duplicate, + request_conflict, + } + } + + pub fn checkpoint(&self) -> Checkpoint { + Checkpoint { + tree_id: 1, + tree_manifest: 0, + root_manifest_generation: 0, + applied_seq: 0, + stream_name: self.stream_name(), + stream_generation: 1, + replay_offset: 0, + } + } + + fn record(sequence: u64, changed: bool) -> WalRecord { + let operation = MutationOperation::Put { + key: b"overwritten".to_vec(), + value: vec![if changed { 255 } else { (sequence % 251) as u8 }; 65536], + }; + WalRecord { + partition_id: PartitionId { high: 1, low: 1 }, + ownership_epoch: 1, + mutation_seq: sequence, + request_id: RequestId { + client_high: 1, + client_low: 1, + client_sequence: sequence, + }, + operation_digest: canonical_operation_digest(&operation), + result: MutationResult::Applied { revision: sequence }, + operation, + } + } +} + +#[async_trait] +impl PartitionJournal for TestGeneratedJournal { + async fn append_frames(&self, _: &[Bytes]) -> Result> { + Err(ChunkKvError::Internal("generated journal is read-only".into())) + } + + async fn read_window(&self, offset: u64, max_bytes: usize) -> Result { + let length = usize::try_from(self.tail().saturating_sub(offset)) + .unwrap() + .min(max_bytes); + let mut out = Vec::with_capacity(length); + while out.len() < length { + let position = usize::try_from(offset).unwrap() + out.len(); + let ordinal = position / self.frame_bytes; + let within = position % self.frame_bytes; + let (sequence, changed) = if ordinal as u64 == self.count { + (1, self.duplicate.unwrap()) + } else { + (ordinal as u64 + 1, false) + }; + let mut record = Self::record(sequence, changed); + if ordinal as u64 == self.count && self.request_conflict { + record.request_id.client_low = 2; + } + let frame = encode_frame(&record)?; + assert_eq!(frame.len(), self.frame_bytes); + let count = (frame.len() - within).min(length - out.len()); + out.extend_from_slice(&frame[within..within + count]); + } + Ok(Bytes::from(out)) + } + + async fn trim_prefix(&self, _: u64) -> Result { + Ok(0) + } + + async fn reclaim_metadata_before(&self, _: u64, _: usize) -> Result { + Ok(0) + } + + async fn checkpoint(&self) -> Result { + Ok(1) + } + + async fn close(&self) -> Result<()> { + Ok(()) + } + + fn stream_name(&self) -> StreamName { + StreamName { high: 1, low: 1 } + } + + fn record_generation(&self) -> u64 { + 1 + } + + fn tail(&self) -> u64 { + (self.count + u64::from(self.duplicate.is_some())) * self.frame_bytes as u64 + } +} diff --git a/lib/crowdb-chunk-kv/tests/replay_memory_test.rs b/lib/crowdb-chunk-kv/tests/replay_memory_test.rs new file mode 100644 index 000000000..8e0538f5a --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/replay_memory_test.rs @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_chunk_kv::{ + memory::MemoryPartitionTree, ChunkKvError, Partition, PartitionConfig, PartitionId, PartitionRange, +}; + +#[path = "common/generated_journal.rs"] +mod journal; + +async fn recover(count: u64, duplicate: Option) -> crowdb_chunk_kv::Result { + recover_journal(journal::TestGeneratedJournal::new(count, duplicate, false)).await +} + +async fn recover_journal(journal: journal::TestGeneratedJournal) -> crowdb_chunk_kv::Result { + let journal = Arc::new(journal); + Partition::recover( + PartitionId { high: 1, low: 1 }, + PartitionRange::default(), + 1, + journal.checkpoint(), + PartitionConfig::default(), + Arc::new(MemoryPartitionTree::default()), + journal, + ) + .await +} + +#[cfg(target_os = "linux")] +fn peak_rss_kib() -> u64 { + std::fs::read_to_string("/proc/self/status") + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("VmHWM:")) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap() +} + +#[cfg(target_os = "linux")] +#[tokio::test] +async fn replay_does_not_retain_half_a_gibibyte_of_overwritten_payloads() { + let before = peak_rss_kib(); + let partition = recover(8192, None).await.unwrap(); + let peak_growth = peak_rss_kib().saturating_sub(before); + println!("WAL replay peak RSS growth: {peak_growth} KiB for 512 MiB of payloads"); + assert!( + peak_growth < 64 * 1024, + "replay retained payload history: {peak_growth} KiB" + ); + let value = partition.get(1, b"overwritten", None).await.unwrap().unwrap(); + assert_eq!(value.revision, 8192); + assert_eq!(value.value, vec![(8192 % 251) as u8; 65536]); +} + +#[tokio::test] +async fn a_late_exact_duplicate_remains_valid() { + let partition = recover(32, Some(false)).await.unwrap(); + assert_eq!(partition.snapshot().applied_seq, 32); +} + +#[tokio::test] +async fn a_late_conflicting_duplicate_is_still_corruption() { + let result = recover(32, Some(true)).await; + assert!(matches!(result, Err(ChunkKvError::JournalCorruption(message)) + if message == "conflicting duplicate mutation sequence")); +} + +#[tokio::test] +async fn identical_operations_with_changed_request_identity_are_conflicting_duplicates() { + let journal = journal::TestGeneratedJournal::new(32, Some(false), true); + let result = recover_journal(journal).await; + assert!(matches!(result, Err(ChunkKvError::JournalCorruption(message)) + if message == "conflicting duplicate mutation sequence")); +} diff --git a/lib/crowdb-chunk-kv/tests/replay_tree_memory_test.rs b/lib/crowdb-chunk-kv/tests/replay_tree_memory_test.rs new file mode 100644 index 000000000..16a3bd85f --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/replay_tree_memory_test.rs @@ -0,0 +1,58 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#![cfg(target_os = "linux")] + +use std::sync::Arc; + +use crowdb_chunk_kv::{CrowdbPartitionTree, Partition, PartitionConfig, PartitionId, PartitionRange}; + +#[path = "common/generated_journal.rs"] +mod journal; + +fn peak_rss_kib() -> u64 { + std::fs::read_to_string("/proc/self/status") + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("VmHWM:")) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap() +} + +#[tokio::test] +async fn native_replay_drains_overwritten_memtables_without_publishing_a_checkpoint() { + let journal = Arc::new(journal::TestGeneratedJournal::new(8192, None, false)); + let config = crowdb_tree_ffi::Config { + page_store: Some(Arc::new(crowdb_tree_ffi::PageStore::open_mem(65536).unwrap())), + ..crowdb_tree_ffi::Config::default() + }; + let tree = Arc::new(CrowdbPartitionTree::open(1, &config).unwrap()); + let before = peak_rss_kib(); + let partition = Partition::recover( + PartitionId { high: 1, low: 1 }, + PartitionRange::default(), + 1, + journal.checkpoint(), + PartitionConfig::default(), + tree, + journal, + ) + .await + .unwrap(); + let peak_growth = peak_rss_kib().saturating_sub(before); + println!("Native tree replay peak RSS growth: {peak_growth} KiB for 512 MiB of overwritten values"); + assert!( + peak_growth < 96 * 1024, + "unflushed replay buffers retained {peak_growth} KiB" + ); + let value = partition.get(1, b"overwritten", None).await.unwrap().unwrap(); + assert_eq!(value.revision, 8192); + assert_eq!(value.value, vec![(8192 % 251) as u8; 65536]); + let observed = partition.observe_tree().unwrap(); + assert_eq!(observed.checkpoint_manifest, 0); + assert_eq!(observed.checkpoint_applied_seq, 0); +} diff --git a/lib/crowdb-chunk-stream/src/memory.rs b/lib/crowdb-chunk-stream/src/memory.rs index 73ac9fc8b..eeddd3563 100644 --- a/lib/crowdb-chunk-stream/src/memory.rs +++ b/lib/crowdb-chunk-stream/src/memory.rs @@ -395,6 +395,10 @@ impl StreamRegistry for MemoryStreamStore { #[async_trait] impl StreamChunkStore for MemoryStreamStore { + fn capacity_limit(&self) -> u64 { + self.chunk_capacity + } + fn bind_writer(&self, _name: StreamName, _epoch: u64, _session: Arc) {} fn mirror_copies(&self) -> u32 { 1 diff --git a/lib/crowdb-chunk-stream/src/production_chunk.rs b/lib/crowdb-chunk-stream/src/production_chunk.rs index 2c55905d3..0a9fe25bc 100644 --- a/lib/crowdb-chunk-stream/src/production_chunk.rs +++ b/lib/crowdb-chunk-stream/src/production_chunk.rs @@ -227,6 +227,10 @@ fn update_state(state: &ChunkStateView, chunk: Chunk) { #[async_trait] impl StreamChunkStore for ProductionStreamChunkStore { + fn capacity_limit(&self) -> u64 { + self.chunk_capacity_bytes + } + fn bind_writer(&self, name: StreamName, epoch: u64, session: Arc) { self.owners.insert(name, (epoch, session)); } @@ -234,7 +238,22 @@ impl StreamChunkStore for ProductionStreamChunkStore { self.mirror_copies } async fn current_capacity(&self, chunk_id: ChunkId) -> Result { - let state = self.state(chunk_id).await?; + // Opening a newer checkpoint can reuse an adapter that observed this + // chunk before a different owner grew it. Cached layout is not an + // authoritative bound for that checkpoint. + let chunk = self + .allocator + .query_chunk(QueryChunkRequest { + chunk_id: Some(chunk_id), + }) + .await + .map_err(io_error)? + .chunk + .ok_or_else(|| StreamError::ReadUnavailable("stream chunk is missing".into()))?; + if chunk.id != Some(chunk_id) { + return Err(StreamError::Corruption("stream chunk identity differs".into())); + } + let state = self.install(chunk)?; chunk_capacity(&state.chunk.load()) } async fn allocate_mirrored( diff --git a/lib/crowdb-chunk-stream/src/production_chunk/allocation.rs b/lib/crowdb-chunk-stream/src/production_chunk/allocation.rs index a096ea596..e2e5767b7 100644 --- a/lib/crowdb-chunk-stream/src/production_chunk/allocation.rs +++ b/lib/crowdb-chunk-stream/src/production_chunk/allocation.rs @@ -78,7 +78,9 @@ impl ProductionStreamChunkStore { // Await every submitted synchronization even if another strip failed. let mut pending = FuturesUnordered::new(); let mut strips = strips.iter(); - for strip in strips.by_ref().take(8) { + // Bound mirrored zeroing bursts so initialization leaves transport + // capacity for writes using the already initialized prefix. + for strip in strips.by_ref().take(2) { pending.push(self.initialize_strip(chunk, strip)); } let mut error = None; diff --git a/lib/crowdb-chunk-stream/src/production_chunk/growth.rs b/lib/crowdb-chunk-stream/src/production_chunk/growth.rs index 5250c405c..96dfdc891 100644 --- a/lib/crowdb-chunk-stream/src/production_chunk/growth.rs +++ b/lib/crowdb-chunk-stream/src/production_chunk/growth.rs @@ -13,9 +13,6 @@ use crowdb_protocol::chunkdb::rpc::{ AppendChunkRequest, AppendChunkResponse, Chunk, QueryChunkRequest, Strip, StripType, }; -const MIN_GROWTH_WINDOW_BYTES: u64 = 2 * STREAM_STRIP_BYTES; -const MAX_GROWTH_WINDOW_BYTES: u64 = 8 * STREAM_STRIP_BYTES; - impl ProductionStreamChunkStore { pub(super) async fn grow_active_chunk( &self, @@ -51,16 +48,11 @@ impl ProductionStreamChunkStore { capacity, })); } - // Small appends must not wait for a large initialization burst. - // Large appends amortize allocation over several following writes. - let window = required_capacity - .saturating_sub(write_cursor(&state)) - .saturating_mul(4) - .clamp(MIN_GROWTH_WINDOW_BYTES, MAX_GROWTH_WINDOW_BYTES) - .next_power_of_two(); + // The stream worker has already budgeted its lookahead. Do not + // multiply speculative bytes again at the physical allocation layer. let target = required_capacity - .div_ceil(window) - .saturating_mul(window) + .div_ceil(STREAM_STRIP_BYTES) + .saturating_mul(STREAM_STRIP_BYTES) .min(self.chunk_capacity_bytes); let growth_timer = PhaseTimer::start(&perf().growth); let response = self.request_growth(&chunk, target, capacity).await?; diff --git a/lib/crowdb-chunk-stream/src/production_chunk/recovery/drain.rs b/lib/crowdb-chunk-stream/src/production_chunk/recovery/drain.rs index e7901ccee..e505c4e53 100644 --- a/lib/crowdb-chunk-stream/src/production_chunk/recovery/drain.rs +++ b/lib/crowdb-chunk-stream/src/production_chunk/recovery/drain.rs @@ -12,6 +12,8 @@ use crowdb_protocol::{ }; use futures::{stream, StreamExt}; +type StripDrainResults = Vec<(Segment, Result<()>)>; + impl ProductionStreamChunkStore { pub(super) async fn drain_recovery_layout( &self, @@ -19,7 +21,8 @@ impl ProductionStreamChunkStore { context: &StreamWriteContext, deadline: std::time::Instant, ) -> Result<()> { - for (index, strip) in chunk.strips.iter().enumerate() { + let drained = self.drain_recovery_segments(chunk, context, deadline).await?; + for (index, (strip, results)) in chunk.strips.iter().zip(drained).enumerate() { let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { return Err(StreamError::Corruption("recovery chunk is not mirrored".into())); }; @@ -28,21 +31,6 @@ impl ProductionStreamChunkStore { "recovery mirror repair is pending".into(), )); } - let results = stream::iter(mirror.segments.iter().copied()) - .map(|segment| async move { - let result = async { - context.check_io()?; - self.disk_writer - .drain_segment(&segment, u64::from(strip.unit_kb) * 1024, deadline) - .await - .map_err(io_error) - } - .await; - (segment, result) - }) - .buffer_unordered(8) - .collect::>() - .await; let failed: Vec<_> = mirror .segments .iter() @@ -70,6 +58,52 @@ impl ProductionStreamChunkStore { Ok(()) } + async fn drain_recovery_segments( + &self, + chunk: &Chunk, + context: &StreamWriteContext, + deadline: std::time::Instant, + ) -> Result> { + let mut segments = Vec::with_capacity(chunk.strips.len() * 2); + let mut drained = Vec::with_capacity(chunk.strips.len()); + for (index, strip) in chunk.strips.iter().enumerate() { + let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { + return Err(StreamError::Corruption("recovery chunk is not mirrored".into())); + }; + if !strip.unavailable_segments.is_empty() { + return Err(StreamError::ReadUnavailable( + "recovery mirror repair is pending".into(), + )); + } + drained.push(Vec::with_capacity(mirror.segments.len())); + segments.extend( + mirror + .segments + .iter() + .map(|segment| (index, u64::from(strip.unit_kb) * 1024, *segment)), + ); + } + let results = stream::iter(segments) + .map(|(index, unit_bytes, segment)| async move { + let result = async { + context.check_io()?; + self.disk_writer + .drain_segment(&segment, unit_bytes, deadline) + .await + .map_err(io_error) + } + .await; + (index, segment, result) + }) + .buffer_unordered(8) + .collect::>() + .await; + for (index, segment, result) in results { + drained[index].push((segment, result)); + } + Ok(drained) + } + async fn mark_recovery_mirrors( &self, chunk: &Chunk, diff --git a/lib/crowdb-chunk-stream/src/storage.rs b/lib/crowdb-chunk-stream/src/storage.rs index 9640a4f15..d5570218c 100644 --- a/lib/crowdb-chunk-stream/src/storage.rs +++ b/lib/crowdb-chunk-stream/src/storage.rs @@ -74,6 +74,11 @@ pub trait StreamMetadataStore: Send + Sync { #[async_trait] pub trait StreamChunkStore: Send + Sync { + /// Maximum physical capacity supported by one chunk in this store. + fn capacity_limit(&self) -> u64 { + crowdb_chunk_client::STREAM_CHUNK_BYTES + } + /// Required synchronized replica count for append traffic accounting. fn mirror_copies(&self) -> u32; fn bind_writer(&self, name: StreamName, epoch: u64, session: std::sync::Arc); diff --git a/lib/crowdb-chunk-stream/src/stream/capacity_plan.rs b/lib/crowdb-chunk-stream/src/stream/capacity_plan.rs index bf9ce1026..30e20c12a 100644 --- a/lib/crowdb-chunk-stream/src/stream/capacity_plan.rs +++ b/lib/crowdb-chunk-stream/src/stream/capacity_plan.rs @@ -5,17 +5,20 @@ use std::time::Duration; -const MIN_AHEAD: u64 = 2 * 1024 * 1024; +const MIN_AHEAD: u64 = crowdb_protocol::frame::MAX_FRAME_BYTES as u64; +const MAX_AHEAD: u64 = crowdb_chunk_client::STREAM_STRIP_BYTES; const INITIAL_PREPARATION: Duration = Duration::from_millis(20); #[derive(Default)] pub(super) struct CapacityPlan { bytes_per_second: u64, preparation: Duration, + last_batch_bytes: u64, } impl CapacityPlan { pub(super) fn record_get(&mut self, bytes: u64, elapsed: Duration) { + self.last_batch_bytes = bytes; let rate = u64::try_from(u128::from(bytes) * 1_000_000_000 / elapsed.as_nanos().max(1)).unwrap_or(u64::MAX); // React immediately to acceleration; decay gradually after a slow write. @@ -31,18 +34,7 @@ impl CapacityPlan { if capacity >= limit { return None; } - // Cover twice the observed preparation time, including allocation and every - // replica's initialization. Bound speculative capacity independently of - // stream length and leave room for the batch already being submitted. - let latency = self.preparation.max(INITIAL_PREPARATION).as_nanos(); - let ahead = u64::try_from( - u128::from(self.bytes_per_second) - .saturating_mul(latency) - .saturating_mul(2) - / 1_000_000_000, - ) - .unwrap_or(u64::MAX) - .clamp(MIN_AHEAD, limit.max(MIN_AHEAD)); + let ahead = self.ahead(limit); if capacity.saturating_sub(cursor) > ahead.saturating_add(batch_bytes) { return None; } @@ -54,6 +46,34 @@ impl CapacityPlan { .min(limit), ) } + + pub(super) fn successor_target(&self, cursor: u64, limit: u64) -> Option { + let ahead = self.ahead(limit); + if limit.saturating_sub(cursor) > ahead.saturating_add(self.last_batch_bytes) { + return None; + } + Some( + ahead + .saturating_mul(2) + .saturating_add(self.last_batch_bytes) + .min(limit), + ) + } + + fn ahead(&self, limit: u64) -> u64 { + // Cover twice the observed preparation time, including allocation and every + // replica's initialization. Bound speculative capacity independently of + // stream length and leave room for the batch already being submitted. + let latency = self.preparation.max(INITIAL_PREPARATION).as_nanos(); + u64::try_from( + u128::from(self.bytes_per_second) + .saturating_mul(latency) + .saturating_mul(2) + / 1_000_000_000, + ) + .unwrap_or(u64::MAX) + .clamp(MIN_AHEAD, limit.clamp(MIN_AHEAD, MAX_AHEAD)) + } } #[cfg(feature = "test-util")] diff --git a/lib/crowdb-chunk-stream/src/stream/successor.rs b/lib/crowdb-chunk-stream/src/stream/successor.rs index 4436bbac7..8e84958cf 100644 --- a/lib/crowdb-chunk-stream/src/stream/successor.rs +++ b/lib/crowdb-chunk-stream/src/stream/successor.rs @@ -19,6 +19,22 @@ pub(super) struct SuccessorPreparation { } pub(super) fn start(state: &mut WorkerState) { + let Some(active) = state.progress.active.as_ref() else { + return; + }; + let Some(required) = state.capacity_plan.successor_target( + active.acknowledged_cursor, + state + .config + .chunk_capacity_bytes + .min(state.chunks.capacity_limit()), + ) else { + return; + }; + prepare(state, required); +} + +fn prepare(state: &mut WorkerState, required: u64) { let Some(authority) = state.authority.as_ref() else { return; }; @@ -35,15 +51,7 @@ pub(super) fn start(state: &mut WorkerState) { epoch: state.writer_epoch, purpose: state.progress.purpose, interval: state.config.liveness_interval, - required: state - .capacity_plan - .target( - 0, - 0, - state.config.chunk_capacity_bytes, - state.config.batch_bytes as u64, - ) - .expect("nonzero chunk capacity"), + required, }; let (handoff, receive) = oneshot::channel(); state.successor = Some(SuccessorPreparation { @@ -200,7 +208,9 @@ async fn allocate( pub(super) async fn take(state: &mut WorkerState) -> Result { let Some(preparation) = state.successor.take() else { - start(state); + // A waiting rotation requires a candidate regardless of the last + // consumption sample. Grow it further only for the actual next append. + prepare(state, 1); let preparation = state.successor.take().ok_or(StreamError::WriteStalled)?; return join(preparation, true).await; }; diff --git a/lib/crowdb-chunk-stream/src/stream/worker.rs b/lib/crowdb-chunk-stream/src/stream/worker.rs index bfdeecfed..e231a8823 100644 --- a/lib/crowdb-chunk-stream/src/stream/worker.rs +++ b/lib/crowdb-chunk-stream/src/stream/worker.rs @@ -170,6 +170,8 @@ async fn process_append_batch( let result = write_with_recovery(state, &requests, bytes).await; match result { Ok(ranges) => { + preparation::start(state, bytes); + successor::start(state); for (request, range) in requests.into_iter().zip(ranges) { state .queued_bytes diff --git a/lib/crowdb-chunk-stream/tests/capacity_plan_test.rs b/lib/crowdb-chunk-stream/tests/capacity_plan_test.rs index deb0d6dfd..a63e3892f 100644 --- a/lib/crowdb-chunk-stream/tests/capacity_plan_test.rs +++ b/lib/crowdb-chunk-stream/tests/capacity_plan_test.rs @@ -11,19 +11,19 @@ const MIB: u64 = 1024 * 1024; fn bandwidth_and_preparation_latency_increase_the_reserved_capacity() { let plan = |bytes, millis| { capacity_target_for_tests( - &[(bytes, Duration::from_millis(100))], + &[(bytes, Duration::from_millis(125))], Duration::from_millis(millis), 0, - 8 * MIB, + MIB / 4, 256 * MIB, 4096, ) }; - assert_eq!(plan(MIB, 50), None, "a slow writer has ample ready capacity"); - let fast = plan(20 * MIB, 50).unwrap(); - let slower_preparation = plan(20 * MIB, 100).unwrap(); - assert_eq!(fast, 40 * MIB + 4096); - assert_eq!(slower_preparation, 80 * MIB + 4096); + assert_eq!(plan(4096, 50), None, "a slow writer has ample ready capacity"); + let fast = plan(MIB / 8, 250).unwrap(); + let slower_preparation = plan(MIB / 8, 500).unwrap(); + assert_eq!(fast, MIB + 4096); + assert_eq!(slower_preparation, 2 * MIB + 4096); } #[test] @@ -38,20 +38,21 @@ fn refill_starts_with_two_preparation_intervals_still_ready() { 4096, ) }; - assert!(request(21 * MIB + 4096 + 1).is_none()); - assert!(request(21 * MIB + 4096).is_some()); + assert!(request(2 * MIB + 4096 + 1).is_none()); + assert!(request(2 * MIB + 4096).is_some()); } #[test] fn cold_start_extreme_rates_and_chunk_end_remain_bounded() { assert_eq!( capacity_target_for_tests(&[], Duration::ZERO, 0, MIB, 256 * MIB, 4096), - Some(4 * MIB + 4096) + None, + "a cold stream must consume its first strip before speculative growth" ); let writes = [(u64::MAX, Duration::ZERO)]; assert_eq!( capacity_target_for_tests(&writes, Duration::MAX, MIB, 2 * MIB, 256 * MIB, 4096), - Some(256 * MIB) + Some(3 * MIB + 4096) ); assert_eq!( capacity_target_for_tests(&writes, Duration::MAX, MIB, 256 * MIB, 256 * MIB, 4096), @@ -68,13 +69,14 @@ fn one_slow_write_does_not_remove_a_fast_writers_margin() { ], Duration::from_millis(50), 0, - 8 * MIB, + MIB, 256 * MIB, 4096, ) .unwrap(); - assert!( - target >= 35 * MIB, - "a slow sample must not immediately shrink the reserve" + assert_eq!( + target, + 2 * MIB + 4096, + "one slow sample preserves the bounded reserve" ); } diff --git a/lib/crowdb-chunk-stream/tests/common/initialization_window.rs b/lib/crowdb-chunk-stream/tests/common/initialization_window.rs new file mode 100644 index 000000000..a5532676e --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/common/initialization_window.rs @@ -0,0 +1,81 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::Disks; +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_chunk_client::{DiskWriter, IoError}; +use crowdb_protocol::diskdb::rpc::Segment; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +pub(crate) struct TestInitializationWindow { + pub(crate) inner: Arc, + limit: usize, + active: AtomicUsize, + rejected: AtomicUsize, +} + +impl TestInitializationWindow { + pub(crate) fn new(inner: Arc, limit: usize) -> Self { + Self { + inner, + limit, + active: AtomicUsize::new(0), + rejected: AtomicUsize::new(0), + } + } + + pub(crate) fn rejected(&self) -> usize { + self.rejected.load(Ordering::Acquire) + } +} + +#[async_trait] +impl DiskWriter for TestInitializationWindow { + async fn write( + &self, + segment: &Segment, + unit_bytes: u64, + data: Bytes, + ) -> crowdb_chunk_client::Result<()> { + self.inner.write(segment, unit_bytes, data).await + } + + async fn write_at_byte_offset( + &self, + segment: &Segment, + unit_bytes: u64, + offset: u64, + data: Bytes, + ) -> crowdb_chunk_client::Result<()> { + self.inner + .write_at_byte_offset(segment, unit_bytes, offset, data) + .await + } + + async fn initialize_synchronized_at_byte_offset( + &self, + segment: &Segment, + unit_bytes: u64, + offset: u64, + data: Bytes, + ) -> crowdb_chunk_client::Result<()> { + let active = self.active.fetch_add(1, Ordering::AcqRel) + 1; + let result = if active > self.limit { + self.rejected.fetch_add(1, Ordering::AcqRel); + Err(IoError::WriteFailed( + "bounded initialization transport is full".into(), + )) + } else { + tokio::task::yield_now().await; + self.inner + .initialize_synchronized_at_byte_offset(segment, unit_bytes, offset, data) + .await + }; + self.active.fetch_sub(1, Ordering::AcqRel); + result + } +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test.rs index ec21915da..16272acd4 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test.rs @@ -40,6 +40,12 @@ mod local_progress; #[path = "production_chunk_test/growth.rs"] mod growth; +#[path = "production_chunk_test/capacity_refresh.rs"] +mod capacity_refresh; + +#[path = "common/initialization_window.rs"] +mod initialization_window; + #[path = "production_chunk_test/preparation.rs"] mod preparation; @@ -654,7 +660,7 @@ async fn production_store_grows_and_writes_across_mirror_strips() { .unwrap(); assert_eq!(active.capacity, 1024 * 1024); let grown = store - .grow_mirrored(name, 9, active.chunk_id, active.capacity + 1) + .grow_mirrored(name, 9, active.chunk_id, 8 * 1024 * 1024) .await .unwrap() .unwrap(); @@ -878,7 +884,10 @@ async fn stream_repair_preserves_a_batch_crossing_mirror_strips() { created_at_ms: StreamRecord::creation_time_ms(), }, 9, - StreamConfig::default(), + StreamConfig { + chunk_capacity_bytes: 2 * 1024 * 1024, + ..StreamConfig::default() + }, Arc::clone(&metadata) as Arc, metadata as Arc, chunks, @@ -899,7 +908,7 @@ async fn stream_repair_preserves_a_batch_crossing_mirror_strips() { assert_eq!(&read[prefix.len()..], suffix.as_ref()); assert_eq!(stream.metrics().rollovers, 0); let chunk = allocator.chunk.lock().unwrap().clone().unwrap(); - assert_eq!(chunk.strips.len(), 8); + assert_eq!(chunk.strips.len(), 2); let Some(Strip::MirrorStrip(mirror)) = &chunk.strips[1].strip else { panic!("expected mirror strip"); }; diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/capacity_refresh.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/capacity_refresh.rs new file mode 100644 index 000000000..53bc47def --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/capacity_refresh.rs @@ -0,0 +1,77 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_chunk_stream::{StreamError, StreamPurpose}; + +#[tokio::test] +async fn reopening_refreshes_cached_capacity_after_another_owner_grows_the_chunk() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let make_chunks = || { + Arc::new( + ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(), + ) + }; + let metadata = Arc::new(MemoryStreamStore::new(1)); + let name = StreamName { high: 277, low: 281 }; + let writer = ChunkStream::create( + StreamRecord { + metadata_group_id: 7, + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, StreamPurpose::Stream) + }, + 9, + StreamConfig::default(), + metadata.clone(), + metadata.clone(), + make_chunks(), + ) + .await + .unwrap(); + let chunk_id = allocator.chunk.lock().unwrap().as_ref().unwrap().id.unwrap(); + let observer = make_chunks(); + assert_eq!(observer.current_capacity(chunk_id).await.unwrap(), 1024 * 1024); + let payload = Bytes::from(vec![0x69; 1536 * 1024]); + writer.append(std::slice::from_ref(&payload)).await.unwrap(); + writer.checkpoint().await.unwrap(); + writer.release_writer().await.unwrap(); + let reader = ChunkStream::open_read_only( + name, + 9, + StreamConfig::default(), + metadata.clone(), + metadata.clone(), + observer.clone(), + ) + .await + .expect("a retained adapter must refresh capacity before validating the newer checkpoint"); + assert_eq!(reader.read_at(0, payload.len()).await.unwrap(), payload); + allocator.fail_query_once.store(true, Ordering::Release); + assert!(observer.current_capacity(chunk_id).await.is_err()); + { + let mut guard = allocator.chunk.lock().unwrap(); + let chunk = guard.as_mut().unwrap(); + chunk.strips.truncate(1); + chunk.capacity = 1024; + chunk.modify_ts += 1; + } + let result = ChunkStream::open_read_only( + name, + 9, + StreamConfig::default(), + metadata.clone(), + metadata, + observer, + ) + .await; + assert!( + matches!(result, Err(StreamError::Corruption(message)) if message.contains("checkpoint exceeds chunk capacity")) + ); +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/growth.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/growth.rs index a5d5c683d..7c9cb3b7a 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test/growth.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/growth.rs @@ -3,6 +3,36 @@ use super::*; +#[tokio::test] +async fn growth_respects_bounded_initialization_admission() { + use super::initialization_window::TestInitializationWindow; + + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let window = Arc::new(TestInitializationWindow::new(disks.clone(), 4)); + let store = + ProductionStreamChunkStore::new(allocator, window.clone(), 30_000, ChunkReadPolicy::default()) + .unwrap(); + let name = StreamName { high: 197, low: 199 }; + let active = store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + let grown = store + .grow_mirrored(name, 9, active.chunk_id, 8 * 1024 * 1024) + .await + .unwrap() + .unwrap(); + assert_eq!(grown.capacity, 8 * 1024 * 1024); + assert_eq!(window.rejected(), 0); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 16); + let media = disks.bytes.lock().unwrap(); + assert_eq!(media.len(), 16); + assert!(media + .values() + .all(|bytes| bytes.len() == 1024 * 1024 && bytes.iter().all(|byte| *byte == 0))); +} + #[tokio::test] async fn a_small_waiting_append_uses_a_smaller_growth_window() { let allocator = Arc::new(Allocator::new()); @@ -56,13 +86,13 @@ async fn a_small_waiting_append_uses_a_smaller_growth_window() { .await .unwrap() .unwrap(); - assert_eq!(grown.capacity, 8 * 1024 * 1024); + assert_eq!(grown.capacity, 4 * 1024 * 1024); assert_eq!(grown.acknowledged_cursor, end); - assert_eq!(disks.fsyncs.load(Ordering::Acquire), 18); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 10); } #[tokio::test] -async fn growth_amortizes_allocation_and_initializes_every_replica_within_the_hard_capacity() { +async fn growth_honors_requested_capacity_and_initializes_every_replica_within_the_hard_limit() { const MIB: u64 = 1024 * 1024; let allocator = Arc::new(Allocator::new()); let disks = Arc::new(Disks::default()); @@ -82,7 +112,7 @@ async fn growth_amortizes_allocation_and_initializes_every_replica_within_the_ha .unwrap(); assert_eq!(active.capacity, MIB); let grown = store - .grow_mirrored(name, 9, active.chunk_id, MIB + 1) + .grow_mirrored(name, 9, active.chunk_id, 8 * MIB) .await .unwrap() .unwrap(); @@ -107,7 +137,7 @@ async fn growth_amortizes_allocation_and_initializes_every_replica_within_the_ha assert_eq!(disks.fsyncs.load(Ordering::Acquire), 16); assert_eq!( store - .grow_mirrored(name, 9, active.chunk_id, 9 * MIB) + .grow_mirrored(name, 9, active.chunk_id, 10 * MIB) .await .unwrap() .unwrap() diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/local_progress.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/local_progress.rs index 2205e14f1..d24254758 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test/local_progress.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/local_progress.rs @@ -102,7 +102,7 @@ async fn grouped_progress_survives_checkpoint_query_renew_growth_and_mirror_repa store.renew_liveness(active.chunk_id, 9).await.unwrap(); let old_layout = allocator.chunk.lock().unwrap().clone(); let grown = store - .grow_mirrored(name, 9, active.chunk_id, active.capacity + 1) + .grow_mirrored(name, 9, active.chunk_id, 4 * 1024 * 1024) .await .unwrap() .unwrap(); diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/preparation.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/preparation.rs index 152b8b7ae..60f40d5c8 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test/preparation.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/preparation.rs @@ -4,6 +4,29 @@ use super::*; use tokio::time::{timeout, Duration}; +#[tokio::test] +async fn a_small_cold_stream_does_not_initialize_megabytes_of_unused_spares() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let stream = test_stream(allocator.clone(), disks.clone()).await; + stream.append(&[Bytes::from_static(b"retained")]).await.unwrap(); + stream.release_writer().await.unwrap(); + assert!(allocator.successors.lock().unwrap().is_empty()); + let initialized: usize = disks + .writes + .lock() + .unwrap() + .iter() + .filter(|(_, _, _, bytes)| bytes.len() == 1024 * 1024) + .map(|(_, _, _, bytes)| bytes.len()) + .sum(); + assert_eq!(initialized, 2 * 1024 * 1024); + assert_eq!( + stream.read_at(0, 8).await.unwrap(), + Bytes::from_static(b"retained") + ); +} + #[tokio::test] async fn preparation_revision_does_not_abort_or_reinitialize_mirror_repair() { let allocator = Arc::new(Allocator::new()); @@ -81,7 +104,8 @@ async fn temporary_allocation_failure_does_not_poison_later_capacity() { allocator.pause_growth.store(true, Ordering::Release); allocator.fail_growth_once.store(true, Ordering::Release); let stream = test_stream(allocator.clone(), Arc::new(Disks::default())).await; - stream.append(&[Bytes::from_static(b"retained")]).await.unwrap(); + let retained = Bytes::from(vec![0x51; 512 * 1024]); + stream.append(std::slice::from_ref(&retained)).await.unwrap(); timeout(Duration::from_secs(1), allocator.growth_started.notified()) .await .unwrap(); @@ -91,7 +115,8 @@ async fn temporary_allocation_failure_does_not_poison_later_capacity() { .await .unwrap() .unwrap(); - assert_eq!(range.begin, 8); + assert_eq!(range.begin, retained.len() as u64); + assert_eq!(stream.read_at(0, retained.len()).await.unwrap(), retained); assert_eq!(stream.read_at(range.begin, data.len()).await.unwrap(), data); stream.close().await.unwrap(); } @@ -103,8 +128,9 @@ async fn uncertain_unused_candidate_is_reclaimed_and_replaced_without_reinitiali // The primary chunk uses disks 1/2, 11/12, ...; its first candidate // has distinct disks 1001/1002, 1011/1012, ... . Fail candidate growth. disks.fail_fsync_once.store(1011, Ordering::Release); - let stream = test_stream(allocator.clone(), disks.clone()).await; - stream.append(&[Bytes::from_static(b"retained")]).await.unwrap(); + let stream = test_stream_with_capacity(allocator.clone(), disks.clone(), 2 * 1024 * 1024).await; + let retained = Bytes::from(vec![0x51; 1536 * 1024]); + stream.append(std::slice::from_ref(&retained)).await.unwrap(); timeout(Duration::from_secs(1), async { while allocator.successors.lock().unwrap().len() < 2 { tokio::task::yield_now().await; @@ -133,13 +159,18 @@ async fn uncertain_unused_candidate_is_reclaimed_and_replaced_without_reinitiali ); assert_eq!(stream.metrics().failed, 0); assert_eq!(stream.metrics().rollovers, 0); - assert_eq!( - stream.read_at(0, 8).await.unwrap(), - Bytes::from_static(b"retained") - ); + assert_eq!(stream.read_at(0, retained.len()).await.unwrap(), retained); } async fn test_stream(allocator: Arc, disks: Arc) -> ChunkStream { + test_stream_with_capacity(allocator, disks, 8 * 1024 * 1024).await +} + +async fn test_stream_with_capacity( + allocator: Arc, + disks: Arc, + capacity: u64, +) -> ChunkStream { let metadata = Arc::new(MemoryStreamStore::new(256 * 1024 * 1024)); let chunks = Arc::new( ProductionStreamChunkStore::new(allocator, disks, 30_000, ChunkReadPolicy::default()).unwrap(), @@ -156,7 +187,10 @@ async fn test_stream(allocator: Arc, disks: Arc) -> ChunkStrea created_at_ms: StreamRecord::creation_time_ms(), }, 9, - StreamConfig::default(), + StreamConfig { + chunk_capacity_bytes: capacity, + ..StreamConfig::default() + }, metadata.clone(), metadata, chunks, @@ -170,10 +204,10 @@ async fn allocation_in_flight_does_not_block_writes_in_ready_capacity() { let allocator = Arc::new(Allocator::new()); allocator.pause_growth.store(true, Ordering::Release); let stream = test_stream(allocator.clone(), Arc::new(Disks::default())).await; - let payload = Bytes::from(vec![0x51; 4096]); + let retained = Bytes::from(vec![0x51; 512 * 1024]); timeout( Duration::from_secs(1), - stream.append(std::slice::from_ref(&payload)), + stream.append(std::slice::from_ref(&retained)), ) .await .unwrap() @@ -181,6 +215,7 @@ async fn allocation_in_flight_does_not_block_writes_in_ready_capacity() { timeout(Duration::from_secs(1), allocator.growth_started.notified()) .await .unwrap(); + let payload = Bytes::from(vec![0x52; 4096]); let range = timeout( Duration::from_secs(1), stream.append(std::slice::from_ref(&payload)), @@ -188,7 +223,7 @@ async fn allocation_in_flight_does_not_block_writes_in_ready_capacity() { .await .unwrap() .unwrap(); - assert_eq!(range.begin, 4096); + assert_eq!(range.begin, retained.len() as u64); allocator.growth_resume.notify_one(); stream.close().await.unwrap(); assert_eq!(stream.read_at(range.begin, payload.len()).await.unwrap(), payload); @@ -200,7 +235,8 @@ async fn only_a_write_reaching_uninitialized_capacity_waits() { let disks = Arc::new(Disks::default()); disks.pause_initialization.store(true, Ordering::Release); let stream = test_stream(allocator.clone(), disks.clone()).await; - stream.append(&[Bytes::from(vec![0x31; 4096])]).await.unwrap(); + let retained = Bytes::from(vec![0x31; 512 * 1024]); + stream.append(std::slice::from_ref(&retained)).await.unwrap(); timeout(Duration::from_secs(1), disks.initialization_started.notified()) .await .unwrap(); @@ -216,14 +252,14 @@ async fn only_a_write_reaching_uninitialized_capacity_waits() { let data = large.clone(); let mut pending = tokio::spawn(async move { writer.append(&[data]).await }); assert!(timeout(Duration::from_millis(20), &mut pending).await.is_err()); - assert_eq!(stream.tail(), 8192); + assert_eq!(stream.tail(), retained.len() as u64 + 4096); disks.initialization_resume.notify_one(); let range = timeout(Duration::from_secs(1), pending) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(range.begin, 8192); + assert_eq!(range.begin, retained.len() as u64 + 4096); assert_eq!(stream.read_at(range.begin, large.len()).await.unwrap(), large); stream.close().await.unwrap(); } @@ -234,7 +270,10 @@ async fn close_drains_preparation_before_sealing() { let disks = Arc::new(Disks::default()); disks.pause_initialization.store(true, Ordering::Release); let stream = test_stream(allocator.clone(), disks.clone()).await; - stream.append(&[Bytes::from_static(b"retained")]).await.unwrap(); + stream + .append(&[Bytes::from(vec![0x51; 512 * 1024])]) + .await + .unwrap(); timeout(Duration::from_secs(1), disks.initialization_started.notified()) .await .unwrap(); diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery_drain.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery_drain.rs index 8110b00fd..e76a23264 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery_drain.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery_drain.rs @@ -3,6 +3,51 @@ use super::*; +#[tokio::test(start_paused = true)] +async fn recovery_drains_a_large_layout_with_bounded_parallelism_across_strips() { + use std::time::Duration; + + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let make = || { + ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap() + }; + let writer = make(); + let name = StreamName::generate(); + let active = writer + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Wal) + .await + .unwrap(); + writer + .grow_mirrored(name, 9, active.chunk_id, 32 * 1024 * 1024) + .await + .unwrap(); + let mut encoded = TestGroupedStrip::default(); + let end = encoded + .append(&writer, name, 9, active.chunk_id, b"retained") + .await; + drop(writer); + let cold = make(); + let memory = MemoryStreamStore::new(1024); + cold.bind_writer(name, 10, memory.claim_writer(name, 10).await.unwrap()); + disks.drain_delay_ms.store(20, Ordering::Release); + let recovered = tokio::time::timeout(Duration::from_millis(400), cold.recover_tail(name, 10, &active)) + .await + .expect("all allocated strips must drain within the original recovery budget") + .unwrap(); + assert_eq!((recovered.physical_end, recovered.logical_end), (end, 8)); + assert_eq!(recovered.extents.len(), 1); + assert_eq!(disks.completed_drains.load(Ordering::Acquire), 64); + assert!((4..=8).contains(&disks.max_active_drains.load(Ordering::Acquire))); + assert_eq!(disks.active_drains.load(Ordering::Acquire), 0); +} + #[tokio::test] async fn a_failed_recovery_drain_requests_layout_repair_before_tail_observation() { for lost_reply in [false, true] { diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/unwritten.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/unwritten.rs index 24a2b3127..1fd1584ac 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test/unwritten.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/unwritten.rs @@ -180,7 +180,10 @@ async fn cross_strip_tail_header_survives_later_growth_and_the_next_append() { created_at_ms: StreamRecord::creation_time_ms(), }, 9, - StreamConfig::default(), + StreamConfig { + chunk_capacity_bytes: 2 * 1024 * 1024, + ..StreamConfig::default() + }, metadata.clone(), metadata.clone(), chunks.clone(), @@ -192,11 +195,11 @@ async fn cross_strip_tail_header_survives_later_growth_and_the_next_append() { let chunk_id = range.chunk_id.unwrap(); let complete_end = 1024 * 1024 - 7; assert_eq!(chunks.durable_cursor(chunk_id, 9).await.unwrap().offset, 0); - assert_eq!(chunks.current_capacity(chunk_id).await.unwrap(), 8 * 1024 * 1024); + assert_eq!(chunks.current_capacity(chunk_id).await.unwrap(), 2 * 1024 * 1024); assert_eq!(observed_end(&allocator, &disks, name).await, complete_end); let publications = metadata.metadata_publish_count(); chunks - .grow_mirrored(name, 9, chunk_id, 9 * 1024 * 1024) + .grow_mirrored(name, 9, chunk_id, 3 * 1024 * 1024) .await .unwrap() .unwrap(); @@ -281,7 +284,7 @@ async fn uncertain_growth_initialization_cannot_be_retried_or_written() { let end = strip.append(&store, name, 9, active.chunk_id, b"retained").await; disks.fail_fsync_once.store(11, Ordering::Release); assert!(store - .grow_mirrored(name, 9, active.chunk_id, active.capacity + 1) + .grow_mirrored(name, 9, active.chunk_id, 4 * 1024 * 1024) .await .is_err()); // All three added strips complete their submitted work, including the healthy @@ -289,7 +292,7 @@ async fn uncertain_growth_initialization_cannot_be_retried_or_written() { assert_eq!(disks.fsyncs.load(Ordering::Acquire), 9); let submissions = disks.writes.lock().unwrap().len(); assert!(store - .grow_mirrored(name, 9, active.chunk_id, active.capacity + 1) + .grow_mirrored(name, 9, active.chunk_id, 4 * 1024 * 1024) .await .is_err()); assert_eq!(disks.writes.lock().unwrap().len(), submissions); diff --git a/lib/crowdb-diskio-client/src/semantic/observation.rs b/lib/crowdb-diskio-client/src/semantic/observation.rs index 8e2fe667a..90ee1540c 100644 --- a/lib/crowdb-diskio-client/src/semantic/observation.rs +++ b/lib/crowdb-diskio-client/src/semantic/observation.rs @@ -24,6 +24,31 @@ pub struct DiskPlacement { } impl DiskTopologyView { + /// Build a metadata-only observation without opening transport connections. + #[cfg(feature = "test-util")] + #[must_use] + pub fn from_routes_for_tests(routes: Vec) -> Self { + let mut snapshot = RouteSnapshot::empty(); + snapshot.generation = 1; + for route in routes { + let endpoint: Arc = Arc::from(route.endpoint); + snapshot.routes.insert( + route.disk_id, + crate::topology::DiskRoute { + rack_id: route.rack_id, + node_id: route.node_id, + disk_group_id: route.disk_group_id, + instance_id: route.instance_id, + pool_key: Arc::from(format!("{}@{}", route.instance_id, endpoint)), + endpoint, + }, + ); + } + Self { + snapshot: Arc::new(snapshot), + } + } + /// Process-local route-cache generation, independent of catalog epochs. #[must_use] pub fn generation(&self) -> u64 { diff --git a/lib/crowdb-kv/Cargo.toml b/lib/crowdb-kv/Cargo.toml index 77550aadc..c76751882 100644 --- a/lib/crowdb-kv/Cargo.toml +++ b/lib/crowdb-kv/Cargo.toml @@ -106,6 +106,14 @@ path = "tests/store_test.rs" name = "wal_test" path = "tests/wal_test.rs" +[[test]] +name = "replay_memory_test" +path = "tests/replay_memory_test.rs" + +[[test]] +name = "replay_compaction_memory_test" +path = "tests/replay_compaction_memory_test.rs" + [[bench]] name = "wal" harness = false diff --git a/lib/crowdb-kv/src/wal/gc.rs b/lib/crowdb-kv/src/wal/gc.rs index bdd81a167..74ae7d980 100644 --- a/lib/crowdb-kv/src/wal/gc.rs +++ b/lib/crowdb-kv/src/wal/gc.rs @@ -96,12 +96,22 @@ pub async fn run_gc_with_watermark(wal: &WalEngine, gc_slot: u64) -> io::Result< let mut unlinked = 0usize; let disk_paths = wal.disk_group_paths(); - // Collect segments eligible for GC. - let eligible: Vec<(u64, usize, std::path::PathBuf)> = wal - .index() - .segments_snapshot() + // Slot progress cannot replace election durability. Keep one segment proving + // the highest term and one carrying the latest vote, including mixed segments. + let segments = wal.index().segments_snapshot(); + let term_segment = segments + .iter() + .max_by_key(|meta| (meta.max_term, meta.segment_id)) + .map(|meta| meta.segment_id); + let vote_segment = segments + .iter() + .filter_map(|meta| meta.vote_term.map(|term| (term, meta.segment_id))) + .max() + .map(|(_, id)| id); + let eligible: Vec<(u64, usize, std::path::PathBuf)> = segments .iter() .filter(|meta| meta.max_slot > 0 && meta.max_slot < gc_slot) + .filter(|meta| Some(meta.segment_id) != term_segment && Some(meta.segment_id) != vote_segment) .map(|meta| { let dir = &disk_paths[meta.disk_idx]; let filename = format!("seg-{:07}.ck", meta.segment_id); diff --git a/lib/crowdb-kv/src/wal/index.rs b/lib/crowdb-kv/src/wal/index.rs index d3d974aff..8a4ff7224 100644 --- a/lib/crowdb-kv/src/wal/index.rs +++ b/lib/crowdb-kv/src/wal/index.rs @@ -38,6 +38,10 @@ pub struct SegmentMeta { pub min_slot: SlotIndex, pub max_slot: SlotIndex, pub record_count: u32, + /// Highest election term observed in any verified record in this segment. + pub max_term: u64, + /// Highest term of a vote record; its segment must survive election-state GC. + pub vote_term: Option, } impl SegmentIndex { @@ -113,6 +117,37 @@ pub struct ShardedSegmentIndex { } impl ShardedSegmentIndex { + /// Seed the existing pipeline shards before any writer can publish new records. + /// + /// # Errors + /// Returns an error when recovered locations refer to an unavailable disk. + pub(super) fn from_replay( + pipeline_count: usize, + group_id: u64, + recovered: SegmentIndex, + ) -> std::io::Result { + let mut index = Self::new(pipeline_count, group_id); + for (slot, location) in recovered.map { + let shard = index.shards.get_mut(location.disk_idx).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "replayed WAL disk is unavailable", + ) + })?; + shard.get_mut().insert(slot, location); + } + for meta in recovered.segments.into_values() { + let shard = index.shards.get_mut(meta.disk_idx).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "replayed WAL disk is unavailable", + ) + })?; + shard.get_mut().register_segment(meta); + } + Ok(index) + } + /// Create one independent shard per WAL pipeline. /// /// # Panics diff --git a/lib/crowdb-kv/src/wal/pipeline_writer.rs b/lib/crowdb-kv/src/wal/pipeline_writer.rs index 3db5e102c..defa218cb 100644 --- a/lib/crowdb-kv/src/wal/pipeline_writer.rs +++ b/lib/crowdb-kv/src/wal/pipeline_writer.rs @@ -75,6 +75,8 @@ pub(crate) struct PendingWrite { pub encoded: EncodedRecord, /// Slot index for index update (0 = metadata record, not indexed). pub slot: SlotIndex, + pub term: u64, + pub is_vote: bool, /// Ack channel: resolves with the record's `SlotLocation` when the /// covering flush succeeds. pub ack: oneshot::Sender>, @@ -640,6 +642,10 @@ async fn write_batch( } } segment.record_count += 1; + segment.max_term = segment.max_term.max(req.term); + if req.is_vote { + segment.vote_term = Some(segment.vote_term.map_or(req.term, |term| term.max(req.term))); + } } // Single durable flush for the whole batch — skipped when skip_fsync is @@ -689,6 +695,8 @@ fn register_sealed(index: &ShardedSegmentIndex, segment: &WalSegment, pipeline_i min_slot: segment.min_slot, max_slot: segment.max_slot, record_count: segment.record_count, + max_term: segment.max_term, + vote_term: segment.vote_term, }; index.register_segment(pipeline_idx, meta); debug!( diff --git a/lib/crowdb-kv/src/wal/record.rs b/lib/crowdb-kv/src/wal/record.rs index 1733296f5..d6e75fa7d 100644 --- a/lib/crowdb-kv/src/wal/record.rs +++ b/lib/crowdb-kv/src/wal/record.rs @@ -253,6 +253,9 @@ impl WALRecord { return Err(RecordError::Truncated); } let frame_len = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize; + if frame_len < HEADER_BODY_LEN + 4 { + return Err(RecordError::Truncated); + } let total = 4 + frame_len; if data.len() < total { return Err(RecordError::Truncated); diff --git a/lib/crowdb-kv/src/wal/replay.rs b/lib/crowdb-kv/src/wal/replay.rs index 53b00bbc1..30f0b52ef 100644 --- a/lib/crowdb-kv/src/wal/replay.rs +++ b/lib/crowdb-kv/src/wal/replay.rs @@ -20,10 +20,12 @@ use super::record::{RecordType, WALRecord}; use super::segment::{parse_segment_filename, SegmentReader}; use super::{IoBackend, OpenOptions}; +mod records; + /// Result of replaying all WAL segments for a group. #[derive(Debug)] pub struct ReplayResult { - /// Verified records in slot order (deduplicated: highest ballot per slot wins). + /// Vote records followed by slot-ordered highest accepted and promised ballots. pub records: Vec, /// Rebuilt segment index. pub index: SegmentIndex, @@ -87,7 +89,7 @@ pub async fn replay_group( ); // ── Step 2–3: scan records, verify CRC, truncate on error ── - let mut verified_records: Vec = Vec::new(); + let mut verified_records = records::ReplayRecords::default(); let mut index = SegmentIndex::new(); let mut max_segment_id: u64 = 0; let total_segments = all_segments.len(); @@ -119,6 +121,8 @@ pub async fn replay_group( let mut seg_min_slot = u64::MAX; let mut seg_max_slot = 0u64; let mut seg_record_count = 0u32; + let mut max_term = 0; + let mut vote_term = None::; loop { match reader.next_record().await { @@ -133,6 +137,10 @@ pub async fn replay_group( } } seg_record_count += 1; + max_term = max_term.max(record.term); + if record.record_type == RecordType::VoteGranted { + vote_term = Some(vote_term.map_or(record.term, |term| term.max(record.term))); + } verified_records.push(record); } Ok(None) => break, // EOF or footer @@ -170,6 +178,8 @@ pub async fn replay_group( min_slot: if seg_min_slot == u64::MAX { 0 } else { seg_min_slot }, max_slot: seg_max_slot, record_count: seg_record_count, + max_term, + vote_term, }; for (slot, file_offset) in &seg_records { index.insert( @@ -186,25 +196,16 @@ pub async fn replay_group( debug!( group_id, - total_records = verified_records.len(), + total_records = verified_records.scanned, max_segment_id, "replay: scan complete" ); - // ── Step 4–7: rebuild in-memory state ────────────── - - // 4. current_term = max term across all records. - let current_term = verified_records.iter().map(|r| r.term).max().unwrap_or(0); - - // 5. voted_for from latest VoteGranted for current_term. - let voted_for = verified_records - .iter() - .rev() - .find(|r| r.record_type == RecordType::VoteGranted && r.term == current_term) - .and_then(WALRecord::voted_for_id); + let current_term = verified_records.current_term; + let voted_for = verified_records.voted_for; Ok(ReplayResult { - records: verified_records, + records: verified_records.into_records(), index, max_segment_id, current_term, diff --git a/lib/crowdb-kv/src/wal/replay/records.rs b/lib/crowdb-kv/src/wal/replay/records.rs new file mode 100644 index 000000000..43e6ef3dc --- /dev/null +++ b/lib/crowdb-kv/src/wal/replay/records.rs @@ -0,0 +1,53 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::collections::BTreeMap; + +use crate::paxos::{PxNodeId, PxTerm}; +use crate::wal::record::{RecordType, WALRecord}; + +#[derive(Default)] +pub(super) struct ReplayRecords { + slots: BTreeMap<(u64, u8), WALRecord>, + votes: Vec, + pub(super) current_term: PxTerm, + pub(super) voted_for: Option, + pub(super) scanned: u64, +} + +impl ReplayRecords { + pub(super) fn push(&mut self, record: WALRecord) { + self.scanned += 1; + // Election safety depends on every verified record, including accepts + // superseded by a higher ballot and records physically ordered by disk. + if record.term > self.current_term { + self.current_term = record.term; + self.voted_for = None; + } + if record.record_type == RecordType::VoteGranted { + if record.term == self.current_term { + self.voted_for = record.voted_for_id(); + } + self.votes.push(record); + return; + } + // A later promise must never erase an older accepted value. Retain the + // highest accepted ballot and the highest promised ballot independently. + let key = (record.slot, record.record_type as u8); + match self.slots.entry(key) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(record); + } + std::collections::btree_map::Entry::Occupied(mut entry) => { + if record.ballot >= entry.get().ballot { + entry.insert(record); + } + } + } + } + + pub(super) fn into_records(mut self) -> Vec { + self.votes.extend(self.slots.into_values()); + self.votes + } +} diff --git a/lib/crowdb-kv/src/wal/segment.rs b/lib/crowdb-kv/src/wal/segment.rs index 55bc4bc1d..18675afcb 100644 --- a/lib/crowdb-kv/src/wal/segment.rs +++ b/lib/crowdb-kv/src/wal/segment.rs @@ -39,6 +39,8 @@ use crate::paxos::PxGroupId; use super::record::{parse_hex_u32, RecordError, WALRecord, WalRecordFormat}; use super::{IoBackend, OpenOptions, WalFile}; +mod read_buffer; + const SEG_MAGIC: u32 = 0x5345_474D; const SEG_VERSION: u16 = 1; pub const SEG_HEADER_LEN: usize = 4 + 2 + 8 + 8; // 22 @@ -68,6 +70,8 @@ pub struct WalSegment { pub min_slot: SlotIndex, pub max_slot: SlotIndex, pub record_count: u32, + pub(super) max_term: u64, + pub(super) vote_term: Option, sealed: bool, record_format: WalRecordFormat, } @@ -126,6 +130,8 @@ impl WalSegment { min_slot: u64::MAX, max_slot: 0, record_count: 0, + max_term: 0, + vote_term: None, sealed: false, record_format, }) @@ -159,6 +165,10 @@ impl WalSegment { } } self.record_count += 1; + self.max_term = self.max_term.max(record.term); + if record.record_type == super::record::RecordType::VoteGranted { + self.vote_term = Some(self.vote_term.map_or(record.term, |term| term.max(record.term))); + } Ok(offset) } @@ -299,6 +309,7 @@ pub struct SegmentReader { offset: u64, /// File size (to detect truncation). file_len: u64, + buffer: read_buffer::SegmentReadBuffer, } impl SegmentReader { @@ -347,6 +358,7 @@ impl SegmentReader { header, offset: data_offset, file_len, + buffer: read_buffer::SegmentReadBuffer::default(), }) } else { // Binary-format header. @@ -359,6 +371,7 @@ impl SegmentReader { header, offset: SEG_HEADER_LEN as u64, file_len, + buffer: read_buffer::SegmentReadBuffer::default(), }) } } @@ -409,12 +422,12 @@ impl SegmentReader { // Peek the next frame marker (the `frame_len` of a record, the // `FOOTER_MAGIC` of a seal footer, or `0` for trailing padding). - let mut frame_hdr = [0u8; 4]; - self.file - .read_exact_at(&mut frame_hdr, self.offset) + let frame_hdr = self + .buffer + .read(&mut self.file, self.file_len, self.offset, 4) .await .map_err(|_| (RecordError::Truncated, self.offset))?; - let marker = u32::from_le_bytes(frame_hdr); + let marker = u32::from_le_bytes(frame_hdr.as_ref().try_into().expect("four-byte frame prefix")); // Footer reached → records are done (sealed segment). if marker == FOOTER_MAGIC { @@ -436,9 +449,14 @@ impl SegmentReader { return Err((RecordError::Truncated, self.offset)); } - let mut buf = vec![0u8; usize::try_from(total).expect("record size exceeds usize")]; - self.file - .read_exact_at(&mut buf, self.offset) + let buf = self + .buffer + .read( + &mut self.file, + self.file_len, + self.offset, + usize::try_from(total).expect("record size exceeds usize"), + ) .await .map_err(|_| (RecordError::Truncated, self.offset))?; @@ -456,12 +474,12 @@ impl SegmentReader { if self.file_len - self.offset < prefix.len() as u64 { return Ok(false); } - let mut buf = vec![0u8; prefix.len()]; - self.file - .read_exact_at(&mut buf, self.offset) + let buf = self + .buffer + .read(&mut self.file, self.file_len, self.offset, prefix.len()) .await .map_err(|_| (RecordError::Truncated, self.offset))?; - Ok(buf == prefix) + Ok(buf.as_ref() == prefix) } async fn next_is_text_footer(&mut self) -> Result { @@ -469,12 +487,12 @@ impl SegmentReader { if self.file_len - self.offset < prefix.len() as u64 { return Ok(false); } - let mut buf = vec![0u8; prefix.len()]; - self.file - .read_exact_at(&mut buf, self.offset) + let buf = self + .buffer + .read(&mut self.file, self.file_len, self.offset, prefix.len()) .await .map_err(|_| (RecordError::Truncated, self.offset))?; - Ok(buf == prefix) + Ok(buf.as_ref() == prefix) } async fn next_text_record( diff --git a/lib/crowdb-kv/src/wal/segment/read_buffer.rs b/lib/crowdb-kv/src/wal/segment/read_buffer.rs new file mode 100644 index 000000000..c8fe2d040 --- /dev/null +++ b/lib/crowdb-kv/src/wal/segment/read_buffer.rs @@ -0,0 +1,50 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::io; + +use bytes::Bytes; + +use crate::wal::WalFile; + +const READ_AHEAD: usize = 1024 * 1024; + +#[derive(Default)] +pub(super) struct SegmentReadBuffer { + bytes: Bytes, + offset: u64, +} + +impl SegmentReadBuffer { + pub(super) async fn read( + &mut self, + file: &mut WalFile, + file_len: u64, + offset: u64, + length: usize, + ) -> io::Result { + if let Some(start) = offset + .checked_sub(self.offset) + .and_then(|start| usize::try_from(start).ok()) + { + if start <= self.bytes.len() && length <= self.bytes.len() - start { + return Ok(self.bytes.slice(start..start + length)); + } + } + let available = file_len + .checked_sub(offset) + .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "WAL read starts past EOF"))?; + if u64::try_from(length).unwrap_or(u64::MAX) > available { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "WAL read extends past EOF", + )); + } + let read_ahead = usize::try_from(available.min(READ_AHEAD as u64)).expect("bounded read size"); + let mut bytes = vec![0; length.max(read_ahead)]; + file.read_exact_at(&mut bytes, offset).await?; + self.bytes = Bytes::from(bytes); + self.offset = offset; + Ok(self.bytes.slice(..length)) + } +} diff --git a/lib/crowdb-kv/src/wal/wal_engine.rs b/lib/crowdb-kv/src/wal/wal_engine.rs index b04151a43..c44a4b4b5 100644 --- a/lib/crowdb-kv/src/wal/wal_engine.rs +++ b/lib/crowdb-kv/src/wal/wal_engine.rs @@ -21,7 +21,7 @@ use crate::metrics::{Bandwidth, Counter, LatencySummary}; use crate::paxos::roles::SlotIndex; use crate::paxos::PxGroupId; -use super::index::{ShardedSegmentIndex, SlotLocation}; +use super::index::{SegmentIndex, ShardedSegmentIndex, SlotLocation}; use super::pipeline::WalPipeline; use super::pipeline_backend::{WalBlockAlignment, WalPipelineBackend}; use super::pipeline_writer::{spawn_pipeline_writer, EncodedRecord, PendingWrite, WriterCommand}; @@ -153,10 +153,48 @@ impl WalEngine { config: WalConfig, group_id: PxGroupId, initial_next_segment_id: u64, + ) -> io::Result> { + Self::create_with_index( + backend, + config, + group_id, + initial_next_segment_id, + SegmentIndex::new(), + ) + .await + } + + /// Resume a WAL with the verified index and next segment id recovered locally. + /// + /// # Errors + /// Returns an error if recovered disks are unavailable or writers cannot start. + pub async fn create_from_replay( + backend: Arc, + config: WalConfig, + group_id: PxGroupId, + max_segment_id: u64, + index: SegmentIndex, + ) -> io::Result> { + let next_segment_id = max_segment_id + .checked_add(1) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "WAL segment id exhausted"))?; + Self::create_with_index(backend, config, group_id, next_segment_id.max(1), index).await + } + + async fn create_with_index( + backend: Arc, + config: WalConfig, + group_id: PxGroupId, + initial_next_segment_id: u64, + recovered_index: SegmentIndex, ) -> io::Result> { let pipeline_count = config.wal_disks.len(); let failed = Arc::new(AtomicBool::new(false)); - let index = Arc::new(ShardedSegmentIndex::new(pipeline_count, group_id)); + let index = Arc::new(ShardedSegmentIndex::from_replay( + pipeline_count, + group_id, + recovered_index, + )?); let next_segment_id = Arc::new(AtomicU64::new(initial_next_segment_id)); let flush_count = Arc::new(AtomicU64::new(0)); let records_flushed = Arc::new(AtomicU64::new(0)); @@ -282,6 +320,8 @@ impl WalEngine { let pending = PendingWrite { encoded, slot: record.slot, + term: record.term, + is_vote: record.record_type == super::record::RecordType::VoteGranted, ack: ack_tx, }; diff --git a/lib/crowdb-kv/tests/replay_compaction_memory_test.rs b/lib/crowdb-kv/tests/replay_compaction_memory_test.rs new file mode 100644 index 000000000..55caf9424 --- /dev/null +++ b/lib/crowdb-kv/tests/replay_compaction_memory_test.rs @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#![cfg(target_os = "linux")] + +use std::io::Write; +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_kv::wal::record::WALRecord; +use crowdb_kv::wal::replay::replay_group; +use crowdb_kv::wal::IoBackend; + +fn peak_rss_kib() -> u64 { + std::fs::read_to_string("/proc/self/status") + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("VmHWM:")) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap() +} + +#[tokio::test] +async fn replay_releases_superseded_accepts_while_scanning_half_a_gibibyte() { + let temp = crowdb_test_harness::test_dirs::tempdir_in_test_data("wal-compaction-memory"); + let directory = temp.path().join("group1"); + std::fs::create_dir(&directory).unwrap(); + let mut file = std::io::BufWriter::new(std::fs::File::create(directory.join("seg-0000001.ck")).unwrap()); + for bytes in [ + &0x5345_474D_u32.to_le_bytes()[..], + &1_u16.to_le_bytes(), + &1_u64.to_le_bytes(), + &1_u64.to_le_bytes(), + ] { + file.write_all(bytes).unwrap(); + } + for sequence in 1_u64..=8192 { + let round = (sequence - 1) / 32 + 1; + let entry = PxLogEntry { + slot: (sequence - 1) % 32 + 1, + ballot: PxBallot::new(round, 7), + term: round, + payload: Bytes::from(vec![u8::try_from(sequence % 251).unwrap(); 65536]), + }; + file.write_all(&WALRecord::from_accepted(1, &entry).encode()) + .unwrap(); + } + file.flush().unwrap(); + drop(file); + let before = peak_rss_kib(); + let result = replay_group( + &Arc::new(IoBackend::block_device()), + &[temp.path().to_path_buf()], + 1, + ) + .await + .unwrap(); + let growth = peak_rss_kib().saturating_sub(before); + println!("Compacted replay peak growth: {growth} KiB for 512 MiB of accepted payloads"); + assert!( + growth < 64 * 1024, + "superseded accepts remained resident during replay: {growth} KiB" + ); + assert_eq!(result.records.len(), 32); + assert_eq!(result.current_term, 256); + assert_eq!(result.index.slot_count(), 32); + assert_eq!(result.index.segments().next().unwrap().record_count, 8192); + for (index, record) in result.records.iter().enumerate() { + let slot = u64::try_from(index).unwrap() + 1; + assert_eq!(record.slot, slot); + assert_eq!(record.ballot, PxBallot::new(256, 7)); + assert_eq!( + record.payload, + vec![u8::try_from((8160 + slot) % 251).unwrap(); 65536] + ); + } +} diff --git a/lib/crowdb-kv/tests/replay_memory_test.rs b/lib/crowdb-kv/tests/replay_memory_test.rs new file mode 100644 index 000000000..010ccadde --- /dev/null +++ b/lib/crowdb-kv/tests/replay_memory_test.rs @@ -0,0 +1,85 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#![cfg(target_os = "linux")] + +use std::io::Write; +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_kv::wal::record::WALRecord; +use crowdb_kv::wal::replay::replay_group; +use crowdb_kv::wal::IoBackend; + +fn peak_rss_kib() -> u64 { + std::fs::read_to_string("/proc/self/status") + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("VmHWM:")) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .parse() + .unwrap() +} + +#[tokio::test] +async fn binary_replay_keeps_small_record_history_within_one_gibibyte() { + const COUNT: u64 = 800_000; + const PAYLOAD: usize = 768; + let temp = crowdb_test_harness::test_dirs::tempdir_in_test_data("wal-small-record-memory"); + let directory = temp.path().join("group1"); + std::fs::create_dir(&directory).unwrap(); + let mut file = std::io::BufWriter::new(std::fs::File::create(directory.join("seg-0000001.ck")).unwrap()); + file.write_all(&0x5345_474D_u32.to_le_bytes()).unwrap(); + file.write_all(&1_u16.to_le_bytes()).unwrap(); + file.write_all(&1_u64.to_le_bytes()).unwrap(); + file.write_all(&1_u64.to_le_bytes()).unwrap(); + for slot in 1..=COUNT { + let entry = PxLogEntry { + slot, + ballot: PxBallot::new(7, 1), + term: 7, + payload: Bytes::from(vec![u8::try_from(slot % 251).unwrap(); PAYLOAD]), + }; + file.write_all(&WALRecord::from_accepted(1, &entry).encode()) + .unwrap(); + } + file.flush().unwrap(); + drop(file); + let before = peak_rss_kib(); + let started = std::time::Instant::now(); + let result = replay_group( + &Arc::new(IoBackend::block_device()), + &[temp.path().to_path_buf()], + 1, + ) + .await + .unwrap(); + let growth = peak_rss_kib().saturating_sub(before); + println!( + "Replay {} records / {} payload bytes: peak growth {} KiB; elapsed {:?}", + COUNT, + COUNT * PAYLOAD as u64, + growth, + started.elapsed() + ); + assert_eq!(result.records.len(), usize::try_from(COUNT).unwrap()); + assert_eq!(result.index.slot_count(), usize::try_from(COUNT).unwrap()); + assert_eq!(result.current_term, 7); + for (index, record) in result.records.iter().enumerate() { + let slot = u64::try_from(index).unwrap() + 1; + assert_eq!(record.slot, slot); + assert_eq!(record.payload.len(), PAYLOAD); + assert!(record + .payload + .iter() + .all(|byte| *byte == u8::try_from(slot % 251).unwrap())); + } + assert!( + growth < 1024 * 1024, + "small-record WAL replay exceeded its 1 GiB memory budget: {growth} KiB" + ); +} diff --git a/lib/crowdb-kv/tests/wal_test.rs b/lib/crowdb-kv/tests/wal_test.rs index 6984e85fe..802100e54 100644 --- a/lib/crowdb-kv/tests/wal_test.rs +++ b/lib/crowdb-kv/tests/wal_test.rs @@ -18,6 +18,9 @@ mod wal_engine_test; #[path = "wal_test/replay_test.rs"] mod replay_test; +#[path = "wal_test/replay_compaction_test.rs"] +mod replay_compaction_test; + #[path = "wal_test/gc_test.rs"] mod gc_test; diff --git a/lib/crowdb-kv/tests/wal_test/gc_test.rs b/lib/crowdb-kv/tests/wal_test/gc_test.rs index c44766dc3..ed7b2079c 100644 --- a/lib/crowdb-kv/tests/wal_test/gc_test.rs +++ b/lib/crowdb-kv/tests/wal_test/gc_test.rs @@ -17,6 +17,125 @@ fn sim_backend() -> Arc { Arc::new(IoBackend::MemBlock(MemBlockDevice::new())) } +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn gc_preserves_election_state_in_a_mixed_old_segment() { + let backend = sim_backend(); + let disks = vec![PathBuf::from("/mixed-vote")]; + let config = WalConfig { + wal_disks: disks.clone(), + wal_segment_size: 256, + ..Default::default() + }; + let wal = WalEngine::create(backend.clone(), config, 1).await.unwrap(); + wal.append(&WALRecord::from_vote_granted(1, 9, 42)).await.unwrap(); + for slot in 1..=20 { + let entry = PxLogEntry { + slot, + ballot: PxBallot::new(9, 42), + term: 9, + payload: Bytes::from_static(b"data"), + }; + wal.append(&WALRecord::from_accepted(1, &entry)).await.unwrap(); + } + wal.seal_all().await.unwrap(); + assert!(run_gc_with_watermark(&wal, 100).await.unwrap() > 0); + let replay = replay_group(&backend, &disks, 1).await.unwrap(); + assert_eq!(replay.current_term, 9); + assert_eq!( + replay.voted_for, + Some(42), + "GC must not permit a second vote after restart" + ); +} + +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn gc_keeps_a_high_term_without_a_vote_in_an_old_segment() { + let backend = sim_backend(); + let disks = vec![PathBuf::from("/mixed-term")]; + let config = WalConfig { + wal_disks: disks.clone(), + wal_segment_size: 256, + ..Default::default() + }; + let wal = WalEngine::create(backend.clone(), config, 1).await.unwrap(); + wal.append(&WALRecord::from_promised(1, 12, 1, PxBallot::new(12, 42))) + .await + .unwrap(); + // Older in-flight accepts can reach a different disk pipeline after a term change. + for slot in 2..=20 { + wal.append(&WALRecord::from_promised(1, 9, slot, PxBallot::new(9, 42))) + .await + .unwrap(); + } + wal.seal_all().await.unwrap(); + assert!(run_gc_with_watermark(&wal, 100).await.unwrap() > 0); + let replay = replay_group(&backend, &disks, 1).await.unwrap(); + assert_eq!(replay.current_term, 12); + assert_eq!(replay.voted_for, None); +} + +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn multi_disk_restart_preserves_index_and_election_state_across_gc() { + let backend = sim_backend(); + let disks = vec![PathBuf::from("/restart-a"), PathBuf::from("/restart-b")]; + let config = WalConfig { + wal_disks: disks.clone(), + wal_segment_size: 256, + ..Default::default() + }; + let mut wal = WalEngine::create(backend.clone(), config.clone(), 1) + .await + .unwrap(); + wal.append(&WALRecord::from_vote_granted(1, 9, 42)).await.unwrap(); + for round in 0..3 { + for slot in round * 24 + 1..=(round + 1) * 24 { + wal.append(&WALRecord::from_accepted( + 1, + &PxLogEntry { + slot, + ballot: PxBallot::new(9, 42), + term: 9, + payload: Bytes::copy_from_slice(&slot.to_le_bytes()), + }, + )) + .await + .unwrap(); + } + wal.seal_all().await.unwrap(); + let mut replay = replay_group(&backend, &disks, 1).await.unwrap(); + let prior_index = replay.index.clone(); + drop(wal); + wal = WalEngine::create_from_replay( + backend.clone(), + config.clone(), + 1, + replay.max_segment_id, + std::mem::take(&mut replay.index), + ) + .await + .unwrap(); + assert_eq!(wal.index().slot_count(), prior_index.slot_count()); + for record in &replay.records { + if record.slot != 0 { + assert_eq!( + wal.index().locate(record.slot).as_ref(), + prior_index.locate(record.slot) + ); + } + } + assert!(run_gc_with_watermark(&wal, (round + 1) * 24).await.unwrap() > 0); + let recovered = replay_group(&backend, &disks, 1).await.unwrap(); + assert_eq!(recovered.current_term, 9); + assert_eq!(recovered.voted_for, Some(42)); + let tip = (round + 1) * 24; + assert!(recovered + .records + .iter() + .any(|record| record.slot == tip && record.payload.as_ref() == tip.to_le_bytes())); + assert!(recovered.records.len() < replay.records.len()); + } +} + #[tokio::test(flavor = "current_thread", start_paused = true)] async fn gc_removes_segments_below_watermark() { let backend = sim_backend(); diff --git a/lib/crowdb-kv/tests/wal_test/index_test.rs b/lib/crowdb-kv/tests/wal_test/index_test.rs index b72365906..0b23c488d 100644 --- a/lib/crowdb-kv/tests/wal_test/index_test.rs +++ b/lib/crowdb-kv/tests/wal_test/index_test.rs @@ -25,6 +25,8 @@ fn meta(seg: u64, min: u64, max: u64) -> SegmentMeta { min_slot: min, max_slot: max, record_count: (max - min + 1) as u32, + max_term: 0, + vote_term: None, } } diff --git a/lib/crowdb-kv/tests/wal_test/record_test.rs b/lib/crowdb-kv/tests/wal_test/record_test.rs index 9bdd1ab3e..634c855c5 100644 --- a/lib/crowdb-kv/tests/wal_test/record_test.rs +++ b/lib/crowdb-kv/tests/wal_test/record_test.rs @@ -7,6 +7,15 @@ use bytes::Bytes; use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; use crowdb_kv::wal::record::{RecordError, RecordType, WALRecord, MIN_RECORD_SIZE}; +#[test] +fn short_frame_marker_is_rejected() { + for frame_len in 0_u32..55 { + let mut frame = vec![0; 128]; + frame[..4].copy_from_slice(&frame_len.to_le_bytes()); + assert!(matches!(WALRecord::decode(&frame), Err(RecordError::Truncated))); + } +} + #[test] fn encode_decode_roundtrip_accepted() { let entry = PxLogEntry { diff --git a/lib/crowdb-kv/tests/wal_test/replay_compaction_test.rs b/lib/crowdb-kv/tests/wal_test/replay_compaction_test.rs new file mode 100644 index 000000000..6f87b83db --- /dev/null +++ b/lib/crowdb-kv/tests/wal_test/replay_compaction_test.rs @@ -0,0 +1,116 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::path::PathBuf; +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use crowdb_kv::wal::record::{RecordType, WALRecord}; +use crowdb_kv::wal::replay::replay_group; +use crowdb_kv::wal::{IoBackend, MemBlockDevice, WalConfig, WalEngine}; + +fn accepted(ballot: u64, term: u64, value: &[u8]) -> WALRecord { + let mut payload = vec![1, 0, 0, 1, 0, 0, 0, b'k']; + payload.extend_from_slice(&u32::try_from(value.len()).unwrap().to_le_bytes()); + payload.extend_from_slice(value); + WALRecord::from_accepted( + 1, + &PxLogEntry { + slot: 1, + ballot: PxBallot::new(ballot, 7), + term, + payload: Bytes::from(payload), + }, + ) +} + +async fn fixture() -> (Arc, WalConfig, Arc) { + let backend = Arc::new(IoBackend::MemBlock(MemBlockDevice::new())); + let config = WalConfig { + wal_disks: vec![PathBuf::from("/compact")], + ..Default::default() + }; + let wal = WalEngine::create(backend.clone(), config.clone(), 1) + .await + .unwrap(); + (backend, config, wal) +} + +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn compaction_preserves_accepted_value_below_an_independent_higher_promise() { + let (backend, config, wal) = fixture().await; + let highest = accepted(10, 10, b"highest"); + for record in [ + WALRecord::from_promised(1, 20, 1, PxBallot::new(20, 7)), + accepted(4, 4, b"old"), + highest.clone(), + WALRecord::from_promised(1, 12, 1, PxBallot::new(12, 7)), + accepted(5, 5, b"later-old"), + ] { + wal.append(&record).await.unwrap(); + } + wal.seal_all().await.unwrap(); + let replay = replay_group(&backend, &config.wal_disks, 1).await.unwrap(); + assert_eq!(replay.records.len(), 2); + assert_eq!(replay.current_term, 20); + assert_eq!( + replay + .index + .segments() + .map(|segment| segment.record_count) + .sum::(), + 5 + ); + let replica = PxLocalReplica::restore_from_replay(7, PxLocalReplicaRole::Follower, &replay) + .await + .unwrap(); + assert_eq!(replica.accepted_at(1).await, highest.to_log_entry()); + assert_eq!(replica.promised_at(1).await, Some(PxBallot::new(20, 7))); + assert_eq!( + replica.learner.engine_get(b"k").await, + Some((1, b"highest".to_vec())) + ); +} + +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn discarded_accepts_still_advance_the_term_and_fence_old_votes() { + let (backend, config, wal) = fixture().await; + for record in [ + accepted(10, 10, b"highest"), + accepted(4, 30, b"old"), + WALRecord::from_vote_granted(1, 30, 42), + WALRecord::from_vote_granted(1, 12, 99), + ] { + wal.append(&record).await.unwrap(); + } + wal.seal_all().await.unwrap(); + let replay = replay_group(&backend, &config.wal_disks, 1).await.unwrap(); + assert_eq!(replay.current_term, 30); + assert_eq!(replay.voted_for, Some(42)); + assert_eq!( + replay + .records + .iter() + .filter(|record| record.record_type == RecordType::Accepted) + .count(), + 1 + ); + wal.append(&accepted(3, 31, b"discarded-new-term")).await.unwrap(); + wal.seal_all().await.unwrap(); + let replay = replay_group(&backend, &config.wal_disks, 1).await.unwrap(); + assert_eq!(replay.current_term, 31); + assert_eq!(replay.voted_for, None); +} + +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn equal_ballot_compaction_keeps_the_last_verified_accept() { + let (backend, config, wal) = fixture().await; + wal.append(&accepted(5, 5, b"earlier")).await.unwrap(); + let latest = accepted(5, 5, b"latest"); + wal.append(&latest).await.unwrap(); + wal.seal_all().await.unwrap(); + let replay = replay_group(&backend, &config.wal_disks, 1).await.unwrap(); + assert_eq!(replay.records, vec![latest]); +} diff --git a/lib/crowdb-kv/tests/wal_test/segment_test.rs b/lib/crowdb-kv/tests/wal_test/segment_test.rs index d190b15af..b55f57dbb 100644 --- a/lib/crowdb-kv/tests/wal_test/segment_test.rs +++ b/lib/crowdb-kv/tests/wal_test/segment_test.rs @@ -38,6 +38,35 @@ fn accepted_record(group: u64, slot: u64, payload_len: usize) -> WALRecord { } } +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn buffered_binary_replay_keeps_payloads_across_refills_and_large_frames() { + let backend = aligned_backend(); + let directory = PathBuf::from("/buffer-boundary/group1"); + let mut segment = WalSegment::create(&backend, &directory, 1, 1).await.unwrap(); + let mut records = Vec::new(); + for (index, length) in [13, 65536, 1_048_500, 37, 1_048_577, 0, 32768] + .into_iter() + .enumerate() + { + let record = accepted_record(1, u64::try_from(index).unwrap() + 1, length); + segment.append(&record).await.unwrap(); + records.push(record); + } + segment.seal().await.unwrap(); + let mut reader = SegmentReader::open(&backend, segment.path()).await.unwrap(); + let mut recovered = Vec::new(); + while let Some((record, _)) = reader.next_record().await.unwrap() { + recovered.push(record); + } + assert!(reader.read_footer().await.unwrap().is_some()); + drop(reader); + drop(segment); + assert_eq!( + recovered, records, + "earlier payload owners must survive every refill" + ); +} + #[tokio::test(flavor = "current_thread", start_paused = true)] async fn create_segment_writes_header() { let backend = sim_backend(); diff --git a/lib/crowdb-tree/include/crowdb-tree/maptable/buffer_pool.h b/lib/crowdb-tree/include/crowdb-tree/maptable/buffer_pool.h index dd997f8e5..47fbd979a 100644 --- a/lib/crowdb-tree/include/crowdb-tree/maptable/buffer_pool.h +++ b/lib/crowdb-tree/include/crowdb-tree/maptable/buffer_pool.h @@ -23,6 +23,7 @@ #include #include +#include #include #include @@ -178,7 +179,7 @@ class BufferPool [[nodiscard]] uint8_t *frame_bytes(uint32_t idx) { - return arena_.data() + (static_cast(idx) * page_bytes_); + return arena_.get() + (static_cast(idx) * page_bytes_); } void unpin(uint32_t idx); @@ -193,14 +194,14 @@ class BufferPool [[nodiscard]] int64_t acquire_victim(); Status write_back_io(uint32_t idx, PageAddr addr); - mutable std::mutex mu_; - std::condition_variable state_cv_; - std::vector arena_; - std::vector frames_; - uint32_t page_bytes_; - uint32_t num_frames_; - PageStore *store_; - uint32_t clock_hand_ = 0; + mutable std::mutex mu_; + std::condition_variable state_cv_; + std::unique_ptr arena_; + std::vector frames_; + uint32_t page_bytes_; + uint32_t num_frames_; + PageStore *store_; + uint32_t clock_hand_ = 0; std::vector ht_key_; // page_id or kInvalidPageId std::vector ht_val_; // frame index diff --git a/lib/crowdb-tree/include/crowdb-tree/snapshot/prepared.h b/lib/crowdb-tree/include/crowdb-tree/snapshot/prepared.h index f5fe4b6d8..64d62ef00 100644 --- a/lib/crowdb-tree/include/crowdb-tree/snapshot/prepared.h +++ b/lib/crowdb-tree/include/crowdb-tree/snapshot/prepared.h @@ -40,7 +40,7 @@ struct PreparedPageWrite uint64_t prior_addr = kNoAddr; uint64_t addr = 0; uint32_t logical_len = 0; // unpadded; mirrors PageBase::durable_plen - std::vector blob; // already IU-padded + std::vector blob; // padded to IU at submission; released after completion }; // A dirty MappingSegment's fresh image write, plus enough identity to @@ -80,10 +80,11 @@ struct PrefetchedPage std::vector blob; // IU-padded page content read from the store }; -// Output of prepare_snapshot_locked(): every byte this snapshot generation -// needs written, computed synchronously under write_mutex_ (the segment +// Output of prepare_snapshot_locked(): encoded page payloads and metadata +// bytes computed synchronously under write_mutex_ (the segment // scan + delta-fold + page/segment-image/directory encode is CPU/memory-only -// -- see the "Lock scope" note on #11). The caller writes +// without device padding). Page payloads gain device padding +// only at submission. The caller writes // `page_writes` and `segment_writes` (any order/concurrency) then // `directory_write`, then a durability barrier, then `anchor_write` -- // writing the anchor before that barrier would violate the crash-safety diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_manifest_build.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_manifest_build.cpp index 8252ff5da..f7f68b1d5 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_manifest_build.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_manifest_build.cpp @@ -128,10 +128,14 @@ Status ChunkPageStore::build_manifest(uint64_t expected_generation, std::shared_ pack.ref.length = static_cast(length); const size_t physical_length = chunk_framed_length(length); std::vector framed; + std::vector payload; + Status staging_status = read_staged_range(offset, length, &payload, cancellation); + if (!staging_status.ok()) { + return staging_status; + } if (physical_length == 0 || crowdb::protocol::encode_frames(chunk_page_frame_magic(purpose), - {.high = active_chunk_id_.high, .low = active_chunk_id_.low}, - std::span(staged_.data() + offset, length), + {.high = active_chunk_id_.high, .low = active_chunk_id_.low}, payload, monotonic_millis(), &framed) != crowdb::protocol::FrameError::Ok) { return Status::resource_exhausted("chunk page frame encoding failed"); } diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp index 483c3135c..d07b4925d 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp @@ -392,6 +392,7 @@ class ChunkPackPipelineImpl final : public ChunkPackPipeline, public std::enable std::memory_order_release); store->staged_initialized_ = false; store->staged_.clear(); + store->staged_base_.reset(); store->dirty_ranges_.clear(); store->inherited_manifest_.reset(); store->inherited_catalog_.reset(); @@ -405,11 +406,18 @@ class ChunkPackPipelineImpl final : public ChunkPackPipeline, public std::enable { auto &job = *writes[index]; orphan_pack_bytes.fetch_add(job.pack.ref.length, std::memory_order_relaxed); - const auto frame_error = crowdb::protocol::encode_frames( - chunk_page_frame_magic(chunk_purpose(job.pack.ref.chunk_id)), - {.high = job.pack.ref.chunk_id.high, .low = job.pack.ref.chunk_id.low}, - std::span(store->staged_.data() + job.source_offset, job.pack.ref.length), - monotonic_millis(), &job.framed); + std::vector payload; + Status staging_status = + store->read_staged_range(job.source_offset, job.pack.ref.length, &payload, cancellation); + if (!staging_status.ok()) { + stop_requested.store(true, std::memory_order_release); + completion.complete(std::move(staging_status)); + return; + } + const auto frame_error = + crowdb::protocol::encode_frames(chunk_page_frame_magic(chunk_purpose(job.pack.ref.chunk_id)), + {.high = job.pack.ref.chunk_id.high, .low = job.pack.ref.chunk_id.low}, + payload, monotonic_millis(), &job.framed); if (frame_error != crowdb::protocol::FrameError::Ok || job.framed.size() != job.physical_length) { stop_requested.store(true, std::memory_order_release); completion.complete(Status::corruption("chunk page frame encoding failed")); diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_page_purpose.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_page_purpose.cpp index 59a710bfb..5ed8d0fac 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_page_purpose.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_page_purpose.cpp @@ -32,12 +32,21 @@ Status ChunkPageStore::write_typed_at(PagePurpose purpose, uint64_t off, const u return Status::resource_exhausted("chunk page write exceeds address space"); } if (!staged_initialized_) { - Status status = materialize_active(&staged_); - if (!status.ok()) { - return status; + staged_base_ = reuse_base_manifest(); + if (staged_base_ != nullptr) { + if (staged_base_->owner_epoch > config_.owner_epoch) { + return Status::corruption("chunk manifest checksum or epoch is invalid"); + } + const RootCatalog &base_catalog = + staged_base_ == inherited_manifest_ && inherited_catalog_ != nullptr ? *inherited_catalog_ : *catalog_; + Status status = validate_manifest(*staged_base_, base_catalog); + if (!status.ok()) { + return status; + } } + staged_.reset(staged_base_ == nullptr ? 0 : staged_base_->logical_size); staged_purposes_.clear(); - auto base = reuse_base_manifest(); + auto base = staged_base_; if (base != nullptr) { for (const auto &pack : base->packs) { staged_purposes_.assign(pack.logical_offset, pack.ref.length, chunk_purpose(pack.ref.chunk_id)); @@ -45,12 +54,13 @@ Status ChunkPageStore::write_typed_at(PagePurpose purpose, uint64_t off, const u } staged_initialized_ = true; } - const size_t end = static_cast(off) + len; - if (end > staged_.size()) { - staged_.resize(end, 0); + const size_t end = static_cast(off) + len; + Status status = prepare_staged_write(off, len); + if (!status.ok()) { + return status; } + staged_.write(off, std::span(buf, len)); if (len != 0) { - std::memcpy(staged_.data() + off, buf, len); dirty_ranges_.emplace_back(off, len); staged_purposes_.assign(off, len, purpose); } diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_page_staging.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_page_staging.cpp new file mode 100644 index 000000000..fbb64cf58 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_page_staging.cpp @@ -0,0 +1,86 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "chunk_page_store.h" + +#include +#include + +namespace crowdb::tree::detail +{ + +Status ChunkPageStore::prepare_staged_write(uint64_t offset, size_t length) +{ + if (staged_base_ == nullptr || length == 0) { + return Status::Ok(); + } + const uint64_t end = offset + length; + for (const auto &pack : staged_base_->packs) { + const uint64_t pack_end = pack.logical_offset + pack.ref.length; + if (pack_end <= offset || pack.logical_offset >= end || (offset <= pack.logical_offset && end >= pack_end) || + staged_.covers(pack.logical_offset, pack.ref.length)) { + continue; + } + std::shared_ptr> persisted; + Status status = read_pack(pack.ref, &persisted, {}); + if (!status.ok()) { + return status; + } + std::vector captured(*persisted); + staged_.overlay(pack.logical_offset, captured); + staged_.write(pack.logical_offset, captured); + } + return Status::Ok(); +} + +Status ChunkPageStore::read_staged_range(uint64_t offset, size_t length, std::vector *out, + ChunkCancellation cancellation) const +{ + if (offset > staged_.size() || length > staged_.size() - offset) { + return Status::corruption("staged chunk range exceeds logical image"); + } + out->assign(length, 0); + if (staged_base_ != nullptr) { + const uint64_t end = offset + length; + for (const auto &pack : staged_base_->packs) { + const uint64_t begin = std::max(offset, pack.logical_offset); + const uint64_t stop = std::min(end, pack.logical_offset + pack.ref.length); + if (begin >= stop || staged_.covers(begin, stop - begin)) { + continue; + } + if (cancellation.cancelled()) { + return Status::unavailable("chunk staging read cancelled"); + } + std::shared_ptr> payload; + Status status = read_pack(pack.ref, &payload, cancellation); + if (!status.ok()) { + return status; + } + std::memcpy(out->data() + begin - offset, payload->data() + begin - pack.logical_offset, stop - begin); + } + } + staged_.overlay(offset, *out); + return cancellation.cancelled() ? Status::unavailable("chunk staging read cancelled") : Status::Ok(); +} + +const ChunkPagePack *ChunkPageStore::find_reusable_pack(const ChunkManifest &base, uint64_t logical_offset, + uint32_t length, ChunkCancellation cancellation) const +{ + const ChunkPagePack *found = find_pack_at(base, logical_offset, length); + if (found == nullptr) { + return nullptr; + } + std::shared_ptr> persisted; + Status status = read_pack(found->ref, &persisted, cancellation); + if (!status.ok() || persisted->size() != length) { + return nullptr; + } + if (!range_was_written(logical_offset, length)) { + return found; + } + std::vector overlaid(*persisted); + staged_.overlay(logical_offset, overlaid); + return overlaid == *persisted ? found : nullptr; +} + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_page_staging.h b/lib/crowdb-tree/src/backend/chunk/chunk_page_staging.h new file mode 100644 index 000000000..1b8dcb02d --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_page_staging.h @@ -0,0 +1,151 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace crowdb::tree::detail +{ + +// Single-writer overlay. Fragments never cross a 64-KiB boundary, bounding +// temporary copies when a later write replaces part of an earlier fragment. +// Trailing zero bytes retain logical coverage without retaining padding buffers. +class ChunkPageStaging +{ + public: + void reset(uint64_t size) + { + fragments_.clear(); + size_ = size; + } + + void clear() + { + reset(0); + } + + [[nodiscard]] size_t size() const + { + return static_cast(size_); + } + + void write(uint64_t offset, std::span bytes) + { + size_ = std::max(size_, offset + bytes.size()); + while (!bytes.empty()) { + const auto length = std::min(bytes.size(), kFragmentBytes - (offset % kFragmentBytes)); + assign(offset, bytes.first(length)); + offset += length; + bytes = bytes.subspan(length); + } + } + + void overlay(uint64_t offset, std::span bytes) const + { + auto next = fragments_.upper_bound(offset); + if (next != fragments_.begin()) { + --next; + } + const uint64_t end = offset + bytes.size(); + for (; next != fragments_.end() && next->first < end; ++next) { + const uint64_t begin = std::max(offset, next->first); + const uint64_t stop = std::min(end, next->first + next->second.length); + if (begin < stop) { + auto target = bytes.subspan(begin - offset, stop - begin); + std::fill(target.begin(), target.end(), 0); + const uint64_t copied_end = std::min(stop, next->first + next->second.bytes.size()); + if (begin < copied_end) { + std::memcpy(target.data(), next->second.bytes.data() + begin - next->first, copied_end - begin); + } + } + } + } + + [[nodiscard]] bool covers(uint64_t offset, uint64_t length) const + { + const uint64_t end = offset + length; + auto next = fragments_.upper_bound(offset); + if (next != fragments_.begin()) { + --next; + } + for (; next != fragments_.end() && offset < end; ++next) { + if (next->first > offset) { + return false; + } + offset = std::max(offset, next->first + next->second.length); + } + return offset >= end; + } + + private: + struct Fragment + { + uint64_t length = 0; + std::vector bytes; + + static Fragment capture(std::span input) + { + const auto last = std::find_if(input.rbegin(), input.rend(), [](uint8_t byte) { return byte != 0; }); + const auto kept = input.size() - static_cast(std::distance(input.rbegin(), last)); + const auto retained = input.first(kept); + return {.length = input.size(), .bytes = std::vector(retained.begin(), retained.end())}; + } + + [[nodiscard]] Fragment slice(uint64_t offset, uint64_t size) const + { + Fragment result; + if (offset < bytes.size()) { + result = capture(std::span(bytes).subspan(offset, std::min(size, bytes.size() - offset))); + } + result.length = size; + return result; + } + }; + + void assign(uint64_t offset, std::span bytes) + { + const uint64_t end = offset + bytes.size(); + auto next = fragments_.upper_bound(offset); + if (next != fragments_.begin()) { + const auto previous = std::prev(next); + if (previous->first + previous->second.length > offset) { + next = previous; + } + } + Fragment prefix; + Fragment suffix; + uint64_t prefix_offset = offset; + while (next != fragments_.end() && next->first < end) { + if (next->first < offset) { + prefix_offset = next->first; + prefix = next->second.slice(0, offset - next->first); + } + if (next->first + next->second.length > end) { + suffix = next->second.slice(end - next->first, next->first + next->second.length - end); + } + next = fragments_.erase(next); + } + if (prefix.length > 0) { + fragments_.emplace(prefix_offset, std::move(prefix)); + } + if (suffix.length > 0) { + fragments_.emplace(end, std::move(suffix)); + } + fragments_.emplace(offset, Fragment::capture(bytes)); + } + + static constexpr uint64_t kFragmentBytes = 64U * 1024U; + uint64_t size_ = 0; + std::map fragments_; +}; + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp index 60292e5e5..a3eecd15b 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp @@ -850,6 +850,9 @@ bool ChunkPageStore::inherited_snapshot_matches(const PageStore &source_store) c std::shared_ptr ChunkPageStore::reuse_base_manifest() const { + if (staged_initialized_) { + return staged_base_; + } auto bootstrap = bootstrap_layout_.load(std::memory_order_acquire); if (bootstrap != nullptr) { return bootstrap; @@ -862,22 +865,6 @@ std::shared_ptr ChunkPageStore::reuse_base_manifest() const return current == nullptr ? inherited_manifest_ : current; } -const ChunkPagePack *ChunkPageStore::find_reusable_pack(const ChunkManifest &base, uint64_t logical_offset, - uint32_t length, ChunkCancellation cancellation) const -{ - const ChunkPagePack *found = find_pack_at(base, logical_offset, length); - if (found == nullptr) { - return nullptr; - } - std::shared_ptr> persisted; - Status status = read_pack(found->ref, &persisted, cancellation); - if (!status.ok() || persisted->size() != length || - std::memcmp(persisted->data(), staged_.data() + logical_offset, length) != 0) { - return nullptr; - } - return found; -} - const ChunkPagePack *ChunkPageStore::find_pack_at(const ChunkManifest &base, uint64_t logical_offset, uint32_t length) { auto found = @@ -939,38 +926,6 @@ Status ChunkPageStore::validate_manifest(const ChunkManifest &manifest, const Ro return Status::Ok(); } -Status ChunkPageStore::materialize_active(std::vector *out) const -{ - auto manifest = reuse_base_manifest(); - if (manifest == nullptr) { - out->clear(); - return Status::Ok(); - } - if (manifest->owner_epoch > config_.owner_epoch) { - return Status::corruption("chunk manifest checksum or epoch is invalid"); - } - const RootCatalog &manifest_catalog = - manifest == inherited_manifest_ && inherited_catalog_ != nullptr ? *inherited_catalog_ : *catalog_; - Status manifest_status = validate_manifest(*manifest, manifest_catalog); - if (!manifest_status.ok()) { - return manifest_status; - } - out->assign(manifest->logical_size, 0); - for (const ChunkPagePack &pack : manifest->packs) { - const ChunkPageRef &ref = pack.ref; - if (pack.logical_offset > out->size() || ref.length > out->size() - pack.logical_offset) { - return Status::corruption("chunk page pack checksum or bounds are invalid"); - } - std::shared_ptr> payload; - Status read_status = read_pack(ref, &payload, {}); - if (!read_status.ok()) { - return read_status; - } - std::memcpy(out->data() + pack.logical_offset, payload->data(), payload->size()); - } - return Status::Ok(); -} - Status ChunkPageStore::read_at(uint64_t off, uint8_t *buf, size_t len) const { return read_at_cancellable(off, buf, len, {}); @@ -1234,6 +1189,7 @@ Status ChunkPageStore::sync_cancellable(ChunkCancellation cancellation) layout_valid_until_ms_.store(monotonic_millis() + config_.layout_validity_ms, std::memory_order_release); staged_initialized_ = false; staged_.clear(); + staged_base_.reset(); dirty_ranges_.clear(); inherited_manifest_.reset(); inherited_catalog_.reset(); diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h index 8957f4b34..762a94bf5 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h +++ b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h @@ -6,6 +6,7 @@ #pragma once #include "chunk_cancellation.h" +#include "chunk_page_staging.h" #include "chunk_purpose.h" #include "chunk_transport.h" #include "crowdb-tree/backend/async_page_store.h" @@ -365,7 +366,9 @@ class ChunkPageStore final : public PageStore, public AsyncPageStore size_t staged_pack_length(uint64_t offset, const ChunkManifest *base) const; static bool range_has_pack(const ChunkManifest &base, uint64_t offset, size_t length); uint64_t chunk_allocation_bytes() const; - Status materialize_active(std::vector *out) const; + Status read_staged_range(uint64_t offset, size_t length, std::vector *out, + ChunkCancellation cancellation = {}) const; + Status prepare_staged_write(uint64_t offset, size_t length); Status refresh_active_chunk(); Status build_manifest(uint64_t expected_generation, std::shared_ptr *out, uint64_t *new_pack_bytes, ChunkCancellation cancellation = {}); @@ -393,7 +396,8 @@ class ChunkPageStore final : public PageStore, public AsyncPageStore Config config_; std::shared_ptr catalog_; std::shared_ptr transport_; - std::vector staged_; + ChunkPageStaging staged_; + std::shared_ptr staged_base_; PagePurposeRanges staged_purposes_; std::vector> dirty_ranges_; std::vector> materialization_live_extents_; diff --git a/lib/crowdb-tree/src/maptable/buffer_pool.cpp b/lib/crowdb-tree/src/maptable/buffer_pool.cpp index dbcd627d0..3fa13339d 100644 --- a/lib/crowdb-tree/src/maptable/buffer_pool.cpp +++ b/lib/crowdb-tree/src/maptable/buffer_pool.cpp @@ -68,7 +68,8 @@ BufferPool::BufferPool(size_t capacity_bytes, uint32_t page_bytes, PageStore *st if (num_frames_ == 0) { num_frames_ = 1; } - arena_.assign(static_cast(num_frames_) * page_bytes_, 0); + // Misses load a complete validated frame; fresh frames are zeroed on acquisition. + arena_ = std::make_unique_for_overwrite(static_cast(num_frames_) * page_bytes_); frames_.resize(num_frames_); stats_.num_frames = num_frames_; diff --git a/lib/crowdb-tree/src/snapshot/persist.cpp b/lib/crowdb-tree/src/snapshot/persist.cpp index a8bedc3a7..9fd503382 100644 --- a/lib/crowdb-tree/src/snapshot/persist.cpp +++ b/lib/crowdb-tree/src/snapshot/persist.cpp @@ -407,7 +407,7 @@ Status Crowdbtree::prepare_snapshot_locked(PreparedSnapshot *out, std::vectorslot_count; ++slot) { - const uint64_t page_id = seg_idx * MappingTable::kSegmentSize + slot; + const uint64_t page_id = (seg_idx * MappingTable::kSegmentSize) + slot; uint64_t word = segment->slots[slot].load(std::memory_order_relaxed); if (!slot_word::is_resident(word)) { continue; @@ -517,7 +517,6 @@ Status Crowdbtree::queue_snapshot_page_locked(SnapshotPrepareContext &ctx, uint6 auto logical = static_cast(blob.size()); *addr = ctx.alloc.alloc(logical); *logical_len = logical; - blob.resize(round_up_to_iu(logical, ctx.iu), 0); ctx.out->page_writes.push_back(PreparedPageWrite{ .page_id = page_id, .page = page, @@ -1054,277 +1053,6 @@ Status Crowdbtree::materialize_ownership(uint64_t *bytes_written, bool *complete return status; } -Status Crowdbtree::snapshot(uint64_t *out_last_applied, uint64_t *out_snapshot_seq) -{ - if (opt_.page_store == nullptr) { - return Status::invalid_argument("snapshot: no page_store"); - } - auto snap_t0 = std::chrono::steady_clock::now(); - acquire_snapshot_slot(); - PreparedSnapshot prepared; - Status ps; - { - auto apply_t0 = std::chrono::steady_clock::now(); - std::scoped_lock lk(write_mutex_); - ps = prepare_snapshot_locked(&prepared); - if (metrics_.snapshot_apply_l != nullptr) { - auto ns = std::chrono::duration_cast(std::chrono::steady_clock::now() - apply_t0) - .count(); - metrics_.snapshot_apply_l->observe(static_cast(ns)); - } - } - if (!ps.ok()) { - CRB_LOG_ERROR("[{}] snapshot: prepare_snapshot_locked failed: {}", name_, ps.to_string()); - release_snapshot_slot(); - return ps; - } - for (auto &w : prepared.page_writes) { - auto write_t0 = std::chrono::steady_clock::now(); - Status s = opt_.page_store->write_at(w.addr, w.blob.data(), w.blob.size()); - if (metrics_.snapshot_page_write_l != nullptr) { - auto ns = std::chrono::duration_cast(std::chrono::steady_clock::now() - write_t0) - .count(); - metrics_.snapshot_page_write_l->observe(static_cast(ns)); - } - if (metrics_.snapshot_page_write_bw != nullptr) { - metrics_.snapshot_page_write_bw->observe(w.blob.size()); - } - if (!s.ok()) { - CRB_LOG_ERROR("[{}] snapshot page write failed: addr={} status={}", name_, w.addr, s.to_string()); - release_snapshot_slot(); - return s; - } - } - for (auto &sw : prepared.segment_writes) { - Status s = opt_.page_store->write_typed_at(PagePurpose::kPageIndex, sw.addr, sw.blob.data(), sw.blob.size()); - if (!s.ok()) { - CRB_LOG_ERROR("[{}] snapshot segment write failed: addr={} status={}", name_, sw.addr, s.to_string()); - release_snapshot_slot(); - return s; - } - } - Status dw = - opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared.directory_write.addr, - prepared.directory_write.blob.data(), prepared.directory_write.blob.size()); - if (!dw.ok()) { - CRB_LOG_ERROR("[{}] snapshot directory write failed: addr={} status={}", name_, prepared.directory_write.addr, - dw.to_string()); - release_snapshot_slot(); - return dw; - } - // Barrier: pages + segment images + directory durable before the anchor - // that references them. - auto fsync_t0 = std::chrono::steady_clock::now(); - Status sync1 = opt_.page_store->sync(); - if (metrics_.fsync_l != nullptr) { - auto ns = - std::chrono::duration_cast(std::chrono::steady_clock::now() - fsync_t0).count(); - metrics_.fsync_l->observe(static_cast(ns)); - } - if (!sync1.ok()) { - CRB_LOG_ERROR("[{}] snapshot data sync failed: {}", name_, sync1.to_string()); - release_snapshot_slot(); - return sync1; - } - Status aw = opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared.anchor_write.addr, - prepared.anchor_write.blob.data(), prepared.anchor_write.blob.size()); - if (!aw.ok()) { - CRB_LOG_ERROR("[{}] snapshot anchor write failed: addr={} status={}", name_, prepared.anchor_write.addr, - aw.to_string()); - release_snapshot_slot(); - return aw; - } - auto fsync2_t0 = std::chrono::steady_clock::now(); - Status sync2 = opt_.page_store->sync(); - if (metrics_.fsync_l != nullptr) { - auto ns = - std::chrono::duration_cast(std::chrono::steady_clock::now() - fsync2_t0).count(); - metrics_.fsync_l->observe(static_cast(ns)); - } - if (!sync2.ok()) { - CRB_LOG_ERROR("[{}] snapshot anchor sync failed: {}", name_, sync2.to_string()); - commit_prepared_snapshot(prepared); - release_snapshot_slot(); - return sync2; - } - - // Observe metadata write bytes: segments + directory + anchor. - if (metrics_.snapshot_meta_write_bw != nullptr) { - uint64_t meta_bytes = 0; - for (const auto &sw : prepared.segment_writes) { - meta_bytes += sw.blob.size(); - } - meta_bytes += prepared.directory_write.blob.size(); - meta_bytes += prepared.anchor_write.blob.size(); - metrics_.snapshot_meta_write_bw->observe(meta_bytes); - } - - finalize_prepared_snapshot(prepared); - - // Publish the exact logical summary only after the root anchor is durable. - if (PageBase *root = resident(root_page_id_.load(std::memory_order_acquire)); root != nullptr) { - PageSummary summary; - if (root->type == page_type::kLeafBase) { - summary = static_cast(root)->summary(); - } - else if (root->type == page_type::kInnerBase) { - summary = static_cast(root)->summary(); - } - if (summary.exact) { - summary_live_kv_.store(summary.live_kv, std::memory_order_release); - summary_live_key_bytes_.store(summary.live_key_bytes, std::memory_order_release); - summary_live_value_bytes_.store(summary.live_value_bytes, std::memory_order_release); - summary_live_logical_bytes_.store(summary.live_logical_bytes, std::memory_order_release); - summary_reachable_overflow_pages_.store(summary.reachable_overflow_pages, std::memory_order_release); - summary_reachable_page_capacity_bytes_.store(summary.reachable_page_capacity_bytes, - std::memory_order_release); - summary_covered_slot_.store(prepared.last_applied_slot, std::memory_order_release); - summary_root_version_.store(prepared.seq, std::memory_order_release); - summary_available_.store(true, std::memory_order_release); - } - } - - release_snapshot_slot(); - if (out_last_applied != nullptr) { - *out_last_applied = prepared.last_applied_slot; - } - if (out_snapshot_seq != nullptr) { - *out_snapshot_seq = prepared.seq; - } - if (metrics_.snapshot_l != nullptr) { - auto ns = - std::chrono::duration_cast(std::chrono::steady_clock::now() - snap_t0).count(); - metrics_.snapshot_l->observe(static_cast(ns)); - } - return Status::Ok(); -} - -void Crowdbtree::snapshot_async( - std::function on_done) // NOLINT(performance-unnecessary-value-param) -{ - if (opt_.page_store == nullptr) { - on_done(Status::invalid_argument("snapshot: no page_store"), 0); - return; - } - if (opt_.async_page_store != nullptr) { - acquire_snapshot_slot(); - auto prepared = std::make_shared(); - Status ps; - { - std::scoped_lock lk(write_mutex_); - ps = prepare_snapshot_locked(prepared.get()); - } - if (!ps.ok()) { - release_snapshot_slot(); - on_done(ps, 0); - return; - } - snapshot_write_next_async(std::move(prepared), 0, std::move(on_done)); - return; - } - // No async backend wired -- run the synchronous path in - // this stack frame; still correct, just not genuinely async. - uint64_t last_applied = 0; - Status st = snapshot(&last_applied); - on_done(st, last_applied); -} - -void Crowdbtree::snapshot_write_next_async( // NOLINT(readability-convert-member-functions-to-static) - std::shared_ptr prepared, // NOLINT(performance-unnecessary-value-param) - size_t idx, std::function on_done) // NOLINT(performance-unnecessary-value-param) -{ - // snapshot_inflight_ (acquired by snapshot_async() before - // prepare_snapshot_locked()) stays held across this entire async chain - // -- see snapshot_async's doc comment on crowdb-tree.h for why an atomic - // spin-gate is used here instead of write_mutex_ (which cannot be - // unlocked from a different thread than the one that locked it, and - // this chain's completions run on the Reactor thread). `prepared` is a - // shared_ptr so each hop's lambda can carry it to the next hop after - // this call's own stack frame returns. - if (idx < prepared->page_writes.size()) { - const PreparedPageWrite &w = prepared->page_writes[idx]; - opt_.async_page_store->submit_write( - w.addr, w.blob.data(), w.blob.size(), - detail::own_async_completion([this, prepared, idx, on_done](const Status &st) mutable { - if (!st.ok()) { - release_snapshot_slot(); - on_done(st, 0); - return; - } - snapshot_write_next_async(std::move(prepared), idx + 1, std::move(on_done)); - })); - return; - } - size_t seg_idx_in_list = idx - prepared->page_writes.size(); - if (seg_idx_in_list < prepared->segment_writes.size()) { - const PreparedSegmentWrite &sw = prepared->segment_writes[seg_idx_in_list]; - opt_.async_page_store->submit_typed_write( - PagePurpose::kPageIndex, sw.addr, sw.blob.data(), sw.blob.size(), - detail::own_async_completion([this, prepared, idx, on_done](const Status &st) mutable { - if (!st.ok()) { - release_snapshot_slot(); - on_done(st, 0); - return; - } - snapshot_write_next_async(std::move(prepared), idx + 1, std::move(on_done)); - })); - return; - } - - const PreparedSnapshotWrite &dw = prepared->directory_write; - opt_.async_page_store->submit_typed_write( - PagePurpose::kPageIndex, dw.addr, dw.blob.data(), dw.blob.size(), - detail::own_async_completion([this, prepared, on_done](const Status &st) mutable { - if (!st.ok()) { - release_snapshot_slot(); - on_done(st, 0); - return; - } - // Barrier: pages + segment images + directory durable before the anchor - // that references them. - Status fs1 = opt_.async_page_store->submit_fsync( - detail::own_async_completion([this, prepared, on_done](const Status &st2) mutable { - if (!st2.ok()) { - release_snapshot_slot(); - on_done(st2, 0); - return; - } - const PreparedSnapshotWrite &aw = prepared->anchor_write; - opt_.async_page_store->submit_typed_write( - PagePurpose::kPageIndex, aw.addr, aw.blob.data(), aw.blob.size(), - detail::own_async_completion([this, prepared, on_done](const Status &st3) mutable { - if (!st3.ok()) { - release_snapshot_slot(); - on_done(st3, 0); - return; - } - Status fs2 = opt_.async_page_store->submit_fsync( - detail::own_async_completion([this, prepared, on_done](const Status &st4) mutable { - if (!st4.ok()) { - commit_prepared_snapshot(*prepared); - release_snapshot_slot(); - on_done(st4, 0); - return; - } - finalize_prepared_snapshot(*prepared); - uint64_t last_applied = prepared->last_applied_slot; - release_snapshot_slot(); - on_done(Status::Ok(), last_applied); - })); - if (!fs2.ok()) { - commit_prepared_snapshot(*prepared); - release_snapshot_slot(); - on_done(fs2, 0); - } - })); - })); - if (!fs1.ok()) { - release_snapshot_slot(); - on_done(fs1, 0); - } - })); -} - Status Crowdbtree::prefetch_sparse_pages(std::vector *out, std::set *selected_blocks) { selected_blocks->clear(); @@ -1386,71 +1114,6 @@ Status Crowdbtree::prefetch_sparse_pages(std::vector *out, std:: return Status::Ok(); } -Status Crowdbtree::persist_compaction_snapshot(std::vector prefetched, - std::set selected_blocks, PreparedSnapshot *prepared) -{ - acquire_snapshot_slot(); - Status prepare; - { - std::scoped_lock lk(write_mutex_); - prepare = prepare_snapshot_locked(prepared, std::move(prefetched), std::move(selected_blocks)); - } - if (!prepare.ok()) { - CRB_LOG_ERROR("[{}] compact_sparse_blocks: prepare failed: {}", name_, prepare.to_string()); - release_snapshot_slot(); - return prepare; - } - - for (auto &write : prepared->page_writes) { - Status status = opt_.page_store->write_at(write.addr, write.blob.data(), write.blob.size()); - if (!status.ok()) { - release_snapshot_slot(); - return status; - } - } - for (auto &write : prepared->segment_writes) { - Status status = - opt_.page_store->write_typed_at(PagePurpose::kPageIndex, write.addr, write.blob.data(), write.blob.size()); - if (!status.ok()) { - release_snapshot_slot(); - return status; - } - } - Status directory = - opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared->directory_write.addr, - prepared->directory_write.blob.data(), prepared->directory_write.blob.size()); - if (!directory.ok()) { - release_snapshot_slot(); - return directory; - } - Status durable_contents = opt_.page_store->sync(); - if (!durable_contents.ok()) { - release_snapshot_slot(); - return durable_contents; - } - Status anchor = - opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared->anchor_write.addr, - prepared->anchor_write.blob.data(), prepared->anchor_write.blob.size()); - if (!anchor.ok()) { - release_snapshot_slot(); - return anchor; - } - Status committed = opt_.page_store->sync(); - if (!committed.ok()) { - std::vector invalid_anchor(prepared->anchor_write.blob.size(), 0); - Status rollback = opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared->anchor_write.addr, - invalid_anchor.data(), invalid_anchor.size()); - if (rollback.ok()) { - (void)opt_.page_store->sync(); - } - release_snapshot_slot(); - return committed; - } - finalize_prepared_snapshot(*prepared); - release_snapshot_slot(); - return Status::Ok(); -} - void Crowdbtree::record_compaction_metrics(const MergeGcStats &stats, uint64_t elapsed_ns) { if (metrics_.merge_gc_blocks_c != nullptr && stats.blocks_selected > 0) { diff --git a/lib/crowdb-tree/src/snapshot/write.cpp b/lib/crowdb-tree/src/snapshot/write.cpp new file mode 100644 index 000000000..cb457f1cb --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/write.cpp @@ -0,0 +1,361 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "async_completion_adapter.h" +#include "crowdb-common/log.h" +#include "crowdb-tree/backend/async_page_store.h" +#include "crowdb-tree/crowdb-tree.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace crowdb::tree +{ +Status Crowdbtree::snapshot(uint64_t *out_last_applied, uint64_t *out_snapshot_seq) +{ + if (opt_.page_store == nullptr) { + return Status::invalid_argument("snapshot: no page_store"); + } + auto snap_t0 = std::chrono::steady_clock::now(); + acquire_snapshot_slot(); + PreparedSnapshot prepared; + Status ps; + { + auto apply_t0 = std::chrono::steady_clock::now(); + std::scoped_lock lk(write_mutex_); + ps = prepare_snapshot_locked(&prepared); + if (metrics_.snapshot_apply_l != nullptr) { + auto ns = std::chrono::duration_cast(std::chrono::steady_clock::now() - apply_t0) + .count(); + metrics_.snapshot_apply_l->observe(static_cast(ns)); + } + } + if (!ps.ok()) { + CRB_LOG_ERROR("[{}] snapshot: prepare_snapshot_locked failed: {}", name_, ps.to_string()); + release_snapshot_slot(); + return ps; + } + for (auto &w : prepared.page_writes) { + w.blob.resize(round_up_to_iu(w.logical_len, opt_.page_store->iu_size()), 0); + auto write_t0 = std::chrono::steady_clock::now(); + Status s = opt_.page_store->write_at(w.addr, w.blob.data(), w.blob.size()); + if (metrics_.snapshot_page_write_l != nullptr) { + auto ns = std::chrono::duration_cast(std::chrono::steady_clock::now() - write_t0) + .count(); + metrics_.snapshot_page_write_l->observe(static_cast(ns)); + } + if (metrics_.snapshot_page_write_bw != nullptr) { + metrics_.snapshot_page_write_bw->observe(w.blob.size()); + } + if (!s.ok()) { + CRB_LOG_ERROR("[{}] snapshot page write failed: addr={} status={}", name_, w.addr, s.to_string()); + release_snapshot_slot(); + return s; + } + std::vector().swap(w.blob); + } + for (auto &sw : prepared.segment_writes) { + Status s = opt_.page_store->write_typed_at(PagePurpose::kPageIndex, sw.addr, sw.blob.data(), sw.blob.size()); + if (!s.ok()) { + CRB_LOG_ERROR("[{}] snapshot segment write failed: addr={} status={}", name_, sw.addr, s.to_string()); + release_snapshot_slot(); + return s; + } + } + Status dw = + opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared.directory_write.addr, + prepared.directory_write.blob.data(), prepared.directory_write.blob.size()); + if (!dw.ok()) { + CRB_LOG_ERROR("[{}] snapshot directory write failed: addr={} status={}", name_, prepared.directory_write.addr, + dw.to_string()); + release_snapshot_slot(); + return dw; + } + // Barrier: pages + segment images + directory durable before the anchor + // that references them. + auto fsync_t0 = std::chrono::steady_clock::now(); + Status sync1 = opt_.page_store->sync(); + if (metrics_.fsync_l != nullptr) { + auto ns = + std::chrono::duration_cast(std::chrono::steady_clock::now() - fsync_t0).count(); + metrics_.fsync_l->observe(static_cast(ns)); + } + if (!sync1.ok()) { + CRB_LOG_ERROR("[{}] snapshot data sync failed: {}", name_, sync1.to_string()); + release_snapshot_slot(); + return sync1; + } + Status aw = opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared.anchor_write.addr, + prepared.anchor_write.blob.data(), prepared.anchor_write.blob.size()); + if (!aw.ok()) { + CRB_LOG_ERROR("[{}] snapshot anchor write failed: addr={} status={}", name_, prepared.anchor_write.addr, + aw.to_string()); + release_snapshot_slot(); + return aw; + } + auto fsync2_t0 = std::chrono::steady_clock::now(); + Status sync2 = opt_.page_store->sync(); + if (metrics_.fsync_l != nullptr) { + auto ns = + std::chrono::duration_cast(std::chrono::steady_clock::now() - fsync2_t0).count(); + metrics_.fsync_l->observe(static_cast(ns)); + } + if (!sync2.ok()) { + CRB_LOG_ERROR("[{}] snapshot anchor sync failed: {}", name_, sync2.to_string()); + commit_prepared_snapshot(prepared); + release_snapshot_slot(); + return sync2; + } + + // Observe metadata write bytes: segments + directory + anchor. + if (metrics_.snapshot_meta_write_bw != nullptr) { + uint64_t meta_bytes = 0; + for (const auto &sw : prepared.segment_writes) { + meta_bytes += sw.blob.size(); + } + meta_bytes += prepared.directory_write.blob.size(); + meta_bytes += prepared.anchor_write.blob.size(); + metrics_.snapshot_meta_write_bw->observe(meta_bytes); + } + + finalize_prepared_snapshot(prepared); + + // Publish the exact logical summary only after the root anchor is durable. + if (PageBase *root = resident(root_page_id_.load(std::memory_order_acquire)); root != nullptr) { + PageSummary summary; + if (root->type == page_type::kLeafBase) { + summary = static_cast(root)->summary(); + } + else if (root->type == page_type::kInnerBase) { + summary = static_cast(root)->summary(); + } + if (summary.exact) { + summary_live_kv_.store(summary.live_kv, std::memory_order_release); + summary_live_key_bytes_.store(summary.live_key_bytes, std::memory_order_release); + summary_live_value_bytes_.store(summary.live_value_bytes, std::memory_order_release); + summary_live_logical_bytes_.store(summary.live_logical_bytes, std::memory_order_release); + summary_reachable_overflow_pages_.store(summary.reachable_overflow_pages, std::memory_order_release); + summary_reachable_page_capacity_bytes_.store(summary.reachable_page_capacity_bytes, + std::memory_order_release); + summary_covered_slot_.store(prepared.last_applied_slot, std::memory_order_release); + summary_root_version_.store(prepared.seq, std::memory_order_release); + summary_available_.store(true, std::memory_order_release); + } + } + + release_snapshot_slot(); + if (out_last_applied != nullptr) { + *out_last_applied = prepared.last_applied_slot; + } + if (out_snapshot_seq != nullptr) { + *out_snapshot_seq = prepared.seq; + } + if (metrics_.snapshot_l != nullptr) { + auto ns = + std::chrono::duration_cast(std::chrono::steady_clock::now() - snap_t0).count(); + metrics_.snapshot_l->observe(static_cast(ns)); + } + return Status::Ok(); +} + +void Crowdbtree::snapshot_async( + std::function on_done) // NOLINT(performance-unnecessary-value-param) +{ + if (opt_.page_store == nullptr) { + on_done(Status::invalid_argument("snapshot: no page_store"), 0); + return; + } + if (opt_.async_page_store != nullptr) { + acquire_snapshot_slot(); + auto prepared = std::make_shared(); + Status ps; + { + std::scoped_lock lk(write_mutex_); + ps = prepare_snapshot_locked(prepared.get()); + } + if (!ps.ok()) { + release_snapshot_slot(); + on_done(ps, 0); + return; + } + snapshot_write_next_async(std::move(prepared), 0, std::move(on_done)); + return; + } + // No async backend wired -- run the synchronous path in + // this stack frame; still correct, just not genuinely async. + uint64_t last_applied = 0; + Status st = snapshot(&last_applied); + on_done(st, last_applied); +} + +void Crowdbtree::snapshot_write_next_async( // NOLINT(readability-convert-member-functions-to-static) + std::shared_ptr prepared, // NOLINT(performance-unnecessary-value-param) + size_t idx, std::function on_done) // NOLINT(performance-unnecessary-value-param) +{ + // snapshot_inflight_ (acquired by snapshot_async() before + // prepare_snapshot_locked()) stays held across this entire async chain + // -- see snapshot_async's doc comment on crowdb-tree.h for why an atomic + // spin-gate is used here instead of write_mutex_ (which cannot be + // unlocked from a different thread than the one that locked it, and + // this chain's completions run on the Reactor thread). `prepared` is a + // shared_ptr so each hop's lambda can carry it to the next hop after + // this call's own stack frame returns. + if (idx < prepared->page_writes.size()) { + PreparedPageWrite &w = prepared->page_writes[idx]; + w.blob.resize(round_up_to_iu(w.logical_len, opt_.page_store->iu_size()), 0); + opt_.async_page_store->submit_write( + w.addr, w.blob.data(), w.blob.size(), + detail::own_async_completion([this, prepared, idx, on_done](const Status &st) mutable { + if (!st.ok()) { + release_snapshot_slot(); + on_done(st, 0); + return; + } + std::vector().swap(prepared->page_writes[idx].blob); + snapshot_write_next_async(std::move(prepared), idx + 1, std::move(on_done)); + })); + return; + } + size_t seg_idx_in_list = idx - prepared->page_writes.size(); + if (seg_idx_in_list < prepared->segment_writes.size()) { + const PreparedSegmentWrite &sw = prepared->segment_writes[seg_idx_in_list]; + opt_.async_page_store->submit_typed_write( + PagePurpose::kPageIndex, sw.addr, sw.blob.data(), sw.blob.size(), + detail::own_async_completion([this, prepared, idx, on_done](const Status &st) mutable { + if (!st.ok()) { + release_snapshot_slot(); + on_done(st, 0); + return; + } + snapshot_write_next_async(std::move(prepared), idx + 1, std::move(on_done)); + })); + return; + } + + const PreparedSnapshotWrite &dw = prepared->directory_write; + opt_.async_page_store->submit_typed_write( + PagePurpose::kPageIndex, dw.addr, dw.blob.data(), dw.blob.size(), + detail::own_async_completion([this, prepared, on_done](const Status &st) mutable { + if (!st.ok()) { + release_snapshot_slot(); + on_done(st, 0); + return; + } + // Barrier: pages + segment images + directory durable before the anchor + // that references them. + Status fs1 = opt_.async_page_store->submit_fsync( + detail::own_async_completion([this, prepared, on_done](const Status &st2) mutable { + if (!st2.ok()) { + release_snapshot_slot(); + on_done(st2, 0); + return; + } + const PreparedSnapshotWrite &aw = prepared->anchor_write; + opt_.async_page_store->submit_typed_write( + PagePurpose::kPageIndex, aw.addr, aw.blob.data(), aw.blob.size(), + detail::own_async_completion([this, prepared, on_done](const Status &st3) mutable { + if (!st3.ok()) { + release_snapshot_slot(); + on_done(st3, 0); + return; + } + Status fs2 = opt_.async_page_store->submit_fsync( + detail::own_async_completion([this, prepared, on_done](const Status &st4) mutable { + if (!st4.ok()) { + commit_prepared_snapshot(*prepared); + release_snapshot_slot(); + on_done(st4, 0); + return; + } + finalize_prepared_snapshot(*prepared); + uint64_t last_applied = prepared->last_applied_slot; + release_snapshot_slot(); + on_done(Status::Ok(), last_applied); + })); + if (!fs2.ok()) { + commit_prepared_snapshot(*prepared); + release_snapshot_slot(); + on_done(fs2, 0); + } + })); + })); + if (!fs1.ok()) { + release_snapshot_slot(); + on_done(fs1, 0); + } + })); +} + +Status Crowdbtree::persist_compaction_snapshot(std::vector prefetched, + std::set selected_blocks, PreparedSnapshot *prepared) +{ + acquire_snapshot_slot(); + Status prepare; + { + std::scoped_lock lk(write_mutex_); + prepare = prepare_snapshot_locked(prepared, std::move(prefetched), std::move(selected_blocks)); + } + if (!prepare.ok()) { + CRB_LOG_ERROR("[{}] compact_sparse_blocks: prepare failed: {}", name_, prepare.to_string()); + release_snapshot_slot(); + return prepare; + } + + for (auto &write : prepared->page_writes) { + write.blob.resize(round_up_to_iu(write.logical_len, opt_.page_store->iu_size()), 0); + Status status = opt_.page_store->write_at(write.addr, write.blob.data(), write.blob.size()); + if (!status.ok()) { + release_snapshot_slot(); + return status; + } + std::vector().swap(write.blob); + } + for (auto &write : prepared->segment_writes) { + Status status = + opt_.page_store->write_typed_at(PagePurpose::kPageIndex, write.addr, write.blob.data(), write.blob.size()); + if (!status.ok()) { + release_snapshot_slot(); + return status; + } + } + Status directory = + opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared->directory_write.addr, + prepared->directory_write.blob.data(), prepared->directory_write.blob.size()); + if (!directory.ok()) { + release_snapshot_slot(); + return directory; + } + Status durable_contents = opt_.page_store->sync(); + if (!durable_contents.ok()) { + release_snapshot_slot(); + return durable_contents; + } + Status anchor = + opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared->anchor_write.addr, + prepared->anchor_write.blob.data(), prepared->anchor_write.blob.size()); + if (!anchor.ok()) { + release_snapshot_slot(); + return anchor; + } + Status committed = opt_.page_store->sync(); + if (!committed.ok()) { + std::vector invalid_anchor(prepared->anchor_write.blob.size(), 0); + Status rollback = opt_.page_store->write_typed_at(PagePurpose::kPageIndex, prepared->anchor_write.addr, + invalid_anchor.data(), invalid_anchor.size()); + if (rollback.ok()) { + (void)opt_.page_store->sync(); + } + release_snapshot_slot(); + return committed; + } + finalize_prepared_snapshot(*prepared); + release_snapshot_slot(); + return Status::Ok(); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/tests/integration/chunk_page_staging_test.cpp b/lib/crowdb-tree/tests/integration/chunk_page_staging_test.cpp new file mode 100644 index 000000000..fb64a3d2a --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_page_staging_test.cpp @@ -0,0 +1,90 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "backend/chunk/chunk_page_store.h" + +#include + +#include +#include +#include +#include + +namespace crowdb::tree::detail +{ +namespace +{ +class ChunkPageStagingTest : public ::testing::TestWithParam +{ +}; + +void publish(ChunkPageStore &store, bool asynchronous) +{ + ASSERT_TRUE(store.sync().ok()); + const uint8_t anchor = 7; + ASSERT_TRUE(store.write_at(0, &anchor, 1).ok()); + if (!asynchronous) { + ASSERT_TRUE(store.sync().ok()); + return; + } + + struct Completion + { + std::atomic done{false}; + bool success = false; + } completion; + + ASSERT_TRUE(store + .submit_fsync({.context = &completion, + .complete_fn = + [](void *context, Status status) { + auto &result = *static_cast(context); + result.success = status.ok(); + result.done.store(true, std::memory_order_release); + result.done.notify_one(); + }}) + .ok()); + completion.done.wait(false, std::memory_order_acquire); + ASSERT_TRUE(completion.success); +} + +TEST_P(ChunkPageStagingTest, SmallOverwriteDoesNotMaterializeThePublishedImage) +{ + auto catalog = std::make_shared(1); + auto transport = std::make_shared(); + const ChunkPageStore::Config config{.tree_id = 921, .owner_epoch = 1, .pack_bytes = 65536, .iu_size = 1}; + std::vector expected(16 * 1024 * 1024, 0x53); + { + ChunkPageStore store(config, catalog, transport); + ASSERT_TRUE(store.write_at(8192, expected.data(), expected.size()).ok()); + publish(store, GetParam()); + } + { + ChunkPageStore store(config, catalog, transport); + const std::array first{1, 2, 3, 4}; + const std::array last{8, 9}; + ASSERT_TRUE(store.write_at(8192 + 65535, first.data(), first.size()).ok()); + ASSERT_TRUE(store.write_at(8192 + 65536, last.data(), last.size()).ok()); + ASSERT_TRUE(store.write_at(131071, first.data(), first.size()).ok()); + ASSERT_TRUE(store.write_at(131072, last.data(), last.size()).ok()); + EXPECT_LE(store.stats().pack_reads, 2U) + << "small staging writes must not read and retain the complete published image"; + expected[65535] = 1; + expected[65536] = 8; + expected[65537] = 9; + expected[65538] = 4; + expected[131071 - 8192] = 1; + expected[131072 - 8192] = 8; + expected[131073 - 8192] = 9; + expected[131074 - 8192] = 4; + publish(store, GetParam()); + } + ChunkPageStore reopened(config, catalog, transport); + std::vector actual(expected.size()); + ASSERT_TRUE(reopened.read_at(8192, actual.data(), actual.size()).ok()); + EXPECT_EQ(actual, expected); +} + +INSTANTIATE_TEST_SUITE_P(PublicationPaths, ChunkPageStagingTest, ::testing::Bool()); +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/integration/snapshot_memory_test.cpp b/lib/crowdb-tree/tests/integration/snapshot_memory_test.cpp new file mode 100644 index 000000000..3a6ff5a41 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/snapshot_memory_test.cpp @@ -0,0 +1,274 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "backend/chunk/chunk_page_staging.h" +#include "crowdb-tree/backend/async_page_store.h" +#include "crowdb-tree/backend/page_store.h" +#include "crowdb-tree/crowdb-tree.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__GLIBC__) && !defined(__SANITIZE_ADDRESS__) +# include +# include + +namespace crowdb::tree +{ +namespace +{ +size_t resident_bytes() +{ + malloc_trim(0); + std::ifstream status("/proc/self/statm"); + size_t virtual_pages = 0; + size_t resident_pages = 0; + status >> virtual_pages >> resident_pages; + return resident_pages * static_cast(sysconf(_SC_PAGESIZE)); +} + +// Reserve the device before sampling allocations. Backend growth must not be +// confused with encoded snapshot pages retained after their writes complete. +class TestSnapshotBufferStore final : public PageStore, public AsyncPageStore +{ + public: + TestSnapshotBufferStore() : bytes_(96 * 1024 * 1024) + { + } + + Status write_at(uint64_t off, const uint8_t *buf, size_t len) override + { + if (baseline > 0 && !prepared_bytes.has_value()) { + const auto current = resident_bytes(); + prepared_bytes = current > baseline ? current - baseline : 0; + } + if (off > bytes_.size() || len > bytes_.size() - off) { + return Status::resource_exhausted("test device is full"); + } + std::memcpy(bytes_.data() + off, buf, len); + size_ = std::max(size_, off + len); + return Status::Ok(); + } + + Status write_typed_at(PagePurpose /*purpose*/, uint64_t off, const uint8_t *buf, size_t len) override + { + if (!retained_bytes.has_value()) { + const auto current = resident_bytes(); + retained_bytes = current > baseline ? current - baseline : 0; + } + return write_at(off, buf, len); + } + + Status read_at(uint64_t off, uint8_t *buf, size_t len) const override + { + if (off > size_ || len > size_ - off) { + return Status::not_found("test device range is absent"); + } + std::memcpy(buf, bytes_.data() + off, len); + return Status::Ok(); + } + + Status sync() override + { + return Status::Ok(); + } + + [[nodiscard]] uint64_t size() const override + { + return size_; + } + + [[nodiscard]] uint32_t iu_size() const override + { + return 65536; + } + + uint64_t submit_read(PageAddr addr, void *buf, size_t len, AsyncCompletion done) override + { + done.complete(read_at(addr, static_cast(buf), len)); + return 0; + } + + uint64_t submit_write(PageAddr addr, const void *buf, size_t len, AsyncCompletion done) override + { + pending_ = Pending{.addr = addr, .data = buf, .length = len, .done = done, .typed = false}; + return 0; + } + + uint64_t submit_typed_write(PagePurpose /*purpose*/, PageAddr addr, const void *buf, size_t len, + AsyncCompletion done) override + { + pending_ = Pending{.addr = addr, .data = buf, .length = len, .done = done, .typed = true}; + return 0; + } + + Status submit_fsync(AsyncCompletion done) override + { + done.complete(sync()); + return Status::Ok(); + } + + void cancel(uint64_t /*operation_id*/) override + { + } + + bool complete_next() + { + if (!pending_.has_value()) { + return false; + } + const auto write = pending_.value(); + pending_.reset(); + const auto *data = static_cast(write.data); + auto status = write.typed ? write_typed_at(PagePurpose::kPageIndex, write.addr, data, write.length) + : write_at(write.addr, data, write.length); + write.done.complete(std::move(status)); + return true; + } + + size_t baseline = 0; + std::optional prepared_bytes; + std::optional retained_bytes; + + private: + struct Pending + { + uint64_t addr; + const void *data; + size_t length; + AsyncCompletion done; + bool typed; + }; + + std::vector bytes_; + uint64_t size_ = 0; + std::optional pending_; +}; + +class SnapshotMemoryTest : public ::testing::TestWithParam +{ +}; + +TEST_P(SnapshotMemoryTest, ReleasesCompletedPageBuffersBeforeWritingMetadata) +{ + TestSnapshotBufferStore store; + Config config; + config.page_store = &store; + config.async_page_store = GetParam() ? &store : nullptr; + config.frame_bytes = 65536; + config.compression = compress_algo::kNone; + const std::string value(65536, 's'); + { + Crowdbtree tree(config); + for (uint64_t i = 0; i < 256; ++i) { + const Batch batch{{{.key = "snapshot-" + std::to_string(i), .kind = OpKind::kPut, .value = value}}}; + ASSERT_TRUE(tree.apply(i + 1, batch).ok()); + } + ASSERT_TRUE(tree.flush().ok()); + store.baseline = resident_bytes(); + ASSERT_GT(store.baseline, 0U); + if (GetParam()) { + bool complete = false; + tree.snapshot_async([&](Status status, uint64_t applied) { + EXPECT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(applied, 256U); + complete = true; + }); + while (!complete) { + ASSERT_TRUE(store.complete_next()); + } + } + else { + ASSERT_TRUE(tree.snapshot().ok()); + } + ASSERT_TRUE(store.retained_bytes.has_value()); + ASSERT_TRUE(store.prepared_bytes.has_value()); + const auto prepared_bytes = store.prepared_bytes.value_or(std::numeric_limits::max()); + std::cout << "Snapshot prepared RSS growth: " << prepared_bytes << " bytes\n"; + EXPECT_LT(prepared_bytes, 48U * 1024 * 1024) << "snapshot preparation retained device padding for every page"; + const auto retained_bytes = store.retained_bytes.value_or(std::numeric_limits::max()); + std::cout << "Snapshot retained RSS growth: " << retained_bytes << " bytes\n"; + EXPECT_LT(retained_bytes, 8U * 1024 * 1024) << "completed page writes still retain their encoded buffers"; + } + config.async_page_store = nullptr; + std::unique_ptr reopened; + ASSERT_TRUE(Crowdbtree::open(config, &reopened).ok()); + for (uint64_t i = 0; i < 256; ++i) { + std::string actual; + uint64_t revision = 0; + ASSERT_TRUE(reopened->get(Slice("snapshot-" + std::to_string(i)), &revision, &actual)); + EXPECT_EQ(actual, value); + EXPECT_EQ(revision, i + 1); + } +} + +INSTANTIATE_TEST_SUITE_P(WritePaths, SnapshotMemoryTest, ::testing::Bool()); + +TEST(SnapshotMemory, PaddedOverlayKeepsZeroCoverageWithoutRetainingPadding) +{ + constexpr size_t fragment_bytes = 65536; + constexpr size_t page_count = 256; + std::vector page(2 * fragment_bytes, 0); + std::fill_n(page.begin(), fragment_bytes + 32, 0x53); + detail::ChunkPageStaging staging; + const auto before = resident_bytes(); + for (size_t i = 0; i < page_count; ++i) { + staging.write(i * page.size(), page); + } + const auto growth = resident_bytes() - before; + std::cout << "Padded overlay RSS growth: " << growth << " bytes\n"; + EXPECT_LT(growth, 24U * 1024 * 1024) << "overlay retained allocation-unit padding alongside logical page bytes"; + const size_t size = page_count * page.size(); + ASSERT_EQ(staging.size(), size); + ASSERT_TRUE(staging.covers(0, size)); + + std::vector expected(size); + for (size_t i = 0; i < page_count; ++i) { + std::copy(page.begin(), page.end(), expected.begin() + static_cast(i * page.size())); + } + const std::array patch{4, 0, 5}; + for (const size_t offset : {fragment_bytes + 1024, (3 * fragment_bytes) - 1}) { + staging.write(offset, patch); + std::copy(patch.begin(), patch.end(), expected.begin() + static_cast(offset)); + } + const std::vector zeros(fragment_bytes, 0); + staging.write(0, zeros); + std::fill_n(expected.begin(), fragment_bytes, 0); + std::vector actual(size, 0xAA); + staging.overlay(0, actual); + EXPECT_EQ(actual, expected); + EXPECT_TRUE(staging.covers(0, size)); +} + +TEST(SnapshotMemory, UnusedFrameCapacityDoesNotPrefaultTheArena) +{ + MemPageStore store(65536); + const auto before = resident_bytes(); + BufferPool pool(256U * 1024 * 1024, 65536, &store); + const auto current = resident_bytes(); + const auto growth = current > before ? current - before : 0; + std::cout << "Unused frame-cache RSS growth: " << growth << " bytes\n"; + EXPECT_LT(growth, 8U * 1024 * 1024); + EXPECT_EQ(pool.stats().num_frames, 4096U); + EXPECT_EQ(pool.stats().used, 0U); + + uint32_t index = 0; + uint8_t *bytes = nullptr; + ASSERT_TRUE(pool.acquire_frame(&index, &bytes).ok()); + EXPECT_TRUE(std::all_of(bytes, bytes + 65536, [](uint8_t byte) { return byte == 0; })); + std::memset(bytes, 0x53, 65536); + pool.release_frame(index); + EXPECT_EQ(pool.stats().num_frames, 4096U); +} +} // namespace +} // namespace crowdb::tree +#endif From 81d07f03398aaa144e8dcec1187bff0a0d18f91b Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 31 Oct 2026 19:28:29 +0800 Subject: [PATCH 09/24] Prefer nearby mirrors during partition recovery --- app/crowdb-chunk-kv-server/src/storage.rs | 8 +- .../chunkio/design-crowdb-chunkio-reader.md | 15 ++- .../src/chunk/chunk_reader.rs | 7 +- .../src/chunk/chunk_reader/locality.rs | 57 ++++++++ .../src/chunk/strip_reader.rs | 4 + .../tests/chunk_reader_test.rs | 6 + .../chunk_reader_test/mirror_locality.rs | 36 +++++ .../tests/common/mirror_locality.rs | 124 ++++++++++++++++++ 8 files changed, 251 insertions(+), 6 deletions(-) create mode 100644 lib/crowdb-chunk-client/src/chunk/chunk_reader/locality.rs create mode 100644 lib/crowdb-chunk-client/tests/chunk_reader_test/mirror_locality.rs create mode 100644 lib/crowdb-chunk-client/tests/common/mirror_locality.rs diff --git a/app/crowdb-chunk-kv-server/src/storage.rs b/app/crowdb-chunk-kv-server/src/storage.rs index 239539f99..6fc078419 100644 --- a/app/crowdb-chunk-kv-server/src/storage.rs +++ b/app/crowdb-chunk-kv-server/src/storage.rs @@ -98,6 +98,10 @@ impl ChunkKvStorage { chunk_capacity_bytes: config.storage.stream_chunk_capacity_bytes, ..StreamConfig::default() }, + ChunkReadPolicy { + preferred_node_id: config.node_id, + ..ChunkReadPolicy::default() + }, ) .await } @@ -109,13 +113,14 @@ impl ChunkKvStorage { writer_lease_ms: u64, tree: TreeStorageGeometry, stream_config: StreamConfig, + read_policy: ChunkReadPolicy, ) -> Result { let streams = Arc::new( ProductionStreamRuntime::new_with_mirror_copies( Arc::clone(&kv), &chunk_io, writer_lease_ms, - ChunkReadPolicy::default(), + read_policy, stream_config, tree.mirror_copies, ) @@ -147,6 +152,7 @@ impl ChunkKvStorage { strip_capacity_bytes: 0, }, StreamConfig::default(), + ChunkReadPolicy::default(), ) .await } diff --git a/doc/design/chunkio/design-crowdb-chunkio-reader.md b/doc/design/chunkio/design-crowdb-chunkio-reader.md index 97ff8421d..ea09f3806 100644 --- a/doc/design/chunkio/design-crowdb-chunkio-reader.md +++ b/doc/design/chunkio/design-crowdb-chunkio-reader.md @@ -35,7 +35,8 @@ lock-free DiskIO route snapshot. It exposes: `ChunkReadPolicy` configures stream-window bytes, a 256-MiB process-wide EC-recovery budget, layout safety margin, bounded layout retries, and the -ad-hoc full-fragment threshold. +ad-hoc full-fragment threshold. An optional preferred node supplies a mirror +read locality hint; ChunkKV uses its configured node identity for WAL replay. ## 2. Location mapping @@ -55,9 +56,15 @@ offset. The output length must equal the requested logical length. ## 3. Strip reads -Mirror reads try usable replicas in metadata order. Each request uses the -exact segment-relative offset and length; DiskIO supports arbitrary byte -ranges. +Mirror reads try usable replicas in metadata order by default. With a preferred +node and complete retained route hints for a strip, they first try that node, +then other nodes in its rack, preserving metadata order within each tier. +Unknown preferred nodes or incomplete strip hints preserve metadata order. +This ordering changes only the read snapshot: it neither publishes placement +nor adds or removes replicas. Layout validity deadlines, exact segment +incarnations, checksum verification, and corruption reporting remain unchanged. +Each request uses the exact segment-relative offset and length; DiskIO supports +arbitrary byte ranges. EC reads derive the shard width from stored segment geometry. The fast path reads only data-shard intersections needed by the request. If a required data diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs index 62fdce7ba..5bd9787bf 100644 --- a/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs @@ -23,6 +23,7 @@ mod frame_views; mod framing; mod layout; mod layout_cache; +mod locality; mod location; mod stream; mod verified_frame; @@ -37,6 +38,8 @@ const DEFAULT_STREAM_WINDOW: usize = 1024 * 1024; /// Bounded retry, streaming, and EC-recovery memory policy. #[derive(Debug, Clone)] pub struct ChunkReadPolicy { + /// Prefer this node, then its rack, when retained routing covers a mirror. + pub preferred_node_id: Option, pub stream_window_bytes: usize, pub stream_slots: usize, pub global_stream_bytes: usize, @@ -92,6 +95,7 @@ impl LayoutSnapshot { impl Default for ChunkReadPolicy { fn default() -> Self { Self { + preferred_node_id: None, stream_window_bytes: DEFAULT_STREAM_WINDOW, stream_slots: 3, global_stream_bytes: 256 * 1024 * 1024, @@ -204,9 +208,10 @@ impl ChunkReader { let deadline = started + Duration::from_millis(response.layout_validity_ms) .saturating_sub(self.policy.layout_safety_margin); - let chunk = response + let mut chunk = response .chunk .ok_or_else(|| ReadError::ChunkDeleted(format!("{}:{}", chunk_id.high, chunk_id.low)))?; + self.prefer_local_mirrors(&mut chunk); Ok(LayoutSnapshot { chunk_id, chunk: Arc::new(chunk), diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader/locality.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader/locality.rs new file mode 100644 index 000000000..c92cb56c3 --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader/locality.rs @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Retained topology orders equivalent mirror reads without granting authority. + +use super::ChunkReader; +use crowdb_protocol::chunkdb::rpc::{Chunk, Strip}; +use std::collections::HashMap; + +impl ChunkReader { + pub(super) fn prefer_local_mirrors(&self, chunk: &mut Chunk) { + let Some(node) = self.policy.preferred_node_id else { + return; + }; + let Some(topology) = self.strip_reader.disk_topology() else { + return; + }; + let placements: HashMap<_, _> = topology + .disks() + .map(|disk| { + ( + (disk.disk_id.high, disk.disk_id.low), + (disk.rack_id, disk.node_id), + ) + }) + .collect(); + let Some(rack) = placements + .values() + .find_map(|(rack, owner)| (*owner == node).then_some(*rack)) + else { + return; + }; + if placements + .values() + .any(|(owner_rack, owner_node)| *owner_node == node && *owner_rack != rack) + { + return; + } + for strip in &mut chunk.strips { + let Some(Strip::MirrorStrip(mirror)) = &mut strip.strip else { + continue; + }; + if !mirror.segments.iter().all(|segment| { + segment + .disk_id + .is_some_and(|id| placements.contains_key(&(id.high, id.low))) + }) { + continue; + } + mirror.segments.sort_by_key(|segment| { + let id = segment.disk_id.expect("complete mirror placement hints"); + let (owner_rack, owner_node) = placements[&(id.high, id.low)]; + (owner_node != node, owner_rack != rack) + }); + } + } +} diff --git a/lib/crowdb-chunk-client/src/chunk/strip_reader.rs b/lib/crowdb-chunk-client/src/chunk/strip_reader.rs index 91de3cc04..f583e88f0 100644 --- a/lib/crowdb-chunk-client/src/chunk/strip_reader.rs +++ b/lib/crowdb-chunk-client/src/chunk/strip_reader.rs @@ -35,6 +35,10 @@ pub struct StripReader { } impl StripReader { + pub(crate) fn disk_topology(&self) -> Option { + self.disk_io.disk_topology() + } + pub fn new( disk_io: Arc, recovery_memory: Arc, diff --git a/lib/crowdb-chunk-client/tests/chunk_reader_test.rs b/lib/crowdb-chunk-client/tests/chunk_reader_test.rs index d8c5f2e0d..63f5ebe68 100644 --- a/lib/crowdb-chunk-client/tests/chunk_reader_test.rs +++ b/lib/crowdb-chunk-client/tests/chunk_reader_test.rs @@ -34,6 +34,12 @@ mod frame_revision; #[path = "chunk_reader_test/layout_cache.rs"] mod layout_cache; +#[path = "chunk_reader_test/mirror_locality.rs"] +mod mirror_locality; + +#[path = "common/mirror_locality.rs"] +mod mirror_locality_fixture; + struct MemoryDiskIo { shards: Vec<(DiskId, Bytes)>, failed: Vec, diff --git a/lib/crowdb-chunk-client/tests/chunk_reader_test/mirror_locality.rs b/lib/crowdb-chunk-client/tests/chunk_reader_test/mirror_locality.rs new file mode 100644 index 000000000..5192344b5 --- /dev/null +++ b/lib/crowdb-chunk-client/tests/chunk_reader_test/mirror_locality.rs @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crate::mirror_locality_fixture::{fixture, read}; + +#[tokio::test] +async fn mirror_reads_prefer_the_requested_node_without_changing_metadata() { + let (reader, disks, allocator, frame) = fixture(Some(3), None, None); + read(&reader, &frame).await; + assert_eq!(*disks.served.lock().unwrap(), vec![3]); + assert_eq!( + allocator.current.lock().unwrap().strips[0].strip, + allocator.first.strips[0].strip + ); +} + +#[tokio::test] +async fn unknown_node_or_incomplete_hints_preserve_every_supplied_replica() { + for (preferred, missing) in [(None, None), (Some(99), None), (Some(3), Some(1))] { + let (reader, disks, _, frame) = fixture(preferred, missing, None); + read(&reader, &frame).await; + assert_eq!(*disks.served.lock().unwrap(), vec![1]); + } +} + +#[tokio::test] +async fn corrupt_preferred_mirror_falls_back_within_its_rack_after_reporting() { + let (reader, disks, allocator, frame) = fixture(Some(3), None, Some(3)); + read(&reader, &frame).await; + assert_eq!(*disks.served.lock().unwrap(), vec![3, 2]); + assert_eq!( + allocator.current.lock().unwrap().strips[0].unavailable_segments, + vec![segment(3)] + ); +} diff --git a/lib/crowdb-chunk-client/tests/common/mirror_locality.rs b/lib/crowdb-chunk-client/tests/common/mirror_locality.rs new file mode 100644 index 000000000..c34deb33f --- /dev/null +++ b/lib/crowdb-chunk-client/tests/common/mirror_locality.rs @@ -0,0 +1,124 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_diskio_client::{DiskTopologyView, TestDiskRoute}; + +pub(super) struct TestLocalityDiskIo { + bytes: Bytes, + topology: Option, + pub(super) served: Mutex>, + corrupt: Option, +} + +#[async_trait] +impl DiskWriter for TestLocalityDiskIo { + fn disk_topology(&self) -> Option { + self.topology.clone() + } + + async fn write(&self, _: &Segment, _: u64, _: Bytes) -> Result<()> { + unreachable!("reader test does not write") + } + + async fn write_at_byte_offset(&self, _: &Segment, _: u64, _: u64, _: Bytes) -> Result<()> { + unreachable!("reader test does not write") + } + + async fn read(&self, segment: &Segment, _: u64, offset: u64, length: u32) -> Result { + let id = segment.disk_id.unwrap().low; + self.served.lock().unwrap().push(id); + let offset = usize::try_from(offset).unwrap(); + let length = usize::try_from(length).unwrap(); + let bytes = self.bytes.slice(offset..offset + length); + if self.corrupt == Some(id) { + let mut bytes = bytes.to_vec(); + bytes[crowdb_protocol::frame::FRAME_V2_HEADER_BYTES] ^= 1; + Ok(Bytes::from(bytes)) + } else { + Ok(bytes) + } + } +} + +pub(super) fn fixture( + preferred: Option, + missing: Option, + corrupt: Option, +) -> ( + ChunkReader, + Arc, + Arc, + Bytes, +) { + let id = ChunkId { high: 83, low: 89 }; + let frame = Bytes::from(encode_frame(FrameMagic::RepoSmallV2, id, b"exact payload", 1).unwrap()); + // Metadata deliberately starts in another rack, followed by a sibling + // node in the preferred rack and finally the preferred node itself. + let replicas = vec![segment(1), segment(2), segment(3)]; + let chunk = Chunk { + id: Some(id), + state: ChunkState::Active as i32, + modify_ts: 1, + capacity: 1, + acknowledged_cursor: frame.len() as u64, + strips: vec![ChunkStrip { + capacity: 1, + unit_kb: 1, + strip: Some(Strip::MirrorStrip(MirrorStrip { segments: replicas })), + ..ChunkStrip::default() + }], + ..Chunk::default() + }; + let allocator = Arc::new(SequenceAllocator { + queries: AtomicUsize::new(0), + first: chunk.clone(), + current: Mutex::new(chunk), + first_layout_validity_ms: 1_000, + full_reply: None, + full_calls: AtomicUsize::new(0), + }); + let disks = Arc::new(TestLocalityDiskIo { + bytes: frame.clone(), + topology: Some(DiskTopologyView::from_routes_for_tests( + (1..=3) + .filter(|id| Some(*id) != missing) + .map(|id| TestDiskRoute { + disk_id: crowdb_diskio_client::DiskId::new(0, id), + rack_id: if id == 1 { 2 } else { 1 }, + node_id: id, + disk_group_id: id, + instance_id: id, + endpoint: format!("127.0.0.1:{}", 12000 + id), + }) + .collect(), + )), + served: Mutex::new(Vec::new()), + corrupt, + }); + let reader = ChunkReader::new( + allocator.clone(), + disks.clone(), + ChunkReadPolicy { + preferred_node_id: preferred, + ..ChunkReadPolicy::default() + }, + ) + .unwrap(); + (reader, disks, allocator, frame) +} + +pub(super) async fn read(reader: &ChunkReader, frame: &Bytes) { + assert_eq!( + reader + .read_verified_frame( + ChunkId { high: 83, low: 89 }, + 0, + frame.len() as u64, + FrameMagic::RepoSmallV2, + ) + .await + .unwrap(), + *frame + ); +} From d396a2207e5a13ef61ee1fe04e68ee40b0f8622c Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 31 Oct 2026 19:49:13 +0800 Subject: [PATCH 10/24] Protect leader votes with the current quorum lease --- .../kv/design-crowdb-kv-leader-election.md | 4 + .../src/cluster/local_replica_election.rs | 9 ++- .../tests/election_test/vote_test.rs | 78 +++++++++++++++++++ 3 files changed, 89 insertions(+), 2 deletions(-) diff --git a/doc/design/kv/design-crowdb-kv-leader-election.md b/doc/design/kv/design-crowdb-kv-leader-election.md index 3e4b2b28d..5566f47aa 100644 --- a/doc/design/kv/design-crowdb-kv-leader-election.md +++ b/doc/design/kv/design-crowdb-kv-leader-election.md @@ -215,6 +215,10 @@ While its lease is valid, a leader may serve linearizable reads from local state The first assumption is enforced by: - Acceptors **promise not to vote for any other candidate** for at least `lease_duration` after granting a heartbeat (this is the Raft "PreVote + lease" pattern). +- The leader rejects both PreVote and RequestVote while its quorum-renewed + effective lease remains valid, even after the original self-vote lockout + expires. Renewing this protection uses the existing atomic lease deadline; + a lagging minority cannot recruit the healthy leader as its extra voter. - The leader treats its lease as expired at `lease_grant_time + lease_duration - max_clock_skew` for safety (see §6.3). ### 6.2 Lease grant and renewal diff --git a/lib/crowdb-kv/src/cluster/local_replica_election.rs b/lib/crowdb-kv/src/cluster/local_replica_election.rs index edb113113..65e5a8a2d 100644 --- a/lib/crowdb-kv/src/cluster/local_replica_election.rs +++ b/lib/crowdb-kv/src/cluster/local_replica_election.rs @@ -127,12 +127,16 @@ impl PxLocalReplica { /// - `req.term > current_term` (we'd vote in that term), AND /// - candidate's log is up-to-date, AND /// - `now >= vote_lockout_until`. + /// + /// A leader's quorum-renewed lease also protects its own vote after the + /// original self-vote lockout has expired. pub(super) fn handle_pre_vote(&self, req: VoteRequestPayload) -> VoteReply { let now = Instant::now(); let state = self.election_state.lock(); let granted = req.term > state.current_term && self.candidate_log_up_to_date(&req) - && now >= state.vote_lockout_until; + && now >= state.vote_lockout_until + && (state.role != PxLocalReplicaRole::Leader || self.lease_read_until() <= now); let term = state.current_term; drop(state); let (contiguous_chosen, last_chosen_term, highest_seen_slot) = self.frontier_triple(); @@ -167,7 +171,8 @@ impl PxLocalReplica { let (granted, term) = { let mut state = self.election_state.lock(); - let lockout_ok = now >= state.vote_lockout_until; + let lockout_ok = now >= state.vote_lockout_until + && (state.role != PxLocalReplicaRole::Leader || self.lease_read_until() <= now); let term_ok = req.term > state.current_term || (req.term == state.current_term && state.voted_for.map_or(true, |v| v == req.candidate_id)); diff --git a/lib/crowdb-kv/tests/election_test/vote_test.rs b/lib/crowdb-kv/tests/election_test/vote_test.rs index dbbb1e251..6abb023d2 100644 --- a/lib/crowdb-kv/tests/election_test/vote_test.rs +++ b/lib/crowdb-kv/tests/election_test/vote_test.rs @@ -9,7 +9,85 @@ use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; use crowdb_kv::cluster::replica::{ReplicaHandler, VoteRequestPayload}; +use crowdb_kv::common::config::PxElectionConfig; use crowdb_kv::paxos::roles::{PxBallot, PxLogEntry}; +use std::time::Instant; + +#[tokio::test] +async fn quorum_leased_leader_rejects_prevote_after_its_self_vote_lockout_expires() { + let replica = PxLocalReplica::new(1, PxLocalReplicaRole::Follower); + replica.become_candidate(1); + replica.become_leader(); + // The original self-vote lockout expires during a long healthy tenure; + // quorum heartbeats continue renewing only the lock-free leader lease. + replica.clear_vote_lockout(); + replica.renew_lease(Instant::now(), &PxElectionConfig::DEFAULT); + let request = make_vote_req(2, 2, 0, 0); + let reply = replica.on_pre_vote(request, 1).await.unwrap(); + assert!( + !reply.granted, + "a valid quorum lease must protect the leader's own vote" + ); + assert_eq!(replica.current_term_snapshot(), 1); + assert_eq!(replica.role(), PxLocalReplicaRole::Leader); + + replica.reset_lease_to(Instant::now()); + assert!(replica.on_pre_vote(request, 1).await.unwrap().granted); +} + +#[tokio::test] +async fn quorum_leased_leader_rejects_request_vote_after_its_self_vote_lockout_expires() { + let replica = PxLocalReplica::new(1, PxLocalReplicaRole::Follower); + replica.become_candidate(1); + replica.become_leader(); + replica.clear_vote_lockout(); + replica.renew_lease(Instant::now(), &PxElectionConfig::DEFAULT); + let request = make_vote_req(2, 2, 0, 0); + let reply = replica.on_request_vote(request, 1).await.unwrap(); + assert!( + !reply.granted, + "a challenger must not depose a quorum-leased leader" + ); + assert_eq!(replica.current_term_snapshot(), 1); + assert_eq!(replica.role(), PxLocalReplicaRole::Leader); + + replica.reset_lease_to(Instant::now()); + assert!(replica.on_request_vote(request, 1).await.unwrap().granted); + assert_eq!(replica.current_term_snapshot(), 2); + assert_eq!(replica.role(), PxLocalReplicaRole::Follower); +} + +#[tokio::test] +async fn two_lagging_voters_cannot_use_the_healthy_leader_to_form_a_prevote_quorum() { + let voters: Vec<_> = (1..=5) + .map(|id| PxLocalReplica::new(id, PxLocalReplicaRole::Follower)) + .collect(); + let leader = &voters[0]; + leader.become_candidate(1); + leader.become_leader(); + leader.clear_vote_lockout(); + leader.renew_lease(Instant::now(), &PxElectionConfig::DEFAULT); + for follower in &voters[1..3] { + assert!( + follower + .on_request_vote(make_vote_req(1, 1, 0, 0), 1) + .await + .unwrap() + .granted + ); + } + for lagging in &voters[3..] { + lagging.become_follower(1); + lagging.clear_vote_lockout(); + } + let request = make_vote_req(2, 5, 0, 0); + let mut grants = 1; // Challenger's own pre-vote. + for peer in &voters[..4] { + grants += usize::from(peer.on_pre_vote(request, 1).await.unwrap().granted); + } + assert_eq!(grants, 2, "only the isolated minority may grant this pre-vote"); + assert_eq!(leader.role(), PxLocalReplicaRole::Leader); +} fn make_vote_req(term: u64, candidate_id: u64, tip_slot: u64, tip_term: u64) -> VoteRequestPayload { VoteRequestPayload { From 7a93c4e5fdca9bc39455331bfae9f3082e167a43 Mon Sep 17 00:00:00 2001 From: Gian Date: Sat, 31 Oct 2026 20:06:54 +0800 Subject: [PATCH 11/24] Preserve running clusters during cleanup and record test handoff --- container/oci-image/README.md | 32 +- container/oci-image/clean.py | 43 ++- container/oci-image/docker_garbage.py | 63 +++ container/oci-image/docker_local_cleanup.py | 8 +- container/oci-image/host_cleanup.py | 4 +- container/oci-image/tests/cleanup_cli_test.py | 55 ++- .../oci-image/tests/docker_garbage_test.py | 130 +++++++ .../oci-image/tests/docker_local_test.py | 17 + .../oci-image/tests/host_cleanup_test.py | 14 + container/single-node-container/README.md | 8 +- doc/design/deploy/design-crowdb-oci-image.md | 18 +- doc/working/plan-physical-node-ci-handoff.md | 365 ++++++++++++++++++ pixi.toml | 5 +- 13 files changed, 718 insertions(+), 44 deletions(-) create mode 100644 container/oci-image/docker_garbage.py create mode 100644 container/oci-image/tests/docker_garbage_test.py create mode 100644 doc/working/plan-physical-node-ci-handoff.md diff --git a/container/oci-image/README.md b/container/oci-image/README.md index f752a568e..9d634992d 100644 --- a/container/oci-image/README.md +++ b/container/oci-image/README.md @@ -110,6 +110,7 @@ pixi run oci-run-1c --cluster demo pixi run oci-run-3c --cluster demo-three --port-base 9190 pixi run oci-stop --cluster demo-three pixi run oci-run-3c --cluster demo-three +pixi run oci-stop --cluster demo-three pixi run oci-clean --cluster demo-three --keep-data # Recreate containers and bridge with the original volumes and accepted policy: pixi run oci-run-3c --cluster demo-three @@ -123,28 +124,39 @@ A different node count or conflicting input requires another cluster name. Ports for each node are UI/S3/Iceberg/Dataset at base/base+1/base+2/base+3; subsequent members add 10 and 20. All published ports bind to `127.0.0.1`. Run and stop default to `local`; stop accepts `--all` for every recorded local -cluster. Clean defaults to every recorded local and host-network cluster and -deletes their node data; use `--cluster NAME` to clean only one. Explicit `--all` -also selects every recorded cluster. Use `--keep-data` to preserve data and -restart receipts; `--delete-data` explicitly selects the default deletion mode. +cluster. Default clean skips running recorded local and host-network clusters, +including all resources of a bridge cluster with any running member. It removes +stopped clusters and their node data, then cleans unused resources on the selected +Docker daemon, including containers without CROWDB receipts: ```bash pixi run oci-clean ``` -With no cluster receipts, clean reports that there are no recorded clusters and -checks for dangling CROWDB images if Docker is installed. An explicitly named -cluster without a receipt remains an error. Containerd-only host cleanup needs -no Docker daemon. +- Remove stopped containers first, then unused networks and both named and + anonymous volumes, followed by unused Docker build cache. Running containers + and referenced resources remain intact. Docker removals never force active + containers or in-use resources; a concurrent new reference makes removal fail. +- Preserve resources belonging to skipped running clusters and their restart + receipts, including detached volumes and stopped members of those clusters. +- Use `--keep-data` to preserve all Docker volumes, node data and restart receipts; + `--delete-data` explicitly selects the default deletion mode. +- `--cluster NAME` explicitly stops and removes one recorded cluster, without + daemon-wide garbage collection. `--all` explicitly stops and removes every + recorded cluster and also cleans unused Docker resources. +- With no cluster receipts, unused Docker resource cleanup still runs if Docker + is installed. An explicitly named cluster without a receipt remains an error. + Containerd-only host cleanup needs no Docker daemon. Host receipts live in `.crowdb-runtime/persistent/oci-clusters/`. The shortcuts verify the Docker daemon/context and exact ownership labels before effects. -Clean stops and removes owned containers, unused owned bridges and owned data +Clean removes selected stopped containers, unused owned bridges and owned data volumes. It also removes the launch receipt and matching optional CLI target binding. `--keep-data` retains volumes and restart inputs. Image cleanup only considers dangling CROWDB images, preserving retained cluster references and images used by any running or stopped container. Attached or relabelled -networks and volumes referenced elsewhere are rejected. No global prune runs. +networks and volumes referenced elsewhere are rejected. Daemon-wide garbage +collection runs only when no `--cluster` is selected. These host scripts invoke Docker directly and require no CLI binary or application target binding. A binding with different ownership is rejected before data deletion. diff --git a/container/oci-image/clean.py b/container/oci-image/clean.py index 9a313d287..3eef66c41 100644 --- a/container/oci-image/clean.py +++ b/container/oci-image/clean.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. -"""Clean recorded Docker and containerd clusters and their data by default.""" +"""Clean stopped clusters and unused Docker resources; preserve running clusters.""" import argparse from pathlib import Path @@ -9,6 +9,7 @@ import sys from docker_local_cleanup import clean, image_garbage, selected +from docker_garbage import collect from docker_local_state import Docker, RECORDS, check_daemon, load from host_cleanup import check_data_users, clean as clean_host, selected_records from host_data_cleanup import HostDataCleanup @@ -18,13 +19,16 @@ def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) scope = parser.add_mutually_exclusive_group() - scope.add_argument('--cluster', help='clean only this recorded cluster') - scope.add_argument('--all', action='store_true', help='clean every recorded cluster (default)') + scope.add_argument('--cluster', help='stop and clean only this recorded cluster') + scope.add_argument('--all', action='store_true', + help='stop and clean every recorded cluster, then clean unused Docker resources') data = parser.add_mutually_exclusive_group() data.add_argument('--keep-data', action='store_true', help='preserve data and restart receipts') - data.add_argument('--delete-data', action='store_true', help='delete owned data (default)') + data.add_argument('--delete-data', action='store_true', + help='delete stopped-cluster data and unused Docker volumes (default)') parser.add_argument('--runtime-root', type=Path, default=RECORDS.parents[1]) args = parser.parse_args(argv) + stop_running = args.cluster is not None or args.all directory = args.runtime_root / 'persistent/oci-clusters' if args.cluster is None: records = selected(None, True, directory) @@ -40,20 +44,39 @@ def main(argv=None): if docker is not None: for record in records: check_daemon(docker, record) + runtime_identity = docker.identity() if docker is not None else None + retained = [] + if not stop_running and docker is not None: + stopped = [] + for record in records: + if any(item['State']['Running'] for item in docker.owned(record, 'container')): + print(f"Skipped running cluster {record['cluster']}") + retained.append(record) + else: + stopped.append(record) + records = stopped plans = [] + stopped_hosts = [] + retained_hosts = [] for record in hosts: runtime = HostRuntime(record['options'], record['cluster']) runtime.check(record) current = runtime.named(record['name']) if current is not None: require_owned(current, record) + if not stop_running and current['State']['Running']: + print(f"Skipped running host cluster {record['cluster']}") + retained_hosts.append(record) + continue if not args.keep_data: check_data_users(runtime, record) plans.append(None if args.keep_data else HostDataCleanup(record, args.runtime_root)) + stopped_hosts.append(record) + hosts = stopped_hosts for record in records: - clean(docker, record, not args.keep_data, directory) + clean(docker, record, not args.keep_data, directory, stop_running=stop_running) for record, plan in zip(hosts, plans): - clean_host(record) + clean_host(record, stop_running=stop_running) if plan is not None: runtime = HostRuntime(record['options'], record['cluster']) runtime.check(record) @@ -62,10 +85,14 @@ def main(argv=None): print(f"Cleaned host container for {record['cluster']}; " + ('preserved data and restart receipts' if args.keep_data else 'deleted node data and receipts; preserved supplied disk files and devices')) - if not records and not hosts: + if not records and not hosts and not retained and not retained_hosts: print('No recorded CROWDB clusters to clean') if docker is not None: - image_garbage(docker, directory, runtime=records[0]['runtime'] if records else docker.identity()) + if args.cluster is None: + collect(docker, retained + (records if args.keep_data else []), + retained_hosts + (hosts if args.keep_data else []), + delete_data=not args.keep_data, runtime=runtime_identity) + image_garbage(docker, directory, runtime=runtime_identity) else: print('No local Docker runtime selected; skipped dangling Docker image cleanup') diff --git a/container/oci-image/docker_garbage.py b/container/oci-image/docker_garbage.py new file mode 100644 index 000000000..9206f76a1 --- /dev/null +++ b/container/oci-image/docker_garbage.py @@ -0,0 +1,63 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Reclaim unused Docker resources without stopping containers or retained clusters.""" + +import json + +from docker_local_state import OWNER, labels, resource_names +from runtime import OWNER as HOST_OWNER + + +def collect(docker, records, hosts, *, delete_data=True, runtime): + """Keep retained receipts and let non-force Docker removals guard reference races.""" + owners = {record['owner'] for record in (*records, *hosts)} + networks = {name for record in records for name in resource_names(record, 'network')} + volumes = {name for record in records for name in resource_names(record, 'volume')} + + def retained(item, kind): + metadata = labels(item, kind) + return any(metadata.get(key) in owners for key in (OWNER, HOST_OWNER)) + + def check_runtime(): + if docker.identity() != runtime: + raise ValueError('Docker endpoint/context changed during garbage cleanup') + + def remove(kind, identity): + check_runtime() + # Never force removal: Docker rejects containers that restarted and + # volumes/networks that acquired a reference since inspection. + print(f'Removing unused Docker {kind}: {identity}', flush=True) + result = docker.call('rm', identity) if kind == 'container' else docker.call(kind, 'rm', identity) + if result: + print(result, flush=True) + + check_runtime() + for candidate in docker.objects('container'): + if retained(candidate, 'container'): + continue + item = json.loads(docker.call('container', 'inspect', candidate['Id']))[0] + if item['State'].get('Status') not in ('created', 'exited', 'dead'): + continue + if item['State']['Running'] or retained(item, 'container'): + continue + remove('container', item['Id']) + + for kind, protected in (('network', networks), ('volume', volumes)): + if kind == 'volume' and not delete_data: + print('Preserved unused Docker volumes (--keep-data)', flush=True) + continue + for candidate in docker.objects(kind, ('dangling=true',)): + identity = candidate['Name'] if kind == 'volume' else candidate['Id'] + item = json.loads(docker.call(kind, 'inspect', identity))[0] + if item['Name'] in protected or retained(item, kind): + continue + if kind == 'network': + if item.get('Containers') or item['Name'] in ('bridge', 'host', 'none'): + continue + elif docker.call('ps', '-aq', '--filter', 'volume=' + item['Name']): + continue + remove(kind, identity) + + check_runtime() + print('Pruning unused Docker build cache', flush=True) + print(docker.call('builder', 'prune', '--all', '--force'), flush=True) diff --git a/container/oci-image/docker_local_cleanup.py b/container/oci-image/docker_local_cleanup.py index c86d78078..d3f5706b4 100644 --- a/container/oci-image/docker_local_cleanup.py +++ b/container/oci-image/docker_local_cleanup.py @@ -36,9 +36,13 @@ def stop(docker, record): docker.call("stop", "--time", "20", item["Id"]) -def clean(docker, record, delete_data=False, directory=RECORDS): +def clean(docker, record, delete_data=False, directory=RECORDS, *, stop_running=True): binding = owned_binding(record, directory) if delete_data else None - stop(docker, record) + check_daemon(docker, record) + if stop_running: + stop(docker, record) + elif any(item['State']['Running'] for item in docker.owned(record, 'container')): + raise ValueError('cluster is running; refusing cleanup') for item in docker.owned(record, "container"): item = current(docker, record, "container", item["Id"]) if item["State"]["Running"]: diff --git a/container/oci-image/host_cleanup.py b/container/oci-image/host_cleanup.py index 88c1b4a34..ef95de4b8 100644 --- a/container/oci-image/host_cleanup.py +++ b/container/oci-image/host_cleanup.py @@ -68,7 +68,7 @@ def selected_records(cluster, root): return [selected(path.stem, root) for path in sorted(directory.glob('*.json'))] -def clean(record): +def clean(record, *, stop_running=True): runtime = HostRuntime(record['options'], record['cluster']) runtime.check(record) current = runtime.named(record['name']) @@ -77,6 +77,8 @@ def clean(record): require_owned(current, record) identity = current['Id'] if current['State']['Running']: + if not stop_running: + raise ValueError('host container is running; refusing cleanup') runtime.check(record) runtime.invoke('stop', identity) current = runtime.inspect(identity) diff --git a/container/oci-image/tests/cleanup_cli_test.py b/container/oci-image/tests/cleanup_cli_test.py index 9b2eeaf32..267bb2a77 100644 --- a/container/oci-image/tests/cleanup_cli_test.py +++ b/container/oci-image/tests/cleanup_cli_test.py @@ -1,6 +1,6 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. -"""Clean command defaults select all receipts and remove owned data.""" +"""Default cleanup preserves running clusters and reclaims unused Docker resources.""" import contextlib import importlib.util @@ -37,20 +37,26 @@ def setUp(self): patch.object(cleanup, 'HostRuntime'), patch.object(cleanup, 'HostDataCleanup'), patch.object(cleanup, 'clean_host'), patch.object(cleanup, 'require_owned'), patch.object(cleanup.shutil, 'which', return_value='/docker'), - patch.object(cleanup, 'check_data_users')] + patch.object(cleanup, 'check_data_users'), patch.object(cleanup, 'collect')] (self.factory, self.select, self.check, self.clean, self.select_hosts, self.garbage, - self.load, self.runtime, self.data, self.clean_host, self.require, self.which, self.users) = [ + self.load, self.runtime, self.data, self.clean_host, self.require, self.which, self.users, + self.collect) = [ item.start() for item in self.patches] + self.runtime.return_value.named.return_value = None self.addCleanup(lambda: [item.stop() for item in reversed(self.patches)]) def test_bare_clean_selects_all_and_deletes_data_before_image_garbage(self): cleanup.main([]) self.select.assert_called_once_with(None, True, cleanup.RECORDS) self.assertEqual(self.clean.call_args_list, - [call(self.docker, record, True, cleanup.RECORDS) for record in self.records]) - self.clean_host.assert_called_once_with(self.hosts[0]) + [call(self.docker, record, True, cleanup.RECORDS, stop_running=False) + for record in self.records]) + self.clean_host.assert_called_once_with(self.hosts[0], stop_running=False) self.data.return_value.remove.assert_called_once_with() - self.garbage.assert_called_once_with(self.docker, cleanup.RECORDS, runtime=self.records[0]['runtime']) + self.collect.assert_called_once_with(self.docker, [], [], delete_data=True, + runtime=self.docker.identity.return_value) + self.garbage.assert_called_once_with(self.docker, cleanup.RECORDS, + runtime=self.docker.identity.return_value) def test_explicit_cluster_and_keep_data(self): self.load.return_value = self.records[0] @@ -59,14 +65,17 @@ def test_explicit_cluster_and_keep_data(self): self.select.assert_not_called() self.load.assert_called_once_with('first', cleanup.RECORDS) self.select_hosts.assert_called_once_with('first', cleanup.RECORDS.parents[1]) - self.clean.assert_called_once_with(self.docker, self.records[0], False, cleanup.RECORDS) + self.clean.assert_called_once_with(self.docker, self.records[0], False, cleanup.RECORDS, + stop_running=True) self.data.assert_not_called() + self.collect.assert_not_called() def test_explicit_all_and_delete_data(self): cleanup.main(['--all', '--delete-data']) self.select.assert_called_once_with(None, True, cleanup.RECORDS) self.assertEqual(self.clean.call_args_list, - [call(self.docker, record, True, cleanup.RECORDS) for record in self.records]) + [call(self.docker, record, True, cleanup.RECORDS, stop_running=True) + for record in self.records]) def test_empty_inventory_still_checks_image_garbage(self): self.select.return_value = [] @@ -84,13 +93,13 @@ def test_containerd_only_cleanup_needs_no_docker_daemon(self): cleanup.main([]) self.factory.assert_not_called() self.garbage.assert_not_called() - self.clean_host.assert_called_once_with(self.hosts[0]) + self.clean_host.assert_called_once_with(self.hosts[0], stop_running=False) self.data.return_value.remove.assert_called_once_with() def test_host_keep_data_preserves_restart_receipts(self): self.load.return_value = None cleanup.main(['--cluster', 'host', '--keep-data']) - self.clean_host.assert_called_once_with(self.hosts[0]) + self.clean_host.assert_called_once_with(self.hosts[0], stop_running=True) self.data.assert_not_called() def test_host_docker_endpoint_does_not_trigger_image_cleanup_on_current_docker_context(self): @@ -98,7 +107,7 @@ def test_host_docker_endpoint_does_not_trigger_image_cleanup_on_current_docker_c self.hosts[0]['options']['runtime'] = 'docker' cleanup.main([]) self.factory.assert_not_called() - self.clean_host.assert_called_once_with(self.hosts[0]) + self.clean_host.assert_called_once_with(self.hosts[0], stop_running=False) self.garbage.assert_not_called() def test_missing_explicit_cluster_is_an_error_before_runtime_access(self): @@ -109,6 +118,30 @@ def test_missing_explicit_cluster_is_an_error_before_runtime_access(self): self.factory.assert_not_called() self.runtime.assert_not_called() + def test_default_retains_running_cluster_and_all_its_resources(self): + self.docker.owned.side_effect = [[dict(State=dict(Running=True))], []] + self.select_hosts.return_value = [] + cleanup.main([]) + self.clean.assert_called_once_with(self.docker, self.records[1], True, cleanup.RECORDS, + stop_running=False) + self.collect.assert_called_once_with(self.docker, self.records[:1], [], delete_data=True, + runtime=self.docker.identity.return_value) + + def test_default_retains_running_host_data_without_planning_deletion(self): + self.runtime.return_value.named.return_value = dict(State=dict(Running=True)) + cleanup.main([]) + self.clean_host.assert_not_called() + self.data.assert_not_called() + self.users.assert_not_called() + self.collect.assert_called_once_with(self.docker, [], self.hosts, delete_data=True, + runtime=self.docker.identity.return_value) + + def test_default_keep_data_protects_receipts_from_global_collection(self): + cleanup.main(['--keep-data']) + self.data.assert_not_called() + self.collect.assert_called_once_with(self.docker, self.records, self.hosts, delete_data=False, + runtime=self.docker.identity.return_value) + def test_all_runtime_identities_are_checked_before_cleanup(self): self.check.side_effect = [None, ValueError('changed daemon')] with self.assertRaisesRegex(ValueError, 'changed daemon'): diff --git a/container/oci-image/tests/docker_garbage_test.py b/container/oci-image/tests/docker_garbage_test.py new file mode 100644 index 000000000..c665a08ae --- /dev/null +++ b/container/oci-image/tests/docker_garbage_test.py @@ -0,0 +1,130 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Garbage cleanup preserves live references and retained cluster resources.""" + +import contextlib +import io +import json +from pathlib import Path +import sys +import unittest +from unittest.mock import MagicMock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from docker_garbage import collect +from docker_local_state import OWNER +from runtime import OWNER as HOST_OWNER + + +class DockerGarbageTest(unittest.TestCase): + def setUp(self): + self.docker = MagicMock() + self.runtime = dict(daemon='local', context='default', endpoint='unix:///docker.sock') + self.docker.identity.return_value = self.runtime + self.items = dict(container={}, network={}, volume={}) + self.used_volumes = set() + self.removed = [] + self.docker.objects.side_effect = lambda kind, *args: list(self.items[kind].values()) + self.docker.call.side_effect = self.call + + def call(self, *args): + if args[1] == 'inspect': + return json.dumps([self.items[args[0]][args[2]]]) + if args[:2] == ('ps', '-aq'): + return 'live-user' if args[-1].removeprefix('volume=') in self.used_volumes else '' + if args[:2] == ('builder', 'prune'): + return 'Total reclaimed space: 0B' + self.removed.append(args) + return '' + + def container(self, identity, status='exited', owner=None): + item = dict(Id=identity, State=dict(Running=status in ('running', 'paused', 'restarting'), + Status=status), Config=dict(Labels={OWNER: owner})) + self.items['container'][identity] = item + return item + + def volume(self, name, owner=None): + self.items['volume'][name] = dict(Name=name, Labels={OWNER: owner}) + + def network(self, name, attached=False, owner=None): + self.items['network'][name] = dict(Id=name, Name=name, Labels={OWNER: owner}, + Containers=dict(live={}) if attached else {}) + + def collect(self, records=(), hosts=(), **kwargs): + with contextlib.redirect_stdout(io.StringIO()): + collect(self.docker, records, hosts, runtime=self.runtime, **kwargs) + + def test_removes_stopped_containers_before_unused_named_and_anonymous_volumes(self): + for status in ('created', 'exited', 'dead', 'running', 'paused', 'restarting'): + self.container(status, status) + self.volume('anonymous') + self.volume('named-data') + self.volume('running-data') + self.used_volumes.add('running-data') + self.network('old-bridge') + self.network('live-bridge', attached=True) + self.collect() + self.assertEqual(self.removed, [('rm', 'created'), ('rm', 'exited'), ('rm', 'dead'), + ('network', 'rm', 'old-bridge'), ('volume', 'rm', 'anonymous'), + ('volume', 'rm', 'named-data')]) + self.docker.call.assert_any_call('builder', 'prune', '--all', '--force') + self.assertFalse(any(item.args[0] in ('stop', 'kill') for item in self.docker.call.call_args_list)) + + def test_retained_cluster_keeps_stopped_members_and_detached_resources(self): + records = [dict(cluster=name, owner=name, nodes=1) for name in ('first', 'second')] + for record in records: + self.container(record['cluster'], owner=record['owner']) + self.volume('crowdb-' + record['cluster'] + '-1-data') + self.volume('extra-' + record['cluster'], owner=record['owner']) + self.network('crowdb-' + record['cluster']) + host = self.container('host-stopped') + host['Config']['Labels'] = {HOST_OWNER: 'host-owner'} + self.collect(records, [dict(owner='host-owner')]) + self.assertEqual(self.removed, []) + + def test_keep_data_preserves_all_volumes_but_removes_stopped_containers(self): + self.container('old') + self.volume('unused-data') + self.collect(delete_data=False) + self.assertEqual(self.removed, [('rm', 'old')]) + self.assertFalse(any(item.args[0] == 'volume' for item in self.docker.call.call_args_list)) + + def test_default_networks_survive_even_if_reported_as_unused(self): + for name in ('bridge', 'host', 'none'): + self.network(name) + self.collect() + self.assertEqual(self.removed, []) + + def test_restarted_container_is_skipped_after_fresh_inspection(self): + old = self.container('restarted') + self.docker.objects.side_effect = lambda kind, *args: [old] if kind == 'container' else [] + self.items['container']['restarted'] = dict(old, State=dict(Running=True, Status='running')) + self.collect() + self.assertEqual(self.removed, []) + + def test_docker_reference_race_failure_is_reported_without_force_or_cache_prune(self): + self.volume('newly-used') + original = self.call + + def call(*args): + if args[:2] == ('volume', 'rm'): + raise RuntimeError('volume is in use') + return original(*args) + + self.docker.call.side_effect = call + with self.assertRaisesRegex(RuntimeError, 'volume is in use'): + self.collect() + self.docker.call.assert_any_call('volume', 'rm', 'newly-used') + self.assertFalse(any(item.args[0] == 'builder' for item in self.docker.call.call_args_list)) + + def test_context_change_before_removal_has_no_effects(self): + self.container('old') + self.docker.identity.side_effect = [self.runtime, dict(daemon='foreign')] + with self.assertRaisesRegex(ValueError, 'endpoint/context changed'): + self.collect() + self.assertEqual(self.removed, []) + self.assertFalse(any(item.args[0] == 'builder' for item in self.docker.call.call_args_list)) + + +if __name__ == '__main__': + unittest.main() diff --git a/container/oci-image/tests/docker_local_test.py b/container/oci-image/tests/docker_local_test.py index f1ea14eb5..858062fcd 100644 --- a/container/oci-image/tests/docker_local_test.py +++ b/container/oci-image/tests/docker_local_test.py @@ -86,6 +86,23 @@ def test_changed_daemon_rejects_before_cleanup_effects(self): clean(self.docker, self.record) self.docker.call.assert_not_called() + def test_running_cluster_is_rejected_before_any_effect_in_stopped_only_mode(self): + self.docker.owned.return_value = [dict(State=dict(Running=False)), dict(State=dict(Running=True))] + with self.assertRaisesRegex(ValueError, 'cluster is running'): + clean(self.docker, self.record, stop_running=False) + self.docker.call.assert_not_called() + + def test_container_restarted_during_cleanup_is_never_stopped(self): + stopped = dict(Id='owned', State=dict(Running=False)) + restarted = dict(stopped, Name='/crowdb-first-1', Config=dict(Labels=self.labels), + State=dict(Running=True)) + self.docker.owned.side_effect = [[stopped], [stopped]] + self.docker.call.return_value = json.dumps([restarted]) + with self.assertRaisesRegex(ValueError, 'container restarted'): + clean(self.docker, self.record, stop_running=False) + self.assertEqual([item.args for item in self.docker.call.call_args_list], + [('container', 'inspect', 'owned')]) + def test_delete_data_rejects_a_volume_referenced_by_another_container(self): volume = {"Name": "crowdb-first-1-data", "Labels": self.labels} self.docker.owned.side_effect = lambda record, kind: [volume] if kind == "volume" else [] diff --git a/container/oci-image/tests/host_cleanup_test.py b/container/oci-image/tests/host_cleanup_test.py index 883a2b978..3ce6e01ed 100644 --- a/container/oci-image/tests/host_cleanup_test.py +++ b/container/oci-image/tests/host_cleanup_test.py @@ -20,6 +20,20 @@ def test_exact_owner_is_stopped_then_removed_without_data_effects(self): clean(self.record) self.assertEqual(self.runtime.effects, [('stop', 'owned-container'), ('rm', 'owned-container')]) + def test_stopped_only_cleanup_never_stops_a_running_host(self): + self.runtime.current = fixture.container(self.record) + with patch('host_cleanup.HostRuntime', return_value=self.runtime): + with self.assertRaisesRegex(ValueError, 'host container is running'): + clean(self.record, stop_running=False) + self.assertEqual(self.runtime.effects, []) + + def test_stopped_only_cleanup_removes_an_exited_host(self): + self.runtime.current = fixture.container(self.record) + self.runtime.current['State']['Running'] = False + with patch('host_cleanup.HostRuntime', return_value=self.runtime): + clean(self.record, stop_running=False) + self.assertEqual(self.runtime.effects, [('rm', 'owned-container')]) + def test_foreign_owner_or_daemon_never_receives_effects(self): for changed_daemon in (False, True): self.runtime.current = fixture.container(self.record) diff --git a/container/single-node-container/README.md b/container/single-node-container/README.md index 403ac9543..b0ba77b41 100644 --- a/container/single-node-container/README.md +++ b/container/single-node-container/README.md @@ -61,9 +61,11 @@ remain independent of public origins. All members of one local cluster share the accepted client credential set. Retain each member's own data volume across stop and image replacement. -Default `oci-clean` removes all recorded clusters, their unused owned bridges -and owned data. Use `--cluster NAME` to select one cluster and `--keep-data` to -retain its volumes and restart receipts. +Default `oci-clean` preserves running clusters and removes stopped clusters, +their owned data, and unused Docker containers, networks, volumes and build +cache. Use `--cluster NAME` to explicitly stop and remove one cluster, or +`--all` to stop and remove all recorded clusters. Use `--keep-data` to retain +volumes and restart receipts. See the [cleanup contract](../oci-image/README.md#local-cluster-shortcuts). Ordinary `docker compose down` also preserves its named volumes. ## Tested S3 client recipes diff --git a/doc/design/deploy/design-crowdb-oci-image.md b/doc/design/deploy/design-crowdb-oci-image.md index 4435269cd..a216a22d8 100644 --- a/doc/design/deploy/design-crowdb-oci-image.md +++ b/doc/design/deploy/design-crowdb-oci-image.md @@ -304,18 +304,22 @@ file hashes. Resume accepts the same inputs and exact owned container, or recreates that container against the retained data. A foreign same-name object, changed daemon or changed configuration fails before start/stop effects. Application UUID binding is separate and can be sealed only after observing -that cluster. One cleanup entry point selects all recorded bridge and host-network -clusters by default, or one explicitly named cluster. Cleanup verifies the same -ownership and removes containers, unused owned bridges, node data and matching -launch/application receipts. Bridge data is held in owned Docker volumes. Host -data deletion requires the retained storage declaration to match accepted media +that cluster. Default cleanup selects stopped recorded bridge and host-network +clusters and preserves a running cluster's containers, data and restart receipts. +Explicit cluster or all-cluster teardown can stop the selected containers. +Cleanup verifies the same ownership and removes containers, unused owned bridges, +node data and matching launch/application receipts. Bridge data is held in owned +Docker volumes. Host data deletion requires the retained storage declaration to match accepted media and rejects symlinked roots, nested mounts, overlapping recorded data directories and bind mounts referenced by another container in the selected runtime. Supplied disk files, raw devices and external policy/password inputs are preserved; cleanup never wipes their contents. Retaining data and restart receipts is an explicit cleanup option. Failures leave ownership retryable until data deletion -and target-binding removal complete. Containerd-only cleanup requires no Docker -daemon or CLI binary. +and target-binding removal complete. Unscoped Docker cleanup also removes stopped +containers without receipts, unreferenced named and anonymous volumes, unused +networks and unused build cache. Retained cluster resources remain protected; +non-force runtime removal rejects concurrent restarts or new references. +Containerd-only cleanup requires no Docker daemon or CLI binary. Explicit monitor liveness/readiness probes report node health. A dedicated systemd unit restarts the containerd daemon, and its restart plugin applies the diff --git a/doc/working/plan-physical-node-ci-handoff.md b/doc/working/plan-physical-node-ci-handoff.md new file mode 100644 index 000000000..b22f8c25e --- /dev/null +++ b/doc/working/plan-physical-node-ci-handoff.md @@ -0,0 +1,365 @@ + + + +# Physical Node and CI Handoff Plan + +Upstream: [Physical Node Test Plan](../dev/physical-node-test-plan.md), +[OCI image operations](../../container/oci-image/README.md), and +[container acceptance](../../e2e-test/README.md). + +Goal: validate the committed recovery fixes with a fresh five-container run +across two physical hosts, including 30 continuous minutes of active load, +the recovery matrix, and the remaining CI cases. + +This is a temporary handoff, updated on 2026-10-31. Delete it after the remaining +work is complete and the final report is retained in the new run directory. + +## 1. Read this first + +- The user deliberately deleted `.crowdb-runtime` and instructed us to retest + next time. The old physical run is interrupted, not an acceptance pass. +- **The full 30-minute workload has not completed.** The formal final workload + had not started at the last recorded checkpoint. Earlier short runs do not + count toward the requested duration. +- **A complete Iceberg SF1 load has not passed.** The latest election fix still + needs physical load verification. Passing unit tests and retained-volume + startup do not establish that the original load failure is fixed. +- The deleted directory contained harness scripts, wheels, logs, ledgers, + image receipts, and controller state. Historical paths below are identifiers, + not available evidence. Reconstruct a fresh harness from current interfaces; + do not launch commands that depend on those deleted files. +- The loader's S3/Dataset implementation is committed and pushed. Its later + 0.1.2 filename changes and all CROWDB changes are committed locally. + **Do not push CROWDB without new user authorization.** No further push was + requested when committing the remaining changes. +- No fresh run was started for this handoff. Keep new artifacts under + `.crowdb-runtime`, as requested by the user. + +## 2. Repositories and publication boundaries + +- **CROWDB**: `/cjdata/cpp/crowdb2`, branch `task-fix-build`. + The recovery checkpoint ends at + `d396a2207e5a13ef61ee1fe04e68ee40b0f8622c`. + These three local commits contain the recovery work: + - `cdf023d642da40f6b2db03c54b3f1a4fdc5fd049`: + `Reduce recovery memory and harden storage liveness`. + - `81d07f03398aaa144e8dcec1187bff0a0d18f91b`: + `Prefer nearby mirrors during partition recovery`. + - `d396a2207e5a13ef61ee1fe04e68ee40b0f8622c`: + `Protect leader votes with the current quorum lease`. + The follow-up commit containing this handoff also records the remaining + OCI cleanup changes. Do not push this repository. +- **TPC loader**: `/cjdata/cpp/crowdb-tpc-loader`, branch `main`. + Current HEAD is `447a1e619550bbdc0150139976e0c442675f92b0`, + `Use readable table-scoped Parquet names for version 0.1.2`. + This follow-up is local only. The local `origin/main` tracking ref remains at + `4cafdc9227871f55942a2b215ccb5a3078ade70d`, + `Add standalone S3 and synthetic Dataset loading`. + This was pushed, including its ancestor `ead5449`, which publishes native + Iceberg upload locations before manifest registration. + [Loader CI run 41194754281](https://github.com/buzzcrow/crowdb-tpc-loader/actions/runs/41194754281) + was recorded successful for Python 3.10, 3.11, and 3.12. +- **Website**: `/cjdata/cpp/crowdb-web`, branch `main`. + `c9306c51c71dd3b71a4daf9255692da53f0f3544`, + `Document standalone S3 and Dataset loader commands`, is local only. + Do not publish it without authorization. Never read or stage its untracked + `.aws` directory. + +The remaining changes were committed at the user's request; no additional +repository was pushed. + +## 3. Physical environment and remaining resources + +- Host A: `192.168.31.129`, previously using interface `wlp37s0`. + Nodes A1/A2/A3 use port offsets 0/1/2. +- Host B: `192.168.31.184`, previously using interface `wlp4s0`. + Nodes B1/B2 use port offsets 0/1. +- Verify addresses, interfaces, host identity, clocks, and ownership again. + These are two physical failure domains and five container voters. +- Existing Ed25519 SSH access works noninteractively. A fresh `BatchMode` + connection to B succeeded during handoff preparation. Use private local + SSH configuration; do not put login accounts or credentials in documents. +- The previous containers used host networking, `1 CPU / 2048 MiB` each, + one Tokio worker per service, five voters, Groups 0/1/3, and 35 managed + services. Test media was 16 GiB per node with a 1 GiB reserve. +- Old run ID and ownership label: + `physical-20261031T024347Z` and + `org.crowdb.e2e.run=physical-20261031T024347Z`. +- Deleting `.crowdb-runtime` did not remove Docker resources. At handoff, + the three A containers still existed and were running, with the expected + ownership label, CPU/memory limits, and no container-level `OOMKilled` flag. + B containers were not freshly inventoried during this handoff; their last + recorded state was running the same upgraded image. +- The old physical controller exited with status 1. The orphaned CI controller, + which was waiting for workload ledgers that no longer exist, was stopped with + `SIGTERM`; its exit was verified. Do not resume either controller or rely on + old session IDs. +- No persistent volumes were removed for this handoff. Cleanup is unfinished. + Inventory both hosts before stopping, replacing, or deleting any resource. + Do not use Docker-wide prune or remove another task's containers/volumes. + +Read-only starting commands, from the CROWDB root: + +```sh +git status --short +git log -4 --oneline +pixi run docker ps -a --filter label=org.crowdb.e2e.run=physical-20261031T024347Z +pixi run ssh -o BatchMode=yes -o IdentitiesOnly=yes -o ConnectTimeout=5 192.168.31.184 true +``` + +For remote executable/test commands, use remote Pixi with an explicitly verified +project manifest or working directory. A bare remote `pixi run` from the login +directory failed because there was no Pixi manifest there; this was not an SSH +authentication failure. Do not guess the old remote staging directory, whose +local receipt was deleted. + +## 4. Latest root cause: healthy leader votes away its quorum + +The last failed physical Iceberg SF1 run did not report OOM. Six tables committed: +`region`, `nation`, `supplier`, `customer`, `part`, and `partsupp`. +The `orders` data upload completed, but a manifest `UploadPart` returned +`INTERNAL_FAILURE`, leaving its outcome uncertain. `lineitem` failed during +native upload with HTTP 500. These outcomes must not be counted as successful +table commits or blindly replayed. + +The first divergence was in Root Group 0 election handling: + +- A3 was the leader with a current quorum lease and a responsive A1/A2 majority. + The two B voters lagged under load and repeatedly attempted elections. +- Around 11:37 UTC, a B challenger obtained a vote from the healthy leader. + Leadership changed, recovery scanned a backlog of roughly 7,000 slots, + process leases expired, and a native file-block intent reached its ChunkKV + operation deadline. +- `handle_pre_vote` and `handle_request_vote` checked the old self-vote lockout. + That lockout can expire during a long leader tenure even while heartbeat + quorum acknowledgments keep the read lease current. +- Commit `d396a220` rejects both vote paths while the replica is Leader and + `lease_read_until() > now`. Normal voting resumes when that lease expires. + It uses the existing lease and vote-handler synchronization; no new hot-path + lock, caller retry, or larger deadline was introduced. + +Relevant files: + +- `lib/crowdb-kv/src/cluster/local_replica_election.rs`. +- `lib/crowdb-kv/tests/election_test/vote_test.rs`. +- `doc/design/kv/design-crowdb-kv-leader-election.md`, section 6.1. + +Recorded verification before runtime deletion: + +- Both new leased-leader vote tests failed before the fix and passed afterward. +- The final election suite passed 79 cases, including the five-voter case where + two lagging voters cannot obtain the healthy leader's third pre-vote. +- Affected KV/server tests passed 733 cases across 31 test binaries. +- Rust formatting and clippy with warnings denied passed. +- The OCI build passed. All five retained-volume nodes were upgraded, and + 35-service readiness passed in 47.582 seconds under the original 180-second + gate; identity restoration took 2.355 seconds. +- The controller had started read-only reconciliation of prior Iceberg failures + when last observed. No subsequent fresh SF1 success was recorded. + +The locally inspected running A image ID was +`sha256:439f85489106fc8a7d01c3ee265e31445c0879bd95b2578eaea7e573fed838d5`. +The recorded OCI config digest was +`sha256:a1f93e79d6fe463733e7a1b789a746c5fe35811dcd53cac26286dce737054519`. +These identify different image objects; do not equate a config digest with a +Docker image ID. Rebuild and verify both hosts for the next run. + +## 5. Previous fixes and what they establish + +- **Mirror locality, commit `81d07f03`**: partition recovery replayed an + approximately 88 MB WAL very slowly. In the diagnostic sample, 761 of 775 + slow reads selected a remote-rack mirror first. Optional + `ChunkReadPolicy.preferred_node_id` orders complete mirror hints by local + node, then local rack, preserving fallback and the original deadline. + Unknown or incomplete placement leaves the original order intact. + Production ChunkKV supplies its node ID; cache contents and replica membership + are unchanged. Files are under `lib/crowdb-chunk-client/src/chunk/`, its + `tests/`, and `app/crowdb-chunk-kv-server/src/storage.rs`. + Recorded results: 427 affected tests passed, one opt-in RDMA test ignored, + clippy passed, and retained-volume readiness completed in 46.796 seconds. + The later SF1 run exposed the separate election issue above. +- **Recovery memory, commit `cdf023d6`**: preserve the replay-built WAL index, + stream replay while retaining independent highest accepted/promise records, + and preserve current term/vote records during WAL garbage collection. + Other changes bound ChunkKV replay buffers, release encoded snapshot pages, + and avoid prefaulting unused frame-cache capacity. Focused memory experiments + and C++/FFI/tree checks passed. An isolated exact-volume Root replay improved + readiness from 97.756 to 1.791 seconds with matching recovered state. + This isolated diagnostic does not replace five-node acceptance. +- **Storage liveness in the same checkpoint**: honor the independently renewed + WAL process lease during ownership transfer; preserve staged strip state + through empty liveness refreshes; renew DiskDB ownership by CAS; refresh + Monitor route authority before optional old-owner confirmation; and retain + bounded recovery progress and mirror-rack consistency. +- **CI fixes in the same checkpoint**: disable initial auto-join before the + explicit fourth-node admission case, and accept only the specified bootstrap + readiness responses under the original limits. Earlier local reruns passed + the three previously failing cases, but closing verification on current HEAD + is still required. + +Do not interpret a previous image's pass as a pass for current HEAD. The old +raw receipts were deleted, so retain fresh evidence for any final claim. + +## 6. TPC loader implementation and acceptance limits + +- The committed implementation supports native Iceberg loading, standalone + S3 generation/upload/verification, and generated Dataset samples. Shared + journaling records acknowledged, failed, and uncertain outcomes. +- Standalone S3 explicitly uses zero SDK mutation retries. A regression checks + that HTTP 500 produces exactly one PUT. Failed/uncertain candidates are + retained for read-only reconciliation instead of automatic mutation replay + or generic deletion. +- Recorded exact-commit checks for the pushed 0.1.1 implementation: 252 tests + passed, six opt-in cases skipped; Ruff, wheel/sdist build, and Twine checks + passed. Its pushed CI was green. +- The subsequent 0.1.2 commit records readable table-scoped Parquet filenames, + the version bump, and matching usage/changelog/test changes. Fresh local + checks also passed: 252 tests, six opt-in skips, Ruff lint/format for 63 files, + wheel/sdist build, and Twine. These are local checks, not fresh remote + acceptance or a published release. New receipts are under + `.crowdb-runtime/artifacts/commit-remaining-20261031T120321Z/`. +- Older physical runs passed Iceberg SF0.01 for eight tables/86,805 rows, + standalone S3 SF1 for eight files/8,661,245 rows/340,201,375 bytes, and Dataset + readback for eight samples. These used older backend states and do not close + the current physical acceptance. The S3 result also predates the final + zero-retry acceptance setup. +- For the next run, build a wheel from the exact committed loader tree, save + its SHA-256 and source identity, and install it into a dedicated run-local + environment. Inspect any new worktree changes first. Recheck all three paths + against the new CROWDB image, including the 0.1.2 filename behavior. + +## 7. Remaining changes now captured + +The user subsequently requested committing the edits previously kept separate +from recovery work. They are now captured without overwriting their content: + +- CROWDB: `container/oci-image/README.md`, `clean.py`, + `docker_local_cleanup.py`, `host_cleanup.py`, `tests/cleanup_cli_test.py`, + `tests/docker_local_test.py`, `tests/host_cleanup_test.py`, untracked + `docker_garbage.py` and `tests/docker_garbage_test.py` under that directory; + `container/single-node-container/README.md`; + `doc/design/deploy/design-crowdb-oci-image.md`; and `pixi.toml`. +- Loader: `CHANGELOG.md`, `README.md`, `pyproject.toml`, + `src/crowdb_tpc_loader/__init__.py`, `src/crowdb_tpc_loader/loader.py`, and + `tests/unit/test_commits.py`. +- CROWDB's cleanup change preserves running recorded clusters by default; + explicit `--cluster`/`--all` can stop selected clusters. Unscoped cleanup also + collects unused Docker resources, while retaining protected receipts and data + according to `--keep-data`. Its affected OCI tooling suite passed all 150 + tests before commit. No cleanup command was run against the physical fixtures + as part of this commit validation. +- Both `/cjdata/cpp/crowdb-web` and `/cjdata/cpp/buzzcrow.github.io` still have + an untracked `.aws` path. Neither was read or staged. There were no other + pending website/journal changes when inspected. + +Capture a fresh `git status` before further edits and preserve any new user work. +The original physical-run hash receipts and other evidence remain deleted. + +## 8. Remaining execution, in dependency order + +- [x] **Implementation checkpoint**: commit the recovery and election changes + locally; push the completed loader implementation. Files: the commit scopes + in sections 2, 4, 5, and 6. +- [x] **Handoff and interruption record**: preserve this state outside the + deleted runtime directory; defer fresh testing to the next session as the + user requested. Files: this document. +- [x] **Remaining changes checkpoint**: commit the OCI cleanup behavior and + handoff in CROWDB, and the 0.1.2 filename changes in the loader, after local + affected checks. Files: the paths in section 7 and this document. +- [ ] **Fresh preparation**: inventory old owned resources on both hosts, + preserve user edits, create a new run ID, reconstruct the harness, build one + frozen-source OCI serially, import/verify it on both hosts, and establish + fresh baseline data and identities. Files: a new run's `run.json`, harness + source, image receipts, and P00-P03 results. +- [ ] **First-divergence regression**: complete a fresh native Iceberg SF1 load + with full file checksums and bidirectional SQL verification. Observe Root + leases, elections, backlog, process epochs, and memory. On failure, print the + exact error, capture the first divergence, and stop dependent work. + Files: new loader reports, operation ledger, and service logs. +- [ ] **Scale and protocol acceptance**: pass isolated native 512-record and + 4096-record cases across all five owners with full readback; verify fresh S3 + SF1 with zero mutation retries and generated Dataset publication/readback. + Files: P04/P05 ledgers and loader reports. +- [ ] **Thirty-minute workload**: run P06-P08 for 1,800 continuous active + seconds after preparation passes. Files: durable native/protocol ledgers, + complete metrics, and final data audit. +- [ ] **Recovery matrix**: execute P09-P16 using the new soak data, including + retained-volume recreation, the unresponsive child, follower/leader lifecycle, + two-node B outage, three-node A quorum loss, allocation safe retry, backlog, + full cold restart, and final occupancy audit. Files: per-case receipts and + fixed-checkpoint/data verification. +- [ ] **Closing CI**: rerun the original failures and affected protocol suites + against current inputs. Files: CI artifacts and final result matrix. +- [ ] **Close the run**: capture failures or the final pass, verify owned-only + cleanup, commit coherent follow-up fixes/results as authorized, and report + unfinished coverage honestly. Do not push CROWDB. Files: `results.json`, + `cleanup.json`, and `summary.md` in the new run directory. + +## 9. Workload and failure contracts to retain + +- Native scale: 512 then at least 4096 records, each 64 KiB, batches of 16, + with the original five-second operation deadline. Cover all five Serving + owners using actual catalog ranges. +- Exclude native key first-byte namespaces reserved by S3, Iceberg, and + Dataset (`S`, `I`, and `D`). A former harness used these prefixes and + contaminated protocol scans. Exact owned-key retirement was recorded before + deletion; do not reuse that harness or delete whole protocol prefixes. +- Intended native soak: one batch in flight, 80/20 put/delete mix with 16 exact + reads per cycle, durable intent/acknowledgment records, bounded live data, and + a final audit of all 4096 keys. Declare actual pacing and concurrency before + starting. Build/deploy/idle time does not count toward 30 minutes. +- Concurrent protocols: bounded S3 put/get/delete plus absence verification; + Iceberg property commit/load/cleanup; Dataset sample publication and complete + field verification. Keep mutation attempts single where required. An unknown + outcome is audited read-only, never retried merely to obtain success. +- Observe all 35 service PIDs and OOM state; collect metrics at least every + five seconds. Readiness observations must postdate the current process start + and be at most three seconds old. Require at least 30% free test media. +- Preserve independent 180-second service-recovery, catalog-takeover, and + per-group fixed-checkpoint catch-up gates. Do not replace them with one + whole-phase duration, increase timeouts, or weaken assertions. +- Any source/harness repair during the soak requires a complete new 30-minute + run after the repair. Earlier 70-second and 148.9-second runs stopped on + harness freshness errors and do not count. +- Run builds, tests, lint, and executables through `pixi run`, including remote + execution. Serialize shared-output builds and freeze production source during + OCI build. Add no new hot-path lock or unsafe exception without following + project rules. + +## 10. Closing CI scope and useful entry points + +Original user-reported runs: + +- [CROWDB run 41003611925](https://github.com/buzzcrow/crowdb/actions/runs/41003611925). +- [CROWDB run 41128190276](https://github.com/buzzcrow/crowdb/actions/runs/41128190276). + +The pending closing queue covered console native operations, deployment +services, deployment reference, Iceberg GC cleanup, and full S3 acceptance. +It did not dispatch on the latest election-fix image before interruption. +Reconstruct commands from current Pixi tasks and `e2e-test/README.md`; old +controller files are gone. Set `CROWDB_E2E_PRINT_SERVICE_ERRORS=1` for diagnosis +and retain complete client/service errors. Container CI does not replace the +physical run. + +Useful files for selecting focused validation: + +- Unit/integration: `lib/crowdb-kv/tests/election_test/vote_test.rs`, + `lib/crowdb-kv/tests/group_test/`, + `lib/crowdb-chunk-client/tests/chunk_reader_test/`, and + `lib/crowdb-chunk-client/tests/common/mirror_locality.rs`. +- Service/transport: `app/crowdb-kv-server/`, + `app/crowdb-chunk-kv-server/`, `app/crowdb-chunkdb/tests/`, and + `container/crowdb-monitor/tests/`. +- Container/protocol: `e2e-test/tests/console/`, + `e2e-test/tests/deployment/`, `e2e-test/tests/access_iceberg/`, and the + S3 acceptance entries documented in the E2E README. +- Design contracts: `doc/design/kv/design-crowdb-kv-leader-election.md`, + `doc/design/chunkio/design-crowdb-chunkio-reader.md`, and + `doc/dev/physical-node-test-plan.md`. + +Historical artifact root, now deleted: +`.crowdb-runtime/artifacts/physical-nodes/physical-20261031T024347Z/`. +Its former controller progression, `CURRENT-STATE.json`, `loader/` reports, +`mirror-locality-*` evidence, and `leader-vote-lease-*` evidence cannot be used +as executable inputs or linked as surviving proof. The commits and this handoff +are the starting point for the next test session. diff --git a/pixi.toml b/pixi.toml index 7d3088b2d..dc8234321 100644 --- a/pixi.toml +++ b/pixi.toml @@ -208,8 +208,9 @@ oci-run-1c = "python container/oci-image/docker-local.py run --nodes 1" oci-run-3c = "python container/oci-image/docker-local.py run --nodes 3" # Stop defaults to --cluster local; --all selects every recorded local cluster. oci-stop = "python container/oci-image/docker-local.py stop" -# Clean defaults to all recorded clusters; --cluster narrows the scope. -# Remove owned containers, bridges and data volumes; --keep-data preserves data. +# Clean stopped clusters and unused Docker containers, networks, volumes and build cache. +# Running clusters are preserved; --cluster/--all explicitly stop selected clusters. +# --keep-data preserves volumes, node data and restart receipts. oci-clean = "python container/oci-image/clean.py" ################################################################################ From 76fc6508189dd31018e953674a8f1c693011498d Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 1 Nov 2026 17:58:23 +0800 Subject: [PATCH 12/24] Stabilize native lifecycle, repair recovery and CI observations --- app/crowdb-chunk-kv-server/src/storage.rs | 40 +- app/crowdb-chunkdb/src/repair.rs | 170 +--- app/crowdb-chunkdb/src/repair/memory.rs | 54 ++ app/crowdb-chunkdb/src/repair/targets.rs | 186 +++++ .../tests/common/repair_targets.rs | 270 +++++++ .../tests/repair_target_recovery_test.rs | 110 +++ app/crowdb-web/src/diskdb/remote.rs | 3 +- app/crowdb-web/src/lib.rs | 7 + app/crowdb-web/src/main.rs | 1 + .../src/services/deployment/remote.rs | 8 +- app/crowdb-web/src/services/observation.rs | 81 +- .../src/services/observation/cache.rs | 224 ++++++ .../src/services/observation/mutation.rs | 170 ++++ .../src/services/observation/sampling.rs | 171 ++++ .../src/services/observation/source.rs | 80 ++ .../src/services/observation/test_support.rs | 32 + .../src/services/observation/worker.rs | 198 +++++ app/crowdb-web/src/services/remote.rs | 142 ++-- app/crowdb-web/src/state.rs | 2 + .../tests/common/service_inventory.rs | 270 +++++++ .../internal/service_start_pending_test.rs | 41 - .../tests/remote_kv_projection_test.rs | 8 + .../service_inventory_observation_test.rs | 237 ++++++ .../src/node/runtime/observation.rs | 8 +- .../src/node/runtime/services.rs | 53 +- .../src/node/runtime/services/execution.rs | 76 +- .../src/node/runtime/services/test_support.rs | 23 +- .../tests/common/service_replacement.rs | 100 +++ .../tests/node_process_operation_test.rs | 95 +++ .../tests/node_service_replacement_test.rs | 70 ++ doc/dev/physical-node-test-plan.md | 21 +- doc/working/plan-ci-stability-review.md | 504 ++++++++++++ doc/working/plan-physical-node-ci-handoff.md | 752 +++++++++--------- e2e-test/browser_batch.py | 37 +- .../e2e/fixtures/journalRuntimeRequest.ts | 52 ++ .../console-ui/e2e/fixtures/nativeProcess.ts | 28 +- .../e2e/fixtures/nativeServiceLifecycle.ts | 122 +++ .../flows/11-cluster-server-lifecycle.spec.ts | 60 +- .../e2e/flows/55-chunk-kv-catalog.spec.ts | 45 +- e2e-test/console_cli_target.py | 48 +- e2e-test/fixture.py | 36 +- e2e-test/fixture_result.py | 10 + e2e-test/oci_shortcuts.py | 6 +- e2e-test/report.py | 4 +- .../tests/access_iceberg/gc/convergence.rs | 36 +- .../access_iceberg/gc/convergence/pressure.rs | 121 ++- .../gc/convergence/pressure/sdk.rs | 234 ++++++ .../access_iceberg/gc/notifications/matrix.rs | 33 +- .../tests/access_iceberg/gc/sdk_pressure.py | 73 +- .../tests/browser_batch_provenance_test.py | 140 ++++ e2e-test/tests/browser_batch_test.py | 22 + e2e-test/tests/console/native_balance.rs | 24 +- e2e-test/tests/console_cli_inventory_test.py | 91 +++ e2e-test/tests/fixture_test.py | 30 +- e2e-test/tests/gc_sdk_pressure_test.py | 121 +++ .../tests/journal_request_stability_test.py | 20 + .../tests/journal_runtime_request_test.mjs | 74 ++ e2e-test/tests/oci_shortcuts_test.py | 62 ++ e2e-test/tests/report_test.py | 40 + lib/crowdb-access-iceberg/src/file/reader.rs | 53 +- .../tests/file_stream_test.rs | 26 +- lib/crowdb-chunk-kv/src/partition.rs | 50 +- lib/crowdb-chunk-kv/src/partition/native.rs | 81 ++ lib/crowdb-chunk-kv/src/partition/tree.rs | 20 +- .../tests/common/root_catalog.rs | 67 ++ .../tests/native_runtime_test.rs | 135 ++++ lib/crowdb-rpc/ffi/src/logging.rs | 1 + lib/crowdb-rpc/ffi/src/owned_route.rs | 72 +- .../ffi/src/owned_route/retirement.rs | 147 ++++ .../ffi/tests/common/owned_route.rs | 184 +++++ .../ffi/tests/common/route_retirement.rs | 79 ++ .../ffi/tests/owned_route_lifecycle_test.rs | 22 + .../ffi/tests/route_retirement_test.rs | 111 +++ lib/crowdb-rpc/src/c_api.cpp | 32 +- .../tests/c_api_submit_lifetime_test.cpp | 124 +++ lib/crowdb-tree/ffi/src/tree.rs | 5 + tools/test-metrics/measure.py | 3 +- 77 files changed, 6009 insertions(+), 979 deletions(-) create mode 100644 app/crowdb-chunkdb/src/repair/memory.rs create mode 100644 app/crowdb-chunkdb/src/repair/targets.rs create mode 100644 app/crowdb-chunkdb/tests/common/repair_targets.rs create mode 100644 app/crowdb-chunkdb/tests/repair_target_recovery_test.rs create mode 100644 app/crowdb-web/src/services/observation/cache.rs create mode 100644 app/crowdb-web/src/services/observation/mutation.rs create mode 100644 app/crowdb-web/src/services/observation/sampling.rs create mode 100644 app/crowdb-web/src/services/observation/source.rs create mode 100644 app/crowdb-web/src/services/observation/test_support.rs create mode 100644 app/crowdb-web/src/services/observation/worker.rs create mode 100644 app/crowdb-web/tests/common/service_inventory.rs delete mode 100644 app/crowdb-web/tests/internal/service_start_pending_test.rs create mode 100644 app/crowdb-web/tests/service_inventory_observation_test.rs create mode 100644 container/crowdb-monitor/tests/common/service_replacement.rs create mode 100644 container/crowdb-monitor/tests/node_process_operation_test.rs create mode 100644 container/crowdb-monitor/tests/node_service_replacement_test.rs create mode 100644 doc/working/plan-ci-stability-review.md create mode 100644 e2e-test/clients/console-ui/e2e/fixtures/journalRuntimeRequest.ts create mode 100644 e2e-test/clients/console-ui/e2e/fixtures/nativeServiceLifecycle.ts create mode 100644 e2e-test/fixture_result.py create mode 100644 e2e-test/tests/access_iceberg/gc/convergence/pressure/sdk.rs create mode 100644 e2e-test/tests/browser_batch_provenance_test.py create mode 100644 e2e-test/tests/console_cli_inventory_test.py create mode 100644 e2e-test/tests/gc_sdk_pressure_test.py create mode 100644 e2e-test/tests/journal_request_stability_test.py create mode 100644 e2e-test/tests/journal_runtime_request_test.mjs create mode 100644 lib/crowdb-chunk-kv/src/partition/native.rs create mode 100644 lib/crowdb-chunk-kv/tests/common/root_catalog.rs create mode 100644 lib/crowdb-chunk-kv/tests/native_runtime_test.rs create mode 100644 lib/crowdb-rpc/ffi/src/owned_route/retirement.rs create mode 100644 lib/crowdb-rpc/ffi/tests/common/owned_route.rs create mode 100644 lib/crowdb-rpc/ffi/tests/common/route_retirement.rs create mode 100644 lib/crowdb-rpc/ffi/tests/owned_route_lifecycle_test.rs create mode 100644 lib/crowdb-rpc/ffi/tests/route_retirement_test.rs create mode 100644 lib/crowdb-rpc/tests/c_api_submit_lifetime_test.cpp diff --git a/app/crowdb-chunk-kv-server/src/storage.rs b/app/crowdb-chunk-kv-server/src/storage.rs index 6fc078419..ea7606605 100644 --- a/app/crowdb-chunk-kv-server/src/storage.rs +++ b/app/crowdb-chunk-kv-server/src/storage.rs @@ -534,16 +534,22 @@ impl ChunkKvStorage { if !fresh { return recover_bootstrap_partition(config, page_store, stream).await; } - let partition = Partition::open_native( - partition_id, - range, - config.owner_epoch, - PartitionConfig::default(), - config.tree_id, - crowdb_tree_ffi::Config::default(), - page_store, - stream, - ) + let owner_epoch = config.owner_epoch; + let tree_id = config.tree_id; + let partition = tokio::task::spawn_blocking(move || { + Partition::open_native( + partition_id, + range, + owner_epoch, + PartitionConfig::default(), + tree_id, + crowdb_tree_ffi::Config::default(), + page_store, + stream, + ) + }) + .await + .map_err(|error| StorageRuntimeError::Partition(format!("bootstrap tree open: {error}")))? .map_err(|error| StorageRuntimeError::Partition(format!("bootstrap tree open: {error}")))?; initialize_bootstrap_root(&partition, config).await?; Ok(partition) @@ -599,10 +605,16 @@ impl ChunkKvStorage { "nonempty child WAL has no committed handoff proof", )); } - crowdb_chunk_kv::CrowdbPartitionTree::open(child.tree_id, &child.tree_config) - .map_err(|error| storage_plan_error(&error.to_string()))? - .unpin_generation(split_plan(transition).transition_id) - .map_err(|error| storage_plan_error(&error.to_string()))?; + let tree_id = child.tree_id; + let tree_config = child.tree_config.clone(); + let transition_id = split_plan(transition).transition_id; + tokio::task::spawn_blocking(move || { + crowdb_chunk_kv::CrowdbPartitionTree::open(tree_id, &tree_config)? + .unpin_generation(transition_id) + }) + .await + .map_err(|error| storage_plan_error(&error.to_string()))? + .map_err(|error| storage_plan_error(&error.to_string()))?; child }; let handoff = Arc::new(split_handoff::SplitHandoffCommit { diff --git a/app/crowdb-chunkdb/src/repair.rs b/app/crowdb-chunkdb/src/repair.rs index b10991abb..e94e98bf9 100644 --- a/app/crowdb-chunkdb/src/repair.rs +++ b/app/crowdb-chunkdb/src/repair.rs @@ -25,6 +25,9 @@ use crate::metrics::RepairMetrics; use crate::task::executor::TaskFuture; use crate::task::{TaskHandler, TaskManager, TaskOutcome, TaskStore, TaskStoreError}; +mod memory; +mod targets; + pub const REPAIR_STRIP_TASK_VERSION: u16 = 1; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -209,7 +212,6 @@ impl RepairStripTaskHandler { self } - #[allow(clippy::too_many_lines)] async fn execute_once(&self, task: &ChunkTaskValue) -> Result { let mut payload = decode_payload(&task.payload)?; let chunk = self.lifecycle.query_chunk(&payload.chunk_id).await?; @@ -224,6 +226,7 @@ impl RepairStripTaskHandler { let segments = strip_segments(strip)?; self.confirm_published_targets(task, &mut payload, &segments) .await?; + self.reconcile_allocated_targets(task, &mut payload).await?; let unavailable: Vec<_> = strip .unavailable_segments .iter() @@ -237,45 +240,9 @@ impl RepairStripTaskHandler { .checked_mul(1024) .ok_or_else(|| RepairRunError::Permanent("unit size overflows".into()))?; let shard_bytes = segment_size(&segments[0], unit_bytes)?; - let required = shard_bytes - .checked_mul(u64::try_from(segments.len().saturating_add(1)).unwrap_or(u64::MAX)) - .and_then(|bytes| usize::try_from(bytes).ok()) - .ok_or_else(|| RepairRunError::Permanent("repair memory calculation overflows".into()))?; - if required > self.memory_limit { - return Err(RepairRunError::Retry(format!( - "repair requires {required} bytes but the limit is {}", - self.memory_limit - ))); - } - let permits = u32::try_from(required) - .map_err(|_| RepairRunError::Permanent("repair memory exceeds semaphore".into()))?; - let _reservation = self - .memory - .clone() - .acquire_many_owned(permits) - .await - .map_err(|_| RepairRunError::Retry("repair memory budget closed".into()))?; - self.metrics.reserve_memory(required); - let _memory_metrics = RepairMemoryGuard { - metrics: Arc::clone(&self.metrics), - bytes: required, - }; + let (_reservation, _memory_metrics) = self.reserve_memory(shard_bytes, segments.len()).await?; - let recovered = match strip.strip.as_ref() { - Some(Strip::MirrorStrip(_)) => self.recover_mirror(strip, &segments).await?, - Some(Strip::EcStrip(ec)) => { - let sealed_bytes = if chunk.acknowledged_cursor == 0 { - u64::from(strip.sealed_length) * 1024 - } else { - chunk - .acknowledged_cursor - .saturating_sub(u64::from(strip.chunk_offset) * 1024) - .min(u64::from(strip.capacity) * 1024) - }; - self.recover_ec(strip, ec, &segments, sealed_bytes).await? - } - None => return Err(RepairRunError::Permanent("strip has no body".into())), - }; + let recovered = self.recover_shards(&chunk, strip, &segments).await?; if !recovered.new_failures.is_empty() { self.persist_new_failures( &chunk, @@ -288,94 +255,9 @@ impl RepairStripTaskHandler { return Err(RepairRunError::Retry("new failed segments were persisted".into())); } - let mut replacement = strip.clone(); - let mut surviving: Vec<_> = segments - .iter() - .copied() - .filter(|segment| !unavailable.contains(segment)) - .collect(); - // Even when node anti-affinity is relaxed for an undersized test - // cluster, never place two shards from one strip on the same disk. - let mut excluded: Vec<_> = segments.iter().filter_map(|segment| segment.disk_id).collect(); - for failed in &unavailable { - let index = segments - .iter() - .position(|segment| segment == failed) - .ok_or_else(|| RepairRunError::Permanent("failed segment disappeared".into()))?; - let target_index = payload.targets.iter().position(|target| target.source == *failed); - let target_index = if let Some(index) = target_index { - index - } else { - let destination = self - .lifecycle - .allocate_repair_segment( - &payload.chunk_id, - failed, - &surviving, - &excluded, - self.allow_unsafe_placement, - ) - .await?; - payload.targets.push(RepairTargetCheckpoint { - source: *failed, - destination, - phase: RepairTargetPhase::Allocated, - }); - self.checkpoint(task, &payload).await?; - payload.targets.len().saturating_sub(1) - }; - let destination = payload.targets[target_index].destination; - if payload.targets[target_index].phase == RepairTargetPhase::Allocated { - let mut data = recovered.shards[index].clone(); - if let Some(Strip::EcStrip(ec)) = &strip.strip { - let sealed = if chunk.acknowledged_cursor == 0 { - u64::from(strip.sealed_length) * 1024 - } else { - chunk - .acknowledged_cursor - .saturating_sub(u64::from(strip.chunk_offset) * 1024) - .min(u64::from(strip.capacity) * 1024) - }; - let valid = if index < ec.data_num as usize { - sealed.saturating_sub(index as u64 * shard_bytes).min(shard_bytes) - } else { - sealed.min(shard_bytes) - }; - data = data.slice(..usize::try_from(valid).unwrap_or(data.len())); - } - if !data.is_empty() { - self.io - .write_segment(&destination, unit_bytes, data) - .await - .map_err(|error| RepairRunError::Retry(error.to_string()))?; - } - self.io - .fsync_segment(&destination) - .await - .map_err(|error| RepairRunError::Retry(error.to_string()))?; - payload.targets[target_index].phase = RepairTargetPhase::Copied; - self.checkpoint(task, &payload).await?; - } - if let Some(shared) = &self.ad_hoc { - shared.rebuilt( - RecoveryKey { - chunk_id: payload.chunk_id, - strip_sequence: payload.strip_sequence, - segment: *failed, - }, - recovered.shards[index].clone(), - ); - } - let new_segment = destination; - replace_segment(&mut replacement, failed, new_segment)?; - surviving.push(new_segment); - if let Some(disk_id) = new_segment.disk_id { - excluded.push(disk_id); - } - } - replacement - .unavailable_segments - .retain(|segment| !unavailable.contains(segment)); + let replacement = self + .prepare_replacement(task, &mut payload, &chunk, strip, &recovered, &unavailable) + .await?; self.lifecycle .publish_tentative_chunk_strip_range( &payload.chunk_id, @@ -459,6 +341,29 @@ impl RepairStripTaskHandler { self.checkpoint(task, payload).await } + async fn recover_shards( + &self, + chunk: &Chunk, + strip: &ChunkStrip, + segments: &[Segment], + ) -> Result { + match strip.strip.as_ref() { + Some(Strip::MirrorStrip(_)) => self.recover_mirror(strip, segments).await, + Some(Strip::EcStrip(ec)) => { + let sealed_bytes = if chunk.acknowledged_cursor == 0 { + u64::from(strip.sealed_length) * 1024 + } else { + chunk + .acknowledged_cursor + .saturating_sub(u64::from(strip.chunk_offset) * 1024) + .min(u64::from(strip.capacity) * 1024) + }; + self.recover_ec(strip, ec, segments, sealed_bytes).await + } + None => Err(RepairRunError::Permanent("strip has no body".into())), + } + } + async fn recover_mirror( &self, strip: &ChunkStrip, @@ -644,17 +549,6 @@ struct RepairStats { bytes: u64, } -struct RepairMemoryGuard { - metrics: Arc, - bytes: usize, -} - -impl Drop for RepairMemoryGuard { - fn drop(&mut self) { - self.metrics.release_memory(self.bytes); - } -} - #[derive(Debug, thiserror::Error)] enum RepairRunError { #[error(transparent)] diff --git a/app/crowdb-chunkdb/src/repair/memory.rs b/app/crowdb-chunkdb/src/repair/memory.rs new file mode 100644 index 000000000..b668d1c0c --- /dev/null +++ b/app/crowdb-chunkdb/src/repair/memory.rs @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Memory admission for one full-segment repair attempt. + +use super::{RepairMetrics, RepairRunError, RepairStripTaskHandler}; +use std::sync::Arc; +use tokio::sync::OwnedSemaphorePermit; + +pub(super) struct RepairMemoryGuard { + metrics: Arc, + bytes: usize, +} + +impl Drop for RepairMemoryGuard { + fn drop(&mut self) { + self.metrics.release_memory(self.bytes); + } +} + +impl RepairStripTaskHandler { + pub(super) async fn reserve_memory( + &self, + shard_bytes: u64, + segment_count: usize, + ) -> Result<(OwnedSemaphorePermit, RepairMemoryGuard), RepairRunError> { + let required = shard_bytes + .checked_mul(u64::try_from(segment_count.saturating_add(1)).unwrap_or(u64::MAX)) + .and_then(|bytes| usize::try_from(bytes).ok()) + .ok_or_else(|| RepairRunError::Permanent("repair memory calculation overflows".into()))?; + if required > self.memory_limit { + return Err(RepairRunError::Retry(format!( + "repair requires {required} bytes but the limit is {}", + self.memory_limit, + ))); + } + let permits = u32::try_from(required) + .map_err(|_| RepairRunError::Permanent("repair memory exceeds semaphore".into()))?; + let reservation = self + .memory + .clone() + .acquire_many_owned(permits) + .await + .map_err(|_| RepairRunError::Retry("repair memory budget closed".into()))?; + self.metrics.reserve_memory(required); + Ok(( + reservation, + RepairMemoryGuard { + metrics: Arc::clone(&self.metrics), + bytes: required, + }, + )) + } +} diff --git a/app/crowdb-chunkdb/src/repair/targets.rs b/app/crowdb-chunkdb/src/repair/targets.rs new file mode 100644 index 000000000..007563e02 --- /dev/null +++ b/app/crowdb-chunkdb/src/repair/targets.rs @@ -0,0 +1,186 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Checkpointed tentative repair destinations and replacement assembly. + +use super::{replace_segment, segment_size, strip_segments, RecoveredShards, RepairRunError}; +use super::{RepairStripTaskHandler, RepairStripTaskV1}; +use crate::ad_hoc::RecoveryKey; +use bytes::Bytes; +use crowdb_protocol::chunk_task::{ChunkTaskValue, RepairTargetCheckpoint, RepairTargetPhase}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkStrip, Strip}; +use crowdb_protocol::common::DiskId; +use crowdb_protocol::diskdb::rpc::Segment; + +impl RepairStripTaskHandler { + pub(super) async fn reconcile_allocated_targets( + &self, + task: &ChunkTaskValue, + payload: &mut RepairStripTaskV1, + ) -> Result<(), RepairRunError> { + if !payload + .targets + .iter() + .any(|target| target.phase == RepairTargetPhase::Allocated) + { + return Ok(()); + } + if payload.targets.iter().any(|target| { + target.phase == RepairTargetPhase::Allocated && target.destination.disk_id.is_none() + }) { + return Err(RepairRunError::Permanent( + "repair destination has no disk identity".into(), + )); + } + let snapshot = self.lifecycle.topology_snapshot(); + if snapshot.is_empty() { + return Err(RepairRunError::Retry( + "repair target topology is unavailable".into(), + )); + } + let healthy = snapshot.healthy_disk_groups(); + let previous = payload.targets.len(); + payload.targets.retain(|target| { + target.phase != RepairTargetPhase::Allocated + || target + .destination + .disk_id + .is_some_and(|disk| healthy.iter().any(|group| group.value.disk_ids.contains(&disk))) + }); + if payload.targets.len() != previous { + // Publish removal before allocating another destination. The exact + // old tentative block remains in DiskDB; its owner scanner applies + // the normal persisted absence grace, including after reply loss. + self.checkpoint(task, payload).await?; + } + Ok(()) + } + + pub(super) async fn prepare_replacement( + &self, + task: &ChunkTaskValue, + payload: &mut RepairStripTaskV1, + chunk: &Chunk, + strip: &ChunkStrip, + recovered: &RecoveredShards, + unavailable: &[Segment], + ) -> Result { + let segments = strip_segments(strip)?; + let unit_bytes = u64::from(strip.unit_kb) * 1024; + let shard_bytes = segment_size(&segments[0], unit_bytes)?; + let mut replacement = strip.clone(); + let mut surviving: Vec<_> = segments + .iter() + .copied() + .filter(|segment| !unavailable.contains(segment)) + .collect(); + // Preserve disk separation even when the node policy is relaxed. + let mut excluded: Vec<_> = segments.iter().filter_map(|segment| segment.disk_id).collect(); + for failed in unavailable { + let index = segments + .iter() + .position(|segment| segment == failed) + .ok_or_else(|| RepairRunError::Permanent("failed segment disappeared".into()))?; + let target = self + .target_index(task, payload, failed, &surviving, &excluded) + .await?; + let destination = payload.targets[target].destination; + if payload.targets[target].phase == RepairTargetPhase::Allocated { + let data = shard_data(chunk, strip, index, shard_bytes, recovered.shards[index].clone()); + self.copy_target(task, payload, target, unit_bytes, data).await?; + } + if let Some(shared) = &self.ad_hoc { + shared.rebuilt( + RecoveryKey { + chunk_id: payload.chunk_id, + strip_sequence: payload.strip_sequence, + segment: *failed, + }, + recovered.shards[index].clone(), + ); + } + replace_segment(&mut replacement, failed, destination)?; + surviving.push(destination); + if let Some(disk_id) = destination.disk_id { + excluded.push(disk_id); + } + } + replacement + .unavailable_segments + .retain(|segment| !unavailable.contains(segment)); + Ok(replacement) + } + + async fn target_index( + &self, + task: &ChunkTaskValue, + payload: &mut RepairStripTaskV1, + failed: &Segment, + surviving: &[Segment], + excluded: &[DiskId], + ) -> Result { + if let Some(index) = payload.targets.iter().position(|target| target.source == *failed) { + return Ok(index); + } + let destination = self + .lifecycle + .allocate_repair_segment( + &payload.chunk_id, + failed, + surviving, + excluded, + self.allow_unsafe_placement, + ) + .await?; + payload.targets.push(RepairTargetCheckpoint { + source: *failed, + destination, + phase: RepairTargetPhase::Allocated, + }); + self.checkpoint(task, payload).await?; + Ok(payload.targets.len().saturating_sub(1)) + } + + async fn copy_target( + &self, + task: &ChunkTaskValue, + payload: &mut RepairStripTaskV1, + index: usize, + unit_bytes: u64, + data: Bytes, + ) -> Result<(), RepairRunError> { + let destination = payload.targets[index].destination; + if !data.is_empty() { + self.io + .write_segment(&destination, unit_bytes, data) + .await + .map_err(|error| RepairRunError::Retry(error.to_string()))?; + } + self.io + .fsync_segment(&destination) + .await + .map_err(|error| RepairRunError::Retry(error.to_string()))?; + payload.targets[index].phase = RepairTargetPhase::Copied; + self.checkpoint(task, payload).await + } +} + +fn shard_data(chunk: &Chunk, strip: &ChunkStrip, index: usize, shard_bytes: u64, data: Bytes) -> Bytes { + let Some(Strip::EcStrip(ec)) = &strip.strip else { + return data; + }; + let sealed = if chunk.acknowledged_cursor == 0 { + u64::from(strip.sealed_length) * 1024 + } else { + chunk + .acknowledged_cursor + .saturating_sub(u64::from(strip.chunk_offset) * 1024) + .min(u64::from(strip.capacity) * 1024) + }; + let valid = if index < ec.data_num as usize { + sealed.saturating_sub(index as u64 * shard_bytes).min(shard_bytes) + } else { + sealed.min(shard_bytes) + }; + data.slice(..usize::try_from(valid).unwrap_or(data.len())) +} diff --git a/app/crowdb-chunkdb/tests/common/repair_targets.rs b/app/crowdb-chunkdb/tests/common/repair_targets.rs new file mode 100644 index 000000000..cf9e15e4b --- /dev/null +++ b/app/crowdb-chunkdb/tests/common/repair_targets.rs @@ -0,0 +1,270 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crate::common::cluster::{ + seed_hardware, ChunkdbHarness, DiskdbServer, KvCluster, DATA_GROUP_ID, STORE_ID, +}; +use crate::common::diskio_stub::TestDiskio; +use crowdb_chunkdb::conversion::io::ConversionDiskIo; +use crowdb_chunkdb::metrics::ChunkdbMetrics; +use crowdb_chunkdb::repair::{decode_payload, encode_payload, RepairCoordinator, RepairStripTaskHandler}; +use crowdb_chunkdb::routing::{default_binding_table, BindingCache}; +use crowdb_chunkdb::task::{TaskExecutor, TaskManager, TaskStore}; +use crowdb_chunkdb::topology::build_snapshot; +use crowdb_common::metrics::MetricsRegistry; +use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_protocol::chunk_task::{ChunkTaskValue, RepairTargetCheckpoint, RepairTargetPhase}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkType, Strip, StripType}; +use crowdb_protocol::common::{ChunkId, InstanceValue}; +use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_protocol::key::InstanceKey; + +pub struct TestRepairTargets { + pub cluster: KvCluster, + pub harness: ChunkdbHarness, + pub tasks: Arc, + pub manager: Arc, + pub io: Arc, + pub chunk: Chunk, + pub failed: Segment, + pub survivor: Segment, + pub target: Segment, + pub task: ChunkTaskValue, + disks: Vec<(u64, Option)>, + _diskdb: DiskdbServer, +} + +impl TestRepairTargets { + pub async fn start() -> Self { + let cluster = KvCluster::start_pinned().await; + let hardware = cluster.make_hardware_client(); + seed_hardware(&hardware).await; + let diskdb = DiskdbServer::start(&cluster).await; + let registry = cluster.make_service_registry_client(); + let mut disks = Vec::new(); + for group in hardware.list_disk_groups().await.unwrap() { + let disk = + TestDiskio::start(®istry, group.dg_id, group.rack_id, group.node_id, group.dg_id).await; + disks.push((group.dg_id, Some(disk))); + } + let harness = ChunkdbHarness::start(&cluster).await; + let chunk = harness + .handler + .allocate_chunk(None, 1, 1, StripType::Mirror, 0, 0, 2, ChunkType::S3, 0, 0) + .await + .unwrap(); + let Strip::MirrorStrip(mirror) = chunk.strips[0].strip.as_ref().unwrap() else { + panic!("expected mirrored chunk"); + }; + let (failed, survivor) = (mirror.segments[0], mirror.segments[1]); + let id = chunk.id.unwrap(); + let target = harness + .handler + .allocate_repair_segment( + &id, + &failed, + &[survivor], + &[failed.disk_id.unwrap(), survivor.disk_id.unwrap()], + false, + ) + .await + .unwrap(); + let io = Arc::new(ConversionDiskIo::connect(®istry, &hardware).await.unwrap()); + let unit = u64::from(chunk.strips[0].unit_kb) * 1024; + let data = bytes::Bytes::from(vec![42; usize::try_from(unit).unwrap()]); + io.write_segment(&survivor, unit, data).await.unwrap(); + io.fsync_segment(&survivor).await.unwrap(); + let marked = mark_failed(&harness, &chunk, failed).await; + let bindings = BindingCache::new(); + bindings + .replace(default_binding_table(STORE_ID, DATA_GROUP_ID)) + .unwrap(); + let tasks = Arc::new(TaskStore::new(cluster.make_crowdb_client(), bindings)); + let task = persist_allocated_target(&harness, &tasks, &marked, target).await; + let manager = Arc::new(TaskManager::new(Arc::clone(&tasks), 8001, 30_000)); + Self { + cluster, + harness, + tasks, + manager, + io, + chunk: marked, + failed, + survivor, + target, + task, + disks, + _diskdb: diskdb, + } + } + + pub async fn stop_target(&mut self, expire_registration: bool) { + let group = self + .harness + .topology + .snapshot() + .disk_location(self.target.disk_id.unwrap()) + .unwrap() + .disk_group_id; + let key = InstanceKey { + service: "diskio".into(), + instance_id: group, + } + .to_path(); + if expire_registration { + let kv = self.cluster.make_crowdb_client(); + let GetOutcome::Found { value, .. } = kv + .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) + .await + .unwrap() + else { + panic!("missing DiskIO registration"); + }; + let mut instance: InstanceValue = serde_json::from_slice(&value).unwrap(); + instance.last_heartbeat_ms = 0; + kv.put( + 0, + 0, + key.as_bytes(), + &serde_json::to_vec(&instance).unwrap(), + None, + ) + .await + .unwrap(); + } + self.disks + .iter_mut() + .find(|(id, _)| *id == group) + .unwrap() + .1 + .take(); + let snapshot = build_snapshot(&self.cluster.make_hardware_client()) + .await + .unwrap(); + assert_eq!( + snapshot.healthy_disk_groups().iter().any(|g| g.dg_id == group), + !expire_registration + ); + self.harness.topology.replace(snapshot); + } + + pub async fn execute(&self) -> ChunkTaskValue { + let ready = self.tasks.scan_ready(now_ms(), 16).await.unwrap(); + let index = ready + .iter() + .find(|index| index.task_id == self.task.task_id) + .unwrap(); + let claim = self.manager.claim(index, now_ms()).await.unwrap().unwrap(); + let metrics = ChunkdbMetrics::register(&mut MetricsRegistry::new()).repair; + let worker = RepairStripTaskHandler::new( + Arc::clone(&self.harness.handler), + Arc::clone(&self.manager), + Arc::clone(&self.io), + 16 * 1024 * 1024, + 1, + false, + metrics, + ); + TaskExecutor::new(Arc::clone(&self.manager), 1, vec![Arc::new(worker)]) + .unwrap() + .execute(claim) + .await + .unwrap(); + self.tasks + .get(&self.task.partition_id, self.task.kind, &self.task.task_id) + .await + .unwrap() + .unwrap() + } + + pub async fn publish_before_checkpoint(&self) -> Chunk { + let unit = u64::from(self.chunk.strips[0].unit_kb) * 1024; + let bytes = self.io.read_segment(&self.survivor, unit).await.unwrap(); + self.io.write_segment(&self.target, unit, bytes).await.unwrap(); + self.io.fsync_segment(&self.target).await.unwrap(); + let mut replacement = self.chunk.strips[0].clone(); + let Strip::MirrorStrip(mirror) = replacement.strip.as_mut().unwrap() else { + unreachable!() + }; + *mirror + .segments + .iter_mut() + .find(|segment| **segment == self.failed) + .unwrap() = self.target; + replacement.unavailable_segments.clear(); + self.harness + .handler + .publish_tentative_chunk_strip_range( + &self.chunk.id.unwrap(), + self.chunk.modify_ts, + 0, + &self.chunk.strips, + &[replacement], + self.task.operation_id, + ) + .await + .unwrap() + } +} + +async fn mark_failed(harness: &ChunkdbHarness, chunk: &Chunk, failed: Segment) -> Chunk { + let mut marked = chunk.strips[0].clone(); + marked.unavailable_segments.push(failed); + harness + .handler + .replace_chunk_strip_range( + &chunk.id.unwrap(), + chunk.modify_ts, + 0, + &chunk.strips, + &[marked], + ChunkId { high: 111, low: 1 }, + ) + .await + .unwrap() +} + +async fn persist_allocated_target( + harness: &ChunkdbHarness, + tasks: &Arc, + chunk: &Chunk, + target: Segment, +) -> ChunkTaskValue { + RepairCoordinator::new(Arc::clone(&harness.handler), Arc::clone(tasks)) + .admit_chunk(chunk, now_ms()) + .await + .unwrap(); + let ready = tasks.scan_ready(now_ms(), 16).await.unwrap(); + let index = ready + .iter() + .find(|index| index.kind == crowdb_protocol::chunk_task::TASK_KIND_REPAIR_STRIP) + .unwrap(); + let task = tasks + .get(&chunk.id.unwrap(), index.kind, &index.task_id) + .await + .unwrap() + .unwrap(); + let mut payload = decode_payload(&task.payload).unwrap(); + payload.targets.push(RepairTargetCheckpoint { + source: payload.failed_segments[0], + destination: target, + phase: RepairTargetPhase::Allocated, + }); + let mut checkpoint = task.clone(); + checkpoint.revision += 1; + checkpoint.payload = encode_payload(&payload).unwrap(); + tasks.write_transition(Some(&task), &checkpoint).await.unwrap(); + checkpoint +} + +fn now_ms() -> u64 { + u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap() +} diff --git a/app/crowdb-chunkdb/tests/repair_target_recovery_test.rs b/app/crowdb-chunkdb/tests/repair_target_recovery_test.rs new file mode 100644 index 000000000..f559c1926 --- /dev/null +++ b/app/crowdb-chunkdb/tests/repair_target_recovery_test.rs @@ -0,0 +1,110 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +mod common; +#[path = "common/repair_targets.rs"] +mod repair_targets; + +use crowdb_chunkdb::repair::decode_payload; +use crowdb_chunkdb::task::SegmentOwnerResolver; +use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_protocol::chunk_task::{ChunkTaskState, RepairTargetPhase}; +use crowdb_protocol::chunkdb::rpc::{SegmentOwnerDisposition, Strip}; +use crowdb_protocol::key::{BinaryKey, BusyBlockKey}; +use repair_targets::TestRepairTargets; +use std::sync::Arc; + +#[tokio::test] +async fn expired_unpublished_repair_target_is_replaced_without_freeing_its_allocation() { + let mut test = TestRepairTargets::start().await; + test.stop_target(true).await; + let task = test.execute().await; + assert_eq!(task.state, ChunkTaskState::Completed, "{}", task.last_error); + let chunk = test + .harness + .handler + .query_chunk(&test.chunk.id.unwrap()) + .await + .unwrap(); + let Strip::MirrorStrip(mirror) = chunk.strips[0].strip.as_ref().unwrap() else { + unreachable!() + }; + assert!(!mirror.segments.contains(&test.failed)); + assert!(!mirror.segments.contains(&test.target)); + let replacement = mirror + .segments + .iter() + .find(|segment| **segment != test.survivor) + .unwrap(); + let unit = u64::from(chunk.strips[0].unit_kb) * 1024; + assert_eq!( + test.io.read_segment(replacement, unit).await.unwrap().as_ref(), + vec![42; usize::try_from(unit).unwrap()] + ); + let owner = SegmentOwnerResolver::new(Arc::clone(&test.harness.handler), Arc::clone(&test.tasks)); + assert_eq!( + owner + .resolve(&test.chunk.id.unwrap(), &test.target) + .await + .unwrap(), + SegmentOwnerDisposition::Absent + ); + let key = BusyBlockKey { + disk_id: test.target.disk_id.unwrap(), + zone_index: test.target.zone_index, + unit_offset: test.target.unit_offset, + } + .to_bytes(); + assert!( + matches!( + test.cluster + .make_crowdb_client() + .get(0, 1, &key, ReadMode::Linearizable, None) + .await + .unwrap(), + GetOutcome::Found { .. } + ), + "unpublished allocation must remain for the DiskDB owner scanner grace period" + ); +} + +#[tokio::test] +async fn live_unpublished_repair_target_is_retained_after_a_transport_failure() { + let mut test = TestRepairTargets::start().await; + test.stop_target(false).await; + let task = test.execute().await; + assert_eq!(task.state, ChunkTaskState::RetryWait); + let payload = decode_payload(&task.payload).unwrap(); + assert_eq!(payload.targets.len(), 1); + assert_eq!(payload.targets[0].destination, test.target); + assert_eq!(payload.targets[0].phase, RepairTargetPhase::Allocated); + let owner = SegmentOwnerResolver::new(Arc::clone(&test.harness.handler), Arc::clone(&test.tasks)); + assert_eq!( + owner + .resolve(&test.chunk.id.unwrap(), &test.target) + .await + .unwrap(), + SegmentOwnerDisposition::TaskPending + ); +} + +#[tokio::test] +async fn published_repair_target_survives_expiry_before_its_task_checkpoint() { + let mut test = TestRepairTargets::start().await; + let published = test.publish_before_checkpoint().await; + test.stop_target(true).await; + let task = test.execute().await; + assert_eq!(task.state, ChunkTaskState::Completed, "{}", task.last_error); + let payload = decode_payload(&task.payload).unwrap(); + assert_eq!(payload.targets.len(), 1); + assert_eq!(payload.targets[0].destination, test.target); + assert_eq!(payload.targets[0].phase, RepairTargetPhase::Confirmed); + assert_eq!( + test.harness + .handler + .query_chunk(&test.chunk.id.unwrap()) + .await + .unwrap(), + published + ); +} diff --git a/app/crowdb-web/src/diskdb/remote.rs b/app/crowdb-web/src/diskdb/remote.rs index 5cad0e4d0..542aa022c 100644 --- a/app/crowdb-web/src/diskdb/remote.rs +++ b/app/crowdb-web/src/diskdb/remote.rs @@ -7,7 +7,6 @@ use crate::{ state::AppState, }; use axum::{http::StatusCode, Json}; -use crowdb_console_shared::deployment::services; use crowdb_protocol::mgmt::node::{NodeServiceAction, NodeServiceIntent}; use std::fmt::Write as _; @@ -37,7 +36,7 @@ pub(super) async fn deploy( configuration, environment: std::collections::BTreeMap::default(), }; - let reply = services::execute(state.kv_client().await.as_ref(), &node, intent) + let reply = crate::services::remote::execute(state, &node, intent) .await .map_err(map_config_err)?; crate::services::remote::refresh(state) diff --git a/app/crowdb-web/src/lib.rs b/app/crowdb-web/src/lib.rs index 47522a48b..85d5676f8 100644 --- a/app/crowdb-web/src/lib.rs +++ b/app/crowdb-web/src/lib.rs @@ -35,6 +35,13 @@ pub mod state; pub use node::observation::{start_node_observer, NodeObserver}; #[cfg(feature = "test-util")] pub use services::observation::remote_access_health_for_tests; +#[cfg(feature = "test-util")] +pub use services::observation::{ + invalidate_service_observation_for_tests, ServiceObservationMutationForTests, +}; +pub use services::observation::{start_service_observer, ServiceObserver}; +#[cfg(feature = "test-util")] +pub use services::observation::{start_service_observer_for_tests, ServiceObservationSourceForTests}; pub use state::AppState; #[cfg(feature = "test-util")] diff --git a/app/crowdb-web/src/main.rs b/app/crowdb-web/src/main.rs index 781346ffd..89e4eab1a 100644 --- a/app/crowdb-web/src/main.rs +++ b/app/crowdb-web/src/main.rs @@ -160,6 +160,7 @@ async fn main() -> Result<(), Box> { let listener = tokio::net::TcpListener::bind(addr).await?; let _node_observer = crowdb_web::start_node_observer(&state)?; + let _service_observer = crowdb_web::start_service_observer(&state)?; let auto_join = crowdb_web::start_node_auto_join(state.clone()); if args.test_mode { let cleanup_state = state.clone(); diff --git a/app/crowdb-web/src/services/deployment/remote.rs b/app/crowdb-web/src/services/deployment/remote.rs index cf0d6d7e6..5c4544a70 100644 --- a/app/crowdb-web/src/services/deployment/remote.rs +++ b/app/crowdb-web/src/services/deployment/remote.rs @@ -62,13 +62,7 @@ pub(super) async fn deploy( configuration, environment: owned_environment(state, body, &node)?, }; - let result = match crowdb_console_shared::deployment::services::execute( - state.kv_client().await.as_ref(), - &node, - intent, - ) - .await - { + let result = match crate::services::remote::execute(state, &node, intent).await { Ok(result) => result, Err(error) if error diff --git a/app/crowdb-web/src/services/observation.rs b/app/crowdb-web/src/services/observation.rs index aa0f2fea2..3b4eb70b7 100644 --- a/app/crowdb-web/src/services/observation.rs +++ b/app/crowdb-web/src/services/observation.rs @@ -2,8 +2,23 @@ // Licensed under the Apache License, Version 2.0. mod access; +mod cache; +mod mutation; +mod sampling; +mod source; +#[cfg(feature = "test-util")] +mod test_support; +mod worker; #[cfg(feature = "test-util")] pub use access::remote_access_health_for_tests; +pub(crate) use cache::ServiceObservations; +#[cfg(feature = "test-util")] +pub use test_support::{invalidate_service_observation_for_tests, ServiceObservationMutationForTests}; +pub use worker::{start_service_observer, ServiceObserver}; +#[cfg(feature = "test-util")] +pub use worker::{ + start_service_observer_for_tests, ServiceObservationSource as ServiceObservationSourceForTests, +}; use crate::state::AppState; use axum::{extract::State, Json}; @@ -49,59 +64,43 @@ pub struct ServerSummary { /// # Panics /// Panics if the `RwLock` is poisoned. pub async fn http_list_servers(State(state): State) -> Json> { - let intents = if state.node_monitor_url.is_some() - && crate::node::cluster_binding(&state).ok().flatten().is_some() - { - tokio::time::timeout( - std::time::Duration::from_secs(3), - super::remote::refresh_with_intents(&state), - ) - .await - .ok() - .and_then(Result::ok) - .unwrap_or_default() - } else { - Vec::new() - }; - let snap = state.monitor_cache.snapshot().await; let cfg = state.config.read().unwrap().clone(); let remote = state.node_monitor_url.is_some(); - let remote_pids = if remote { - super::remote::processes(&state, &cfg, &intents).await + let binding = remote + .then(|| crate::node::cluster_binding(&state).ok().flatten()) + .flatten(); + let observed = state.service_observations.latest.load_full(); + // Remote inventory readers never perform authority, SSH, or endpoint I/O. + let snap = if remote { + std::collections::BTreeMap::default() } else { - super::remote::ProcessObservations::default() + state.monitor_cache.snapshot().await }; let rows = futures::future::join_all(cfg.servers.iter().map(|s| { let state = &state; let cfg = &cfg; let snap = &snap; - let remote_pids = &remote_pids; - let intents = &intents; + let observed = &observed; + let binding = &binding; async move { - let runtime_pid = match s.node_id { - Some(node_id) if s.service_type == ServiceType::Diskdb => { - state.diskdb_runtime_pid(node_id.to_string()) - } - Some(node_id) if s.service_type == ServiceType::PaxosKv => { - state.runtime_pid(node_id.to_string()) - } - _ => None, - }; - let pid = if remote { - let process_id = if s.service_type == ServiceType::PaxosKv { - "kv" - } else { - &s.id - }; - s.node_id - .and_then(|node| remote_pids.pids.get(&(node, process_id.to_owned())).copied()) + let (pid, health, start_pending) = if remote { + observed.read(binding.as_ref(), s.node_id.and_then(|id| cfg.node(id)), s) } else { - runtime_pid.or_else(|| { + let runtime_pid = match s.node_id { + Some(node_id) if s.service_type == ServiceType::Diskdb => { + state.diskdb_runtime_pid(node_id.to_string()) + } + Some(node_id) if s.service_type == ServiceType::PaxosKv => { + state.runtime_pid(node_id.to_string()) + } + _ => None, + }; + let pid = runtime_pid.or_else(|| { s.pid .filter(|pid| crowdb_console_shared::lifecycle::process_is_alive(*pid)) - }) + }); + (pid, observe_health(state, cfg, s, pid, snap, &[]).await, false) }; - let health = observe_health(state, cfg, s, pid, snap, intents).await; ServerSummary { id: s.id.clone(), node_id: s.node_id, @@ -111,7 +110,7 @@ pub async fn http_list_servers(State(state): State) -> Json +// Licensed under the Apache License, Version 2.0. + +//! Identity-fenced facts sampled independently of inventory readers. + +use arc_swap::ArcSwap; +use crowdb_console_shared::{ + cluster::NodeHealth, + config::{NodeEntry, ServerEntry, ServiceType}, +}; +use crowdb_protocol::mgmt::node::{NodeBinding, NodeServiceAction, NodeServiceIntent}; +use std::{ + collections::BTreeMap, + sync::{atomic::AtomicBool, Arc}, + time::Duration, +}; +use tokio::{sync::Notify, time::Instant}; + +pub(super) const MAX_AGE: Duration = Duration::from_secs(3); +pub(super) type ServiceKey = (u64, String); + +#[derive(Default)] +pub(crate) struct ServiceObservations { + pub(crate) latest: ArcSwap, + pub(super) changed: Notify, + pub(super) running: AtomicBool, +} + +#[derive(Clone, Default)] +pub(crate) struct Inventory { + pub binding: Option, + pub authority_epoch: u64, + pub authority: Option, + pub(super) epochs: BTreeMap, + pub(super) processes: BTreeMap, + pub(super) health: BTreeMap, +} + +#[derive(Clone)] +pub(crate) struct Authority { + pub started: Instant, + pub intents: Vec, +} + +#[derive(Clone)] +pub(super) struct Sample { + pub binding: NodeBinding, + pub node: NodeEntry, + pub entry: ServerEntry, + pub intent: Option, + pub epoch: u64, +} + +#[derive(Clone)] +pub(super) struct ProcessFact { + pub(super) sample: Sample, + pub(super) started: Instant, + pub(super) pid: Option, + pub(super) exhausted: bool, +} + +#[derive(Clone)] +pub(super) struct HealthFact { + sample: Sample, + started: Instant, + pub(super) health: NodeHealth, +} + +impl Inventory { + pub fn intent(&self, entry: &ServerEntry) -> Option<&NodeServiceIntent> { + self.authority.as_ref()?.intents.iter().find(|intent| { + Some(intent.node_id) == entry.node_id + && (intent.service_id == entry.id + || (entry.service_type == ServiceType::PaxosKv && intent.kind == "kv")) + }) + } + + pub(super) fn sample(&self, binding: &NodeBinding, node: &NodeEntry, entry: &ServerEntry) -> Sample { + Sample { + binding: binding.clone(), + node: node.clone(), + entry: entry.clone(), + intent: self.intent(entry).cloned(), + epoch: self.epoch(node.id, &entry.id), + } + } + + pub(super) fn epoch(&self, node: u64, service: &str) -> u64 { + self.epochs.get(&(node, service.to_owned())).copied().unwrap_or(0) + } + + fn accepts(&self, sample: &Sample) -> bool { + self.binding.as_ref() == Some(&sample.binding) + && self.epoch(sample.node.id, &sample.entry.id) == sample.epoch + && self.intent(&sample.entry) == sample.intent.as_ref() + } + + pub fn read( + &self, + binding: Option<&NodeBinding>, + node: Option<&NodeEntry>, + entry: &ServerEntry, + ) -> (Option, NodeHealth, bool) { + let Some(node) = node.filter(|_| self.binding.as_ref() == binding && binding.is_some()) else { + return (None, NodeHealth::Unknown, false); + }; + let key = (node.id, entry.id.clone()); + let valid = |sample: &Sample| self.accepts(sample) && sample.node == *node && sample.entry == *entry; + let process = self + .processes + .get(&key) + .filter(|fact| valid(&fact.sample) && fact.started.elapsed() <= MAX_AGE); + let pid = process.and_then(|fact| fact.pid); + let fresh_intent = self + .authority + .as_ref() + .is_some_and(|authority| authority.started.elapsed() <= MAX_AGE); + let health = self + .health + .get(&key) + .filter(|fact| valid(&fact.sample) && fact.started.elapsed() <= MAX_AGE) + .filter(|_| entry.service_type != ServiceType::AccessServer || fresh_intent) + .map_or(NodeHealth::Unknown, |fact| fact.health); + let pending = fresh_intent + && pid.is_none() + && self.intent(entry).is_some_and(|intent| { + matches!( + intent.action, + NodeServiceAction::Start | NodeServiceAction::Restart + ) + }) + && !process.is_some_and(|fact| fact.exhausted); + (pid, health, pending) + } +} + +impl ServiceObservations { + pub fn synchronize(&self, binding: Option<&NodeBinding>) { + self.latest.rcu(|current| { + if current.binding.as_ref() == binding { + return Arc::clone(current); + } + Arc::new(Inventory { + binding: binding.cloned(), + authority_epoch: current.authority_epoch + 1, + ..Inventory::default() + }) + }); + } + + pub fn authority( + &self, + binding: &NodeBinding, + epoch: u64, + started: Instant, + intents: &[NodeServiceIntent], + ) -> bool { + let mut accepted = false; + let mut changed = false; + self.latest.rcu(|current| { + accepted = current.binding.as_ref() == Some(binding) + && current.authority_epoch == epoch + && current + .authority + .as_ref() + .map_or(true, |authority| started >= authority.started); + if !accepted { + changed = false; + return Arc::clone(current); + } + changed = current + .authority + .as_ref() + .map_or(true, |authority| authority.intents != intents); + let mut next = (**current).clone(); + next.authority = Some(Authority { + started, + intents: intents.to_vec(), + }); + Arc::new(next) + }); + if changed { + self.changed.notify_waiters(); + } + accepted + } + + pub(super) fn process(&self, sample: &Sample, started: Instant, pid: Option, exhausted: bool) { + self.latest.rcu(|current| { + if !current.accepts(sample) { + return Arc::clone(current); + } + let mut next = (**current).clone(); + next.processes.insert( + (sample.node.id, sample.entry.id.clone()), + ProcessFact { + sample: sample.clone(), + started, + pid, + exhausted, + }, + ); + Arc::new(next) + }); + } + + pub(super) fn health(&self, sample: &Sample, started: Instant, health: NodeHealth) { + self.latest.rcu(|current| { + if !current.accepts(sample) { + return Arc::clone(current); + } + let mut next = (**current).clone(); + next.health.insert( + (sample.node.id, sample.entry.id.clone()), + HealthFact { + sample: sample.clone(), + started, + health, + }, + ); + Arc::new(next) + }); + } +} diff --git a/app/crowdb-web/src/services/observation/mutation.rs b/app/crowdb-web/src/services/observation/mutation.rs new file mode 100644 index 000000000..b94a59b0b --- /dev/null +++ b/app/crowdb-web/src/services/observation/mutation.rs @@ -0,0 +1,170 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Fence observations across accepted, failed, and cancelled lifecycle work. + +use super::cache::{Authority, ProcessFact, ServiceKey, ServiceObservations}; +use crate::state::AppState; +use crowdb_console_shared::config::ServiceType; +use crowdb_protocol::mgmt::node::{NodeBinding, NodeServiceIntent}; +use std::sync::Arc; +use tokio::time::Instant; + +impl ServiceObservations { + pub fn mutation(state: &AppState, node: u64, service: &str) -> Mutation { + let requested = service.to_owned(); + let config = state.config.read().unwrap(); + let service = config + .servers + .iter() + .find(|entry| { + entry.node_id == Some(node) + && (entry.id == service + || (entry.service_type == ServiceType::PaxosKv && service == format!("kv-{node}"))) + }) + .map_or_else(|| service.to_owned(), |entry| entry.id.clone()); + drop(config); + let key = (node, service); + let binding = crate::node::cluster_binding(state).ok().flatten(); + state.service_observations.synchronize(binding.as_ref()); + let mut epoch = 0; + let mut previous_operation = None; + state.service_observations.latest.rcu(|current| { + previous_operation = current.authority.as_ref().and_then(|authority| { + authority + .intents + .iter() + .find(|intent| intent.node_id == node && intent.service_id == requested) + .map(|intent| intent.operation_id.clone()) + }); + let mut next = (**current).clone(); + *next.epochs.entry(key.clone()).or_default() += 1; + epoch = next.epoch(node, &key.1); + next.authority_epoch += 1; + next.processes.remove(&key); + next.health.remove(&key); + Arc::new(next) + }); + Mutation { + observations: Arc::clone(&state.service_observations), + key, + epoch, + binding, + previous_operation, + started: Instant::now(), + } + } +} + +/// Cancellation and unknown outcomes fence in-flight reads without replaying mutations. +pub(crate) struct Mutation { + observations: Arc, + key: ServiceKey, + epoch: u64, + binding: Option, + previous_operation: Option, + started: Instant, +} + +impl Mutation { + pub(crate) fn complete( + &self, + state: &AppState, + mut intent: NodeServiceIntent, + result: &serde_json::Value, + ) { + let Some(operation) = result["operation_id"].as_str() else { + return; + }; + operation.clone_into(&mut intent.operation_id); + let Some(binding) = self + .binding + .as_ref() + .filter(|binding| crate::node::cluster_binding(state).ok().flatten().as_ref() == Some(*binding)) + else { + return; + }; + let config = state.config.read().unwrap().clone(); + let Some(node) = config.node(self.key.0) else { + return; + }; + let mut accepted = false; + self.observations.latest.rcu(|current| { + accepted = false; + if current.binding.as_ref() != Some(binding) + || current.epoch(self.key.0, &self.key.1) != self.epoch + { + return Arc::clone(current); + } + let known = current.authority.as_ref().and_then(|authority| { + authority + .intents + .iter() + .find(|known| known.service_id == intent.service_id && known.node_id == intent.node_id) + }); + if known.is_some_and(|known| { + known.operation_id != intent.operation_id + && Some(&known.operation_id) != self.previous_operation.as_ref() + }) { + return Arc::clone(current); + } + let mut next = (**current).clone(); + next.authority_epoch += 1; + *next.epochs.entry(self.key.clone()).or_default() += 1; + let authority = next.authority.get_or_insert_with(|| Authority { + started: self.started, + intents: Vec::new(), + }); + authority + .intents + .retain(|known| known.service_id != intent.service_id); + authority.intents.push(intent.clone()); + let entry = crate::services::remote::project(&intent).or_else(|| { + config + .servers + .iter() + .find(|entry| entry.id == self.key.1) + .cloned() + }); + if let Some(entry) = entry { + let sample = next.sample(binding, node, &entry); + let pid = result["pid"] + .as_u64() + .and_then(|pid| u32::try_from(pid).ok()) + .filter(|pid| *pid > 0); + next.processes.insert( + self.key.clone(), + ProcessFact { + sample, + started: self.started, + pid, + exhausted: false, + }, + ); + } + next.health.remove(&self.key); + accepted = true; + Arc::new(next) + }); + if accepted { + crate::services::remote::project_intents(state, &[intent]); + } + } +} + +impl Drop for Mutation { + fn drop(&mut self) { + self.observations.latest.rcu(|current| { + if current.binding != self.binding || current.epoch(self.key.0, &self.key.1) != self.epoch { + return Arc::clone(current); + } + let mut next = (**current).clone(); + *next.epochs.entry(self.key.clone()).or_default() += 1; + next.authority_epoch += 1; + next.processes.remove(&self.key); + next.health.remove(&self.key); + Arc::new(next) + }); + self.observations.changed.notify_waiters(); + } +} diff --git a/app/crowdb-web/src/services/observation/sampling.rs b/app/crowdb-web/src/services/observation/sampling.rs new file mode 100644 index 000000000..3a42f3fb0 --- /dev/null +++ b/app/crowdb-web/src/services/observation/sampling.rs @@ -0,0 +1,171 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Each monitor and each endpoint has its own sampling cadence. + +use super::{ + cache::{Sample, MAX_AGE}, + source::ServiceObservationSource, + worker::phase_error, +}; +use crate::state::AppState; +use crowdb_console_shared::{ + cluster::NodeHealth, + config::{NodeEntry, ServerEntry, ServiceType}, +}; +use crowdb_protocol::mgmt::node::NodeBinding; +use std::{sync::Arc, time::Duration}; +use tokio::time::{Instant, MissedTickBehavior}; + +pub(super) async fn sample_node( + state: AppState, + source: Arc, + binding: NodeBinding, + node: NodeEntry, +) { + let mut interval = tokio::time::interval(Duration::from_secs(1)); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + tokio::select! { _ = interval.tick() => {}, () = state.service_observations.changed.notified() => {} } + let config = state.config.read().unwrap().clone(); + if crate::node::cluster_binding(&state).ok().flatten().as_ref() != Some(&binding) + || config.node(node.id) != Some(&node) + { + return; + } + let observed = state.service_observations.latest.load_full(); + let samples: Vec<_> = config + .servers + .iter() + .filter(|entry| entry.node_id == Some(node.id)) + .map(|entry| observed.sample(&binding, &node, entry)) + .collect(); + sample_processes(&state, source.as_ref(), &binding, &node, &samples).await; + } +} + +async fn sample_processes( + state: &AppState, + source: &dyn ServiceObservationSource, + binding: &NodeBinding, + node: &NodeEntry, + samples: &[Sample], +) { + let started = Instant::now(); + let result = match tokio::time::timeout(MAX_AGE, source.processes(state, binding, node)).await { + Ok(Ok(value)) => Some(value), + Ok(Err(error)) => { + phase_error(state, node.id, "processes", started, &error); + None + } + Err(_) => { + phase_error( + state, + node.id, + "processes", + started, + "three-second sampling deadline exceeded", + ); + None + } + }; + let bootstrap = serde_json::to_value(&binding.bootstrap).expect("bootstrap serialization is infallible"); + let value = result.filter(|value| { + value["node_id"] == node.id + && value["cluster_id"] == binding.bootstrap.cluster_id + && value["bootstrap"] == bootstrap + }); + if value.is_none() { + tracing::debug!( + node_id = node.id, + phase = "processes", + "no current authenticated process observation" + ); + } + tracing::debug!( + node_id = node.id, + phase = "processes", + elapsed_ms = started.elapsed().as_millis(), + "service inventory sampled" + ); + for sample in samples { + let id = if sample.entry.service_type == ServiceType::PaxosKv { + "kv" + } else { + &sample.entry.id + }; + let owned = value.as_ref().filter(|value| { + sample + .intent + .as_ref() + .map_or(sample.entry.service_type == ServiceType::PaxosKv, |intent| { + value["process_operations"][id] == intent.operation_id + }) + }); + let pid = owned + .and_then(|value| value["processes"][id].as_u64()) + .and_then(|pid| u32::try_from(pid).ok()) + .filter(|pid| *pid > 0); + let exhausted = value + .as_ref() + .zip(sample.intent.as_ref()) + .is_some_and(|(value, intent)| { + value["service_failures"][&intent.service_id]["operation_id"] == intent.operation_id + && value["service_failures"][&intent.service_id]["exhausted"] == true + }); + state + .service_observations + .process(sample, started, pid, exhausted); + } +} + +pub(super) async fn sample_service_health( + state: AppState, + source: Arc, + binding: NodeBinding, + node: NodeEntry, + entry: ServerEntry, +) { + let mut interval = tokio::time::interval(Duration::from_secs(1)); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + tokio::select! { _ = interval.tick() => {}, () = state.service_observations.changed.notified() => {} } + let config = state.config.read().unwrap().clone(); + if crate::node::cluster_binding(&state).ok().flatten().as_ref() != Some(&binding) + || config.node(node.id) != Some(&node) + || !config.servers.contains(&entry) + { + return; + } + let observed = state.service_observations.latest.load_full(); + let sample = observed.sample(&binding, &node, &entry); + let intents = observed + .authority + .as_ref() + .filter(|authority| authority.started.elapsed() <= MAX_AGE) + .map_or(&[][..], |authority| authority.intents.as_slice()); + let started = Instant::now(); + let pid = observed + .read(Some(&sample.binding), Some(&sample.node), &sample.entry) + .0; + let response = tokio::select! { + biased; + () = state.service_observations.changed.notified() => continue, + result = tokio::time::timeout(MAX_AGE, source.health(&state, &config, &sample.entry, pid, intents)) => result, + }; + let health = if let Ok(health) = response { + health + } else { + phase_error( + &state, + sample.node.id, + "health", + started, + "three-second sampling deadline exceeded", + ); + NodeHealth::Unknown + }; + tracing::debug!(node_id = sample.node.id, service_id = %sample.entry.id, phase = "health", elapsed_ms = started.elapsed().as_millis(), ?health, "service endpoint sampled"); + state.service_observations.health(&sample, started, health); + } +} diff --git a/app/crowdb-web/src/services/observation/source.rs b/app/crowdb-web/src/services/observation/source.rs new file mode 100644 index 000000000..9d3f0c549 --- /dev/null +++ b/app/crowdb-web/src/services/observation/source.rs @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Authenticated process and real endpoint transports for inventory sampling. + +use crate::state::AppState; +use crowdb_console_shared::{ + cluster::NodeHealth, + config::{NodeEntry, ServerEntry}, + ConsoleConfig, +}; +use crowdb_protocol::mgmt::node::{NodeBinding, NodeControl, NodeServiceIntent}; +use serde_json::Value; + +/// Observation transports; injected only by focused regression fixtures. +#[async_trait::async_trait] +pub trait ServiceObservationSource: Send + Sync + 'static { + /// Read the committed service view without accepting mutations. + async fn intents(&self, state: &AppState) -> Result, String>; + /// Read authenticated, bound monitor-owned children. + async fn processes( + &self, + state: &AppState, + binding: &NodeBinding, + node: &NodeEntry, + ) -> Result; + /// Probe the service's actual endpoint, independently of process presence. + async fn health( + &self, + state: &AppState, + config: &ConsoleConfig, + entry: &ServerEntry, + pid: Option, + intents: &[NodeServiceIntent], + ) -> NodeHealth; +} + +pub(super) struct Production; +#[async_trait::async_trait] +impl ServiceObservationSource for Production { + async fn intents(&self, state: &AppState) -> Result, String> { + crowdb_console_shared::deployment::services::list(state.kv_client().await.as_ref()) + .await + .map_err(|error| error.to_string()) + } + async fn processes( + &self, + state: &AppState, + binding: &NodeBinding, + node: &NodeEntry, + ) -> Result { + let command = NodeControl::Processes { + cluster_id: binding.bootstrap.cluster_id.clone(), + node_id: node.id, + }; + let result = if node.id == binding.node_id { + crowdb_console_shared::deployment::admission::local_control( + &crate::node::control_socket(), + &command, + ) + .await + } else { + let mut target = node.clone(); + target.ssh_key = Some(crate::node::node_key_path(state).to_string_lossy().into_owned()); + crowdb_console_shared::deployment::admission::remote_control(&target, &command).await + }; + result.map_err(|error| error.to_string()) + } + async fn health( + &self, + state: &AppState, + config: &ConsoleConfig, + entry: &ServerEntry, + pid: Option, + intents: &[NodeServiceIntent], + ) -> NodeHealth { + let snapshot = state.monitor_cache.snapshot().await; + super::observe_health(state, config, entry, pid, &snapshot, intents).await + } +} diff --git a/app/crowdb-web/src/services/observation/test_support.rs b/app/crowdb-web/src/services/observation/test_support.rs new file mode 100644 index 000000000..fd74ad7b3 --- /dev/null +++ b/app/crowdb-web/src/services/observation/test_support.rs @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{cache::ServiceObservations, mutation::Mutation}; +use crate::AppState; +use crowdb_protocol::mgmt::node::NodeServiceIntent; + +/// Holds the production observation fence while a fixture controls a mutation. +pub struct ServiceObservationMutationForTests { + state: AppState, + mutation: Mutation, +} + +/// Invalidate one target using the same fence as remote lifecycle execution. +#[must_use] +pub fn invalidate_service_observation_for_tests( + state: &AppState, + node: u64, + service: &str, +) -> ServiceObservationMutationForTests { + ServiceObservationMutationForTests { + state: state.clone(), + mutation: ServiceObservations::mutation(state, node, service), + } +} + +impl ServiceObservationMutationForTests { + /// Publish an actual monitor reply through the production completion path. + pub fn complete(&self, intent: NodeServiceIntent, reply: &serde_json::Value) { + self.mutation.complete(&self.state, intent, reply); + } +} diff --git a/app/crowdb-web/src/services/observation/worker.rs b/app/crowdb-web/src/services/observation/worker.rs new file mode 100644 index 000000000..3ec5777f2 --- /dev/null +++ b/app/crowdb-web/src/services/observation/worker.rs @@ -0,0 +1,198 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Authority and each node publish independently; reads never drive sampling. + +use super::{ + cache::MAX_AGE, + sampling::{sample_node, sample_service_health}, + source::Production, + ServiceObservations, +}; +use crate::state::AppState; +use crowdb_console_shared::config::{NodeEntry, ServerEntry}; +use crowdb_protocol::mgmt::node::NodeBinding; +use std::{ + collections::BTreeMap, + sync::{atomic::Ordering, Arc}, + time::Duration, +}; +use tokio::{ + task::{AbortHandle, JoinSet}, + time::{Instant, MissedTickBehavior}, +}; + +pub use super::source::ServiceObservationSource; + +/// Stops the coordinator and all its node samplers when the Web listener exits. +pub struct ServiceObserver(tokio::task::JoinHandle<()>); +impl Drop for ServiceObserver { + fn drop(&mut self) { + self.0.abort(); + } +} +struct SamplingLease(Arc); +impl Drop for SamplingLease { + fn drop(&mut self) { + self.0.running.store(false, Ordering::Release); + } +} + +/// Start independent service inventory observations for monitor-owned Web. +/// +/// # Errors +/// Rejects an already running observer. +pub fn start_service_observer(state: &AppState) -> Result, &'static str> { + if state.node_monitor_url.is_none() { + return Ok(None); + } + start(state, Arc::new(Production)).map(Some) +} + +/// Start the production sampler with deterministic observation transports. +/// +/// # Errors +/// Rejects an unowned Web or a duplicate observer. +#[cfg(feature = "test-util")] +pub fn start_service_observer_for_tests( + state: &AppState, + source: Arc, +) -> Result { + start(state, source) +} + +fn start( + state: &AppState, + source: Arc, +) -> Result { + if state.node_monitor_url.is_none() { + return Err("Service sampling requires an owning monitor"); + } + state + .service_observations + .running + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| "Service observation sampler is already running")?; + let state = state.clone(); + let lease = SamplingLease(Arc::clone(&state.service_observations)); + Ok(ServiceObserver(tokio::spawn(async move { + let _lease = lease; + coordinate(state, source).await; + }))) +} + +async fn coordinate(state: AppState, source: Arc) { + let mut children = JoinSet::new(); + let mut authority_task = children.spawn(authority(state.clone(), Arc::clone(&source))); + let mut nodes: BTreeMap = BTreeMap::new(); + let mut health: BTreeMap<(u64, String), (NodeBinding, NodeEntry, ServerEntry, AbortHandle)> = + BTreeMap::new(); + let mut interval = tokio::time::interval(Duration::from_millis(100)); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + tokio::select! { + _ = interval.tick() => { + if authority_task.is_finished() { + authority_task = children.spawn(authority(state.clone(), Arc::clone(&source))); + } + let binding = crate::node::cluster_binding(&state).ok().flatten(); + state.service_observations.synchronize(binding.as_ref()); + let config = state.config.read().unwrap().clone(); + nodes.retain(|id, (old_binding, old_node, task)| { + let retain = !task.is_finished() && binding.as_ref() == Some(old_binding) && config.node(*id) == Some(old_node); + if !retain { task.abort(); } + retain + }); + health.retain(|(node, id), (old_binding, old_node, entry, task)| { + let retain = !task.is_finished() && binding.as_ref() == Some(old_binding) + && config.node(*node) == Some(old_node) && config.servers.iter().any(|current| current.id == *id && current == entry); + if !retain { task.abort(); } + retain + }); + if let Some(binding) = binding { + for node in &config.nodes { + if nodes.contains_key(&node.id) { continue; } + let handle = children.spawn(sample_node(state.clone(), Arc::clone(&source), binding.clone(), node.clone())); + nodes.insert(node.id, (binding.clone(), node.clone(), handle)); + } + for entry in &config.servers { + let Some(node) = entry.node_id.and_then(|id| config.node(id)) else { continue; }; + let key = (node.id, entry.id.clone()); + if health.contains_key(&key) { continue; } + let handle = children.spawn(sample_service_health(state.clone(), Arc::clone(&source), binding.clone(), node.clone(), entry.clone())); + health.insert(key, (binding.clone(), node.clone(), entry.clone(), handle)); + } + } + } + result = children.join_next(), if !children.is_empty() => { + if let Some(Err(error)) = result { + if !error.is_cancelled() { tracing::error!(%error, "service observation worker stopped; unavailable observations expire"); } + } + } + } + } +} + +async fn authority(state: AppState, source: Arc) { + let mut interval = tokio::time::interval(Duration::from_secs(1)); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + tokio::select! { _ = interval.tick() => {}, () = state.service_observations.changed.notified() => {} } + let Some(binding) = crate::node::cluster_binding(&state).ok().flatten() else { + continue; + }; + state.service_observations.synchronize(Some(&binding)); + let epoch = state.service_observations.latest.load().authority_epoch; + let started = Instant::now(); + let intents = match tokio::time::timeout(MAX_AGE, source.intents(&state)).await { + Ok(Ok(intents)) => intents, + Ok(Err(error)) => { + phase_error(&state, binding.node_id, "authority", started, &error); + continue; + } + Err(_) => { + phase_error( + &state, + binding.node_id, + "authority", + started, + "three-second sampling deadline exceeded", + ); + continue; + } + }; + tracing::debug!( + phase = "authority", + elapsed_ms = started.elapsed().as_millis(), + "service inventory sampled" + ); + if crate::node::cluster_binding(&state).ok().flatten().as_ref() != Some(&binding) { + continue; + } + let intents: Vec<_> = intents + .into_iter() + .filter(|intent| intent.cluster_id == binding.bootstrap.cluster_id) + .collect(); + if state + .service_observations + .authority(&binding, epoch, started, &intents) + { + crate::services::remote::project_intents(&state, &intents); + } + } +} + +pub(super) fn phase_error(state: &AppState, node: u64, phase: &str, started: Instant, error: &str) { + let diagnostic: String = error + .chars() + .filter(|character| !character.is_control()) + .take(256) + .collect(); + if state.should_warn( + &format!("service-observation/{node}/{phase}"), + Duration::from_secs(30), + ) { + tracing::warn!(node_id = node, phase, elapsed_ms = started.elapsed().as_millis(), error = %diagnostic, + "service observation unavailable; stale evidence expires and the next sample retries the read"); + } +} diff --git a/app/crowdb-web/src/services/remote.rs b/app/crowdb-web/src/services/remote.rs index 86fa5267a..78ab8295f 100644 --- a/app/crowdb-web/src/services/remote.rs +++ b/app/crowdb-web/src/services/remote.rs @@ -13,82 +13,9 @@ use crowdb_console_shared::{ config::{ServerEntry, ServiceType}, deployment::services, }; -use crowdb_protocol::mgmt::node::{NodeControl, NodeServiceAction, NodeServiceIntent}; +use crowdb_protocol::mgmt::node::{NodeServiceAction, NodeServiceIntent}; use serde_json::{json, Value}; -#[derive(Default)] -pub(super) struct ProcessObservations { - pub pids: std::collections::BTreeMap<(u64, String), u32>, - exhausted: std::collections::BTreeSet<(u64, String)>, -} - -impl ProcessObservations { - fn record(&mut self, node: u64, value: &Value, intents: &[NodeServiceIntent]) { - if let Some(processes) = value["processes"].as_object() { - for (id, value) in processes { - if let Some(pid) = value - .as_u64() - .and_then(|pid| u32::try_from(pid).ok()) - .filter(|pid| *pid > 0) - { - self.pids.insert((node, id.clone()), pid); - } - } - } - for intent in intents.iter().filter(|intent| intent.node_id == node) { - let failure = &value["service_failures"][&intent.service_id]; - if failure["operation_id"] == intent.operation_id && failure["exhausted"] == true { - self.exhausted.insert((node, intent.service_id.clone())); - } - } - } - - pub fn start_pending(&self, entry: &ServerEntry, remote: bool, pid: Option) -> bool { - remote - && entry.auto_start - && pid.is_none() - && !entry - .node_id - .is_some_and(|node| self.exhausted.contains(&(node, entry.id.clone()))) - } -} - -pub(super) async fn processes( - state: &AppState, - config: &crowdb_console_shared::ConsoleConfig, - intents: &[NodeServiceIntent], -) -> ProcessObservations { - let Some(binding) = crate::node::cluster_binding(state).ok().flatten() else { - return ProcessObservations::default(); - }; - let cluster = &binding.bootstrap.cluster_id; - let observations = futures::future::join_all(config.nodes.iter().map(|node| async move { - let mut target = node.clone(); - target.ssh_key = Some(crate::node::node_key_path(state).to_string_lossy().into_owned()); - let command = NodeControl::Processes { - cluster_id: cluster.clone(), - node_id: node.id, - }; - let result = tokio::time::timeout( - std::time::Duration::from_secs(3), - crowdb_console_shared::deployment::admission::remote_control(&target, &command), - ) - .await - .ok()? - .ok()?; - if result["node_id"] != node.id || result["cluster_id"] != *cluster { - return None; - } - Some((node.id, result)) - })) - .await; - let mut processes = ProcessObservations::default(); - for (node, observation) in observations.into_iter().flatten() { - processes.record(node, &observation, intents); - } - processes -} - pub(crate) async fn act( state: &AppState, id: &str, @@ -115,9 +42,7 @@ pub(crate) async fn act( node.ssh_key = Some(crate::node::node_key_path(state).to_string_lossy().into_owned()); intent.operation_id = uuid::Uuid::new_v4().to_string(); intent.action = action; - let result = services::execute(state.kv_client().await.as_ref(), &node, intent) - .await - .map_err(map_config_err)?; + let result = execute(state, &node, intent).await.map_err(map_config_err)?; refresh(state).await.map_err(map_config_err)?; Ok(Json( json!({"id": id, "pid":result["pid"], "removed":matches!(action, NodeServiceAction::Delete), "data_preserved":true}), @@ -131,12 +56,43 @@ pub(crate) async fn refresh(state: &AppState) -> crowdb_console_shared::error::R pub(super) async fn refresh_with_intents( state: &AppState, ) -> crowdb_console_shared::error::Result> { - let intents = services::list(state.kv_client().await.as_ref()).await?; - let mut config = state - .config - .write() - .map_err(|error| crowdb_console_shared::error::Error::Config(error.to_string()))?; - for intent in &intents { + let binding = crate::node::cluster_binding(state).ok().flatten(); + state.service_observations.synchronize(binding.as_ref()); + let epoch = state.service_observations.latest.load().authority_epoch; + let started = tokio::time::Instant::now(); + let intents: Vec<_> = services::list(state.kv_client().await.as_ref()) + .await? + .into_iter() + .filter(|intent| { + binding + .as_ref() + .is_some_and(|binding| intent.cluster_id == binding.bootstrap.cluster_id) + }) + .collect(); + if binding.as_ref().is_some_and(|binding| { + crate::node::cluster_binding(state).ok().flatten().as_ref() == Some(binding) + && state + .service_observations + .authority(binding, epoch, started, &intents) + }) { + project_intents(state, &intents); + } + Ok(intents) +} + +pub(super) fn project_intents(state: &AppState, intents: &[NodeServiceIntent]) { + let mut config = state.config.write().unwrap(); + let current = state.service_observations.latest.load_full(); + if crate::node::cluster_binding(state).ok().flatten() != current.binding { + return; + } + let Some(authority) = ¤t.authority else { + return; + }; + for intent in intents { + if !authority.intents.contains(intent) { + continue; + } // KV endpoints come from admitted node bootstrap, not auxiliary // service configuration. Its lifecycle intent must not erase them. if intent.kind == "kv" { @@ -150,10 +106,9 @@ pub(super) async fn refresh_with_intents( config.servers.push(entry); } } - Ok(intents) } -fn project(intent: &NodeServiceIntent) -> Option { +pub(super) fn project(intent: &NodeServiceIntent) -> Option { let value: toml::Value = toml::from_str(&intent.configuration).ok()?; let (kind, http, rpc) = match intent.kind.as_str() { "diskdb" => ( @@ -259,11 +214,16 @@ pub(crate) async fn kv_action( configuration: format!("node_id = {node_id}\n"), environment: std::collections::BTreeMap::new(), }; - services::execute(state.kv_client().await.as_ref(), &node, intent) - .await - .map_err(map_config_err) + execute(state, &node, intent).await.map_err(map_config_err) } -#[cfg(test)] -#[path = "../../tests/internal/service_start_pending_test.rs"] -mod tests; +pub(crate) async fn execute( + state: &AppState, + node: &crowdb_console_shared::config::NodeEntry, + intent: NodeServiceIntent, +) -> crowdb_console_shared::error::Result { + let mutation = super::observation::ServiceObservations::mutation(state, node.id, &intent.service_id); + let result = services::execute(state.kv_client().await.as_ref(), node, intent.clone()).await?; + mutation.complete(state, intent, &result); + Ok(result) +} diff --git a/app/crowdb-web/src/state.rs b/app/crowdb-web/src/state.rs index 4a0428b51..2120606b8 100644 --- a/app/crowdb-web/src/state.rs +++ b/app/crowdb-web/src/state.rs @@ -22,6 +22,7 @@ use crowdb_console_shared::{config::ServerEntry, ConsoleConfig}; #[derive(Clone)] pub struct AppState { pub(crate) node_observations: Arc, + pub(crate) service_observations: Arc, pub(crate) service_operations: Arc>>, pub config: Arc>, pub(crate) config_path: Option, @@ -100,6 +101,7 @@ impl AppState { pub fn with_runtime_root(config: ConsoleConfig, runtime_root: PathBuf) -> Self { Self { node_observations: Arc::default(), + service_observations: Arc::default(), service_operations: Arc::new(arc_swap::ArcSwap::from_pointee(std::collections::HashSet::new())), config: Arc::new(RwLock::new(config)), config_path: None, diff --git a/app/crowdb-web/tests/common/service_inventory.rs b/app/crowdb-web/tests/common/service_inventory.rs new file mode 100644 index 000000000..db9b2e857 --- /dev/null +++ b/app/crowdb-web/tests/common/service_inventory.rs @@ -0,0 +1,270 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{ + body::{to_bytes, Body}, + http::Request, +}; +use crowdb_console_shared::{ + cluster::NodeHealth, + config::{NodeEntry, ServerEntry, ServiceType}, + ConsoleConfig, +}; +use crowdb_protocol::mgmt::{ + node::{NodeBinding, NodeServiceAction, NodeServiceIntent}, + SystemBootstrapIdentity, +}; +use crowdb_web::{AppState, ServiceObservationSourceForTests, ServiceObserver}; +use serde_json::{json, Value}; +use std::{ + collections::BTreeMap, + sync::{ + atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::sync::Notify; +use tower::ServiceExt; + +pub struct TestSource { + pub intents: arc_swap::ArcSwap>, + pub fail_authority: AtomicBool, + pub hold_authority: AtomicBool, + pub authority_entered: Notify, + pub authority_release: Notify, + pub authority_finished: Notify, + pub hold_node: AtomicU32, + pub process_entered: Notify, + pub process_release: Notify, + pub process_finished: Notify, + pub hold_health: AtomicBool, + pub health_entered: Notify, + pub health_release: Notify, + pub pid: AtomicU32, + pub wrong_bootstrap: AtomicBool, + pub wrong_operation: AtomicBool, + pub exhausted: AtomicBool, + pub calls: AtomicUsize, +} + +#[async_trait::async_trait] +impl ServiceObservationSourceForTests for TestSource { + async fn intents(&self, _: &AppState) -> Result, String> { + self.calls.fetch_add(1, Ordering::Relaxed); + if self.fail_authority.load(Ordering::Relaxed) { + return Err("fixture authority unavailable".into()); + } + let result = self.intents.load_full().as_ref().clone(); + if self.hold_authority.load(Ordering::Relaxed) { + self.authority_entered.notify_one(); + self.authority_release.notified().await; + self.authority_finished.notify_one(); + } + Ok(result) + } + + async fn processes( + &self, + _: &AppState, + binding: &NodeBinding, + node: &NodeEntry, + ) -> Result { + self.calls.fetch_add(1, Ordering::Relaxed); + let intent = self + .intents + .load_full() + .iter() + .find(|intent| intent.node_id == node.id) + .unwrap() + .clone(); + let pid = self.pid.load(Ordering::Relaxed) + u32::try_from(node.id).unwrap(); + let wrong_operation = self.wrong_operation.load(Ordering::Relaxed); + let mut bootstrap = binding.bootstrap.clone(); + if self.wrong_bootstrap.load(Ordering::Relaxed) { + bootstrap.operation_id = "foreign-bootstrap".into(); + } + if self.hold_node.load(Ordering::Relaxed) == u32::try_from(node.id).unwrap() { + self.process_entered.notify_one(); + self.process_release.notified().await; + self.process_finished.notify_one(); + } + let operation = if wrong_operation { + "old-operation" + } else { + &intent.operation_id + }; + Ok( + json!({"node_id": node.id, "cluster_id": binding.bootstrap.cluster_id, "bootstrap": bootstrap, + "processes": if self.pid.load(Ordering::Relaxed) == 0 { json!({}) } else { json!({intent.service_id.clone(): pid}) }, + "process_operations": {intent.service_id.clone(): operation}, + "service_failures": {intent.service_id: {"operation_id":operation, "exhausted":self.exhausted.load(Ordering::Relaxed)}} }), + ) + } + + async fn health( + &self, + _: &AppState, + _: &ConsoleConfig, + entry: &ServerEntry, + _: Option, + _: &[NodeServiceIntent], + ) -> NodeHealth { + self.calls.fetch_add(1, Ordering::Relaxed); + if self.hold_health.load(Ordering::Relaxed) && entry.id == "chunkdb-sibling" { + self.health_entered.notify_one(); + self.health_release.notified().await; + } + NodeHealth::Up + } +} + +pub struct TestServiceInventory { + pub state: AppState, + pub source: Arc, + pub observer: Option, + pub root: crowdb_test_harness::test_dirs::TestDir, + app: axum::Router, +} + +impl TestServiceInventory { + pub fn new(access: bool) -> Self { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("service-inventory"); + let binding = NodeBinding { + node_id: 1, + bootstrap: SystemBootstrapIdentity { + cluster_id: "inventory-cluster".into(), + operation_id: "bootstrap".into(), + configuration_digest: "a".repeat(64), + }, + management_seeds: vec!["http://127.0.0.1:1".into()], + }; + std::fs::write( + root.path().join("node-binding.json"), + serde_json::to_vec(&binding).unwrap(), + ) + .unwrap(); + let mut config = ConsoleConfig::default(); + let mut intents = Vec::new(); + for node_id in [1, 2] { + config.nodes.push(NodeEntry { + id: node_id, + rack_id: 1, + host: format!("127.0.0.{node_id}"), + ssh_port: 22, + ssh_user: "fixture".into(), + ssh_credential_ref: None, + ssh_key: None, + ssh_password: None, + }); + let id = format!("{}-{node_id}", if access { "access" } else { "chunkdb" }); + let mut entry = ServerEntry::new(&id, format!("http://127.0.0.{node_id}:7300")); + entry.node_id = Some(node_id); + entry.service_type = if access { + ServiceType::AccessServer + } else { + ServiceType::Chunkdb + }; + entry.auto_start = true; + entry.rest_port = Some(7300); + if access { + entry.access_origins.insert("iceberg".into(), entry.url.clone()); + } else { + entry.rpc_url = Some(format!("127.0.0.{node_id}:7400")); + entry.rpc_port = Some(7400); + } + config.servers.push(entry); + intents.push(NodeServiceIntent { cluster_id: binding.bootstrap.cluster_id.clone(), + operation_id: format!("operation-{node_id}"), node_id, service_id: id, kind: if access { "access" } else { "chunkdb" }.into(), action: NodeServiceAction::Start, + configuration: if access { format!("[iceberg]\nlisten = '127.0.0.{node_id}:7300'\n[health]\nlisten = '127.0.0.{node_id}:7500'\n") } + else { format!("[server]\nhttp_listen_addr = '127.0.0.{node_id}:7300'\nrpc_listen_addr = '127.0.0.{node_id}:7400'\n") }, + environment: BTreeMap::default() }); + } + let state = AppState::with_runtime_root(config, root.path().join("console")) + .with_node_monitor("http://127.0.0.1:1") + .unwrap(); + let app = crowdb_web::router(state.clone()); + let source = Arc::new(TestSource { + intents: arc_swap::ArcSwap::from_pointee(intents), + fail_authority: AtomicBool::new(false), + hold_authority: AtomicBool::new(false), + authority_entered: Notify::new(), + authority_release: Notify::new(), + authority_finished: Notify::new(), + hold_node: AtomicU32::new(0), + process_entered: Notify::new(), + process_release: Notify::new(), + process_finished: Notify::new(), + hold_health: AtomicBool::new(false), + health_entered: Notify::new(), + health_release: Notify::new(), + pid: AtomicU32::new(100), + wrong_bootstrap: AtomicBool::new(false), + wrong_operation: AtomicBool::new(false), + exhausted: AtomicBool::new(false), + calls: AtomicUsize::new(0), + }); + Self { + state, + source, + observer: None, + root, + app, + } + } + + pub fn start(&mut self) { + self.observer = + Some(crowdb_web::start_service_observer_for_tests(&self.state, self.source.clone()).unwrap()); + } + + pub async fn rows(&self) -> Value { + let response = tokio::time::timeout( + Duration::from_millis(200), + self.app.clone().oneshot( + Request::builder() + .uri("/api/servers") + .body(Body::empty()) + .unwrap(), + ), + ) + .await + .expect("inventory GET waited for authority, SSH, or endpoint work") + .unwrap(); + assert!(response.status().is_success()); + serde_json::from_slice(&to_bytes(response.into_body(), 65_536).await.unwrap()).unwrap() + } + + pub async fn wait_pid(&self, node: u64, pid: Option) -> Value { + let id = self + .source + .intents + .load_full() + .iter() + .find(|intent| intent.node_id == node) + .map(|intent| intent.service_id.clone()); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let rows = self.rows().await; + let row = rows + .as_array() + .unwrap() + .iter() + .find(|row| row["node_id"] == node && id.as_ref().map_or(true, |id| row["id"] == *id)) + .unwrap(); + if row["pid"].as_u64() == pid.map(u64::from) { + return row.clone(); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("sampler did not publish the expected monitor PID") + } + + pub async fn signalled(signal: &Notify) { + tokio::time::timeout(Duration::from_secs(2), signal.notified()) + .await + .expect("observation source did not reach its deterministic barrier"); + } +} diff --git a/app/crowdb-web/tests/internal/service_start_pending_test.rs b/app/crowdb-web/tests/internal/service_start_pending_test.rs deleted file mode 100644 index 390aeedea..000000000 --- a/app/crowdb-web/tests/internal/service_start_pending_test.rs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::{NodeServiceAction, NodeServiceIntent, ProcessObservations, ServerEntry}; -use serde_json::json; - -#[test] -fn accepted_start_waits_until_owned_pid_or_current_operation_exhaustion() { - let intent = NodeServiceIntent { - node_id: 2, - service_id: "chunkdb-7".into(), - kind: "chunkdb".into(), - cluster_id: "cluster".into(), - operation_id: "current".into(), - action: NodeServiceAction::Start, - configuration: String::new(), - environment: std::collections::BTreeMap::new(), - }; - let mut entry = ServerEntry::new("chunkdb-7", "http://127.0.0.1:7300"); - entry.node_id = Some(2); - entry.auto_start = true; - for (operation, exhausted, pending) in [ - ("old", true, true), - ("current", false, true), - ("current", true, false), - ] { - let mut observation = ProcessObservations::default(); - observation.record( - 2, - &json!({"processes":{}, "service_failures": { - "chunkdb-7": {"operation_id": operation, "exhausted": exhausted} - }}), - std::slice::from_ref(&intent), - ); - assert_eq!(observation.start_pending(&entry, true, None), pending); - assert!(!observation.start_pending(&entry, true, Some(42))); - assert!(!observation.start_pending(&entry, false, None)); - } - entry.auto_start = false; - assert!(!ProcessObservations::default().start_pending(&entry, true, None)); -} diff --git a/app/crowdb-web/tests/remote_kv_projection_test.rs b/app/crowdb-web/tests/remote_kv_projection_test.rs index 2953c2a5d..864cec16e 100644 --- a/app/crowdb-web/tests/remote_kv_projection_test.rs +++ b/app/crowdb-web/tests/remote_kv_projection_test.rs @@ -99,6 +99,14 @@ async fn kv_lifecycle_intents_preserve_admitted_endpoints_while_auxiliary_servic .with_node_monitor("http://127.0.0.1:12345") .unwrap(); *state.kv_client.write().await = Some(kv); + let _observer = crowdb_web::start_service_observer(&state).unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + while state.config.read().unwrap().servers.len() != 2 { + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + }) + .await + .expect("background authority projection did not publish auxiliary registration"); let response = router(state) .oneshot( Request::builder() diff --git a/app/crowdb-web/tests/service_inventory_observation_test.rs b/app/crowdb-web/tests/service_inventory_observation_test.rs new file mode 100644 index 000000000..349397c05 --- /dev/null +++ b/app/crowdb-web/tests/service_inventory_observation_test.rs @@ -0,0 +1,237 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/service_inventory.rs"] +mod fixture; +use crowdb_protocol::mgmt::node::NodeServiceAction; +use fixture::TestServiceInventory; +use serde_json::json; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +#[tokio::test] +async fn inventory_readers_do_not_wait_for_an_unrelated_monitor_or_refresh_evidence_age() { + let mut fixture = TestServiceInventory::new(false); + fixture.source.hold_node.store(2, Ordering::Relaxed); + fixture.start(); + TestServiceInventory::signalled(&fixture.source.process_entered).await; + let row = fixture.wait_pid(1, Some(101)).await; + assert_eq!(row["pid_node_id"], 1); + assert_eq!(fixture.rows().await.as_array().unwrap().len(), 2); + assert!(crowdb_web::start_service_observer(&fixture.state).is_err()); + fixture.observer.take(); + tokio::task::yield_now().await; + let calls = fixture.source.calls.load(Ordering::Relaxed); + for rows in futures::future::join_all((0..32).map(|_| fixture.rows())).await { + assert_eq!(rows[0]["pid"], 101); + } + assert_eq!(fixture.source.calls.load(Ordering::Relaxed), calls); + tokio::time::sleep(Duration::from_millis(3050)).await; + let rows = fixture.rows().await; + assert!(rows[0]["pid"].is_null()); + assert!(rows[0]["pid_node_id"].is_null()); + assert_eq!(rows[0]["health"], "unknown"); + assert_eq!(fixture.source.calls.load(Ordering::Relaxed), calls); +} + +#[tokio::test] +async fn a_late_pre_mutation_process_sample_cannot_replace_a_confirmed_pid_or_hide_siblings() { + let mut fixture = TestServiceInventory::new(false); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + fixture.wait_pid(2, Some(102)).await; + fixture.source.hold_node.store(1, Ordering::Relaxed); + TestServiceInventory::signalled(&fixture.source.process_entered).await; + let intent = fixture.source.intents.load_full()[0].clone(); + let mutation = + crowdb_web::invalidate_service_observation_for_tests(&fixture.state, 1, &intent.service_id); + let invalidated = fixture.rows().await; + assert!(invalidated + .as_array() + .unwrap() + .iter() + .find(|row| row["node_id"] == 1) + .unwrap()["pid"] + .is_null()); + assert_eq!(fixture.wait_pid(2, Some(102)).await["pid"], 102); + // The same durable operation may resume with a replacement process. Its + // epoch still fences a delayed observation with the identical operation ID. + fixture.source.pid.store(500, Ordering::Relaxed); + mutation.complete( + intent.clone(), + &json!({"operation_id":intent.operation_id,"pid":501}), + ); + assert_eq!(fixture.wait_pid(1, Some(501)).await["pid"], 501); + fixture.source.hold_node.store(0, Ordering::Relaxed); + fixture.source.process_release.notify_one(); + TestServiceInventory::signalled(&fixture.source.process_finished).await; + for _ in 0..32 { + assert_eq!(fixture.wait_pid(1, Some(501)).await["pid"], 501); + tokio::task::yield_now().await; + } +} + +#[tokio::test] +async fn failed_or_unknown_mutation_does_not_publish_an_assumed_pid() { + let mut fixture = TestServiceInventory::new(false); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + fixture.source.hold_node.store(1, Ordering::Relaxed); + TestServiceInventory::signalled(&fixture.source.process_entered).await; + let mutation = crowdb_web::invalidate_service_observation_for_tests(&fixture.state, 1, "chunkdb-1"); + drop(mutation); + assert!(fixture + .rows() + .await + .as_array() + .unwrap() + .iter() + .find(|row| row["node_id"] == 1) + .unwrap()["pid"] + .is_null()); + fixture.source.pid.store(800, Ordering::Relaxed); + fixture.source.hold_node.store(0, Ordering::Relaxed); + fixture.source.process_release.notify_one(); + TestServiceInventory::signalled(&fixture.source.process_finished).await; + fixture.wait_pid(1, Some(801)).await; +} + +#[tokio::test] +async fn an_unknown_outcome_also_fences_samples_started_during_the_mutation() { + let mut fixture = TestServiceInventory::new(false); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + let mutation = crowdb_web::invalidate_service_observation_for_tests(&fixture.state, 1, "chunkdb-1"); + fixture.source.hold_node.store(1, Ordering::Relaxed); + TestServiceInventory::signalled(&fixture.source.process_entered).await; + fixture.source.pid.store(800, Ordering::Relaxed); + drop(mutation); + fixture.source.hold_node.store(0, Ordering::Relaxed); + fixture.source.process_release.notify_one(); + TestServiceInventory::signalled(&fixture.source.process_finished).await; + let rows = fixture.rows().await; + assert!(rows + .as_array() + .unwrap() + .iter() + .find(|row| row["node_id"] == 1) + .unwrap()["pid"] + .is_null()); + fixture.wait_pid(1, Some(801)).await; +} + +#[tokio::test] +async fn a_slow_sibling_health_probe_does_not_delay_target_process_or_health_resampling() { + let mut fixture = TestServiceInventory::new(false); + let mut sibling = fixture.state.config.read().unwrap().servers[0].clone(); + sibling.id = "chunkdb-sibling".into(); + fixture.state.config.write().unwrap().servers.push(sibling); + fixture.source.hold_health.store(true, Ordering::Relaxed); + fixture.start(); + TestServiceInventory::signalled(&fixture.source.health_entered).await; + fixture.wait_pid(1, Some(101)).await; + let intent = fixture.source.intents.load_full()[0].clone(); + let mutation = + crowdb_web::invalidate_service_observation_for_tests(&fixture.state, 1, &intent.service_id); + fixture.source.pid.store(500, Ordering::Relaxed); + mutation.complete( + intent.clone(), + &json!({"operation_id":intent.operation_id,"pid":501}), + ); + drop(mutation); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let row = fixture.wait_pid(1, Some(501)).await; + if row["health"] == "up" { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("target readiness waited behind a sibling health probe"); +} + +#[tokio::test] +async fn fresh_bound_processes_remain_observable_when_authority_expires_but_access_health_is_unknown() { + let mut fixture = TestServiceInventory::new(true); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + fixture.source.fail_authority.store(true, Ordering::Relaxed); + tokio::time::sleep(Duration::from_millis(3150)).await; + let row = fixture.wait_pid(1, Some(101)).await; + assert_eq!(row["pid_node_id"], 1); + assert_eq!(row["health"], "unknown"); + assert_eq!(row["start_pending"], false); +} + +#[tokio::test] +async fn changed_endpoint_wrong_bootstrap_and_wrong_executed_operation_never_reuse_a_pid() { + let mut fixture = TestServiceInventory::new(false); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + fixture.source.wrong_bootstrap.store(true, Ordering::Relaxed); + fixture.wait_pid(1, None).await; + fixture.source.wrong_bootstrap.store(false, Ordering::Relaxed); + fixture.source.wrong_operation.store(true, Ordering::Relaxed); + tokio::time::sleep(Duration::from_millis(1100)).await; + assert!(fixture.wait_pid(1, None).await["pid_node_id"].is_null()); + fixture.source.wrong_operation.store(false, Ordering::Relaxed); + fixture.wait_pid(1, Some(101)).await; + fixture.state.config.write().unwrap().nodes[0].host = "127.0.0.99".into(); + let rows = fixture.rows().await; + let row = rows + .as_array() + .unwrap() + .iter() + .find(|row| row["node_id"] == 1) + .unwrap(); + assert!(row["pid"].is_null()); + assert_eq!(row["health"], "unknown"); + std::fs::remove_file(fixture.root.path().join("node-binding.json")).unwrap(); + for row in fixture.rows().await.as_array().unwrap() { + assert!(row["pid"].is_null()); + } +} + +#[tokio::test] +async fn pending_start_uses_only_the_current_operation_exhaustion() { + let mut fixture = TestServiceInventory::new(false); + fixture.source.pid.store(0, Ordering::Relaxed); + fixture.source.exhausted.store(true, Ordering::Relaxed); + fixture.source.wrong_operation.store(true, Ordering::Relaxed); + fixture.start(); + tokio::time::sleep(Duration::from_millis(150)).await; + assert_eq!(fixture.wait_pid(1, None).await["start_pending"], true); + fixture.source.wrong_operation.store(false, Ordering::Relaxed); + tokio::time::sleep(Duration::from_millis(1100)).await; + assert_eq!(fixture.wait_pid(1, None).await["start_pending"], false); +} + +#[tokio::test] +async fn a_late_authority_sample_cannot_resurrect_a_deleted_registration() { + let mut fixture = TestServiceInventory::new(false); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + fixture.source.hold_authority.store(true, Ordering::Relaxed); + TestServiceInventory::signalled(&fixture.source.authority_entered).await; + let mutation = crowdb_web::invalidate_service_observation_for_tests(&fixture.state, 1, "chunkdb-1"); + let mut intents = fixture.source.intents.load_full().as_ref().clone(); + intents[0].action = NodeServiceAction::Delete; + intents[0].operation_id = "delete-operation".into(); + fixture.source.intents.store(Arc::new(intents.clone())); + mutation.complete( + intents[0].clone(), + &json!({"operation_id":"delete-operation","pid":null}), + ); + assert_eq!(fixture.rows().await.as_array().unwrap().len(), 1); + fixture.source.hold_authority.store(false, Ordering::Relaxed); + fixture.source.authority_release.notify_one(); + TestServiceInventory::signalled(&fixture.source.authority_finished).await; + for _ in 0..32 { + assert_eq!(fixture.rows().await.as_array().unwrap().len(), 1); + tokio::task::yield_now().await; + } +} diff --git a/container/crowdb-monitor/src/node/runtime/observation.rs b/container/crowdb-monitor/src/node/runtime/observation.rs index 04ad92d0b..d9d1e8488 100644 --- a/container/crowdb-monitor/src/node/runtime/observation.rs +++ b/container/crowdb-monitor/src/node/runtime/observation.rs @@ -35,6 +35,10 @@ pub(super) fn processes( // This authenticated read observes only local children. It needs no live // Group-0 grant and remains available while that authority is unavailable. let live = manager.live_pids()?; - let failures = super::services::failures(root, &live)?; - Ok(json!({"cluster_id":cluster,"node_id":node,"processes":live,"service_failures":failures})) + let failures = super::services::failures(root, &live, &binding)?; + let operations = super::services::process_operations(root, &live, &binding)?; + Ok( + json!({"cluster_id":cluster,"node_id":node,"bootstrap":binding.bootstrap, + "processes":live,"process_operations":operations,"service_failures":failures}), + ) } diff --git a/container/crowdb-monitor/src/node/runtime/services.rs b/container/crowdb-monitor/src/node/runtime/services.rs index 579a03b40..c34ec18e7 100644 --- a/container/crowdb-monitor/src/node/runtime/services.rs +++ b/container/crowdb-monitor/src/node/runtime/services.rs @@ -132,6 +132,11 @@ pub(super) fn pending( &intent.service_id }; let alive = processes.owns(id) && processes.alive(id)?; + let binding: NodeBinding = + durable::read(&profile.paths.data_root.join("node-binding.json"))?.ok_or("node is unbound")?; + if previous.bootstrap.as_ref() != Some(&binding.bootstrap) { + return Ok(true); + } if previous.attempts >= 5 && !alive && matches!( @@ -147,7 +152,7 @@ pub(super) fn pending( Ok(matches!( intent.action, NodeServiceAction::Start | NodeServiceAction::Restart - ) && !alive) + ) && (!alive || previous.pid != processes.pid(id))) } fn bounded_error(error: &str) -> String { @@ -157,6 +162,7 @@ fn bounded_error(error: &str) -> String { pub(super) fn failures( root: &std::path::Path, live: &std::collections::BTreeMap, + binding: &NodeBinding, ) -> Result { let directory = root.join("services"); let mut failures = serde_json::Map::new(); @@ -169,6 +175,12 @@ pub(super) fn failures( continue; } if let Some(execution) = durable::read::(&entry.path().join("execution.json"))? { + if execution.bootstrap.as_ref() != Some(&binding.bootstrap) + || execution.intent.cluster_id != binding.bootstrap.cluster_id + || execution.intent.node_id != binding.node_id + { + continue; + } if let Some(error) = execution.error { let process = if execution.intent.kind == "kv" { "kv" @@ -191,3 +203,42 @@ pub(super) fn failures( } Ok(Value::Object(failures)) } + +/// Couple each live managed PID to the durable execution that actually owns it. +pub(super) fn process_operations( + root: &std::path::Path, + live: &std::collections::BTreeMap, + binding: &NodeBinding, +) -> Result { + let directory = root.join("services"); + let mut operations = serde_json::Map::new(); + if !directory.exists() { + return Ok(Value::Object(operations)); + } + for entry in std::fs::read_dir(directory)? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + if let Some(execution) = durable::read::(&entry.path().join("execution.json"))? { + if execution.bootstrap.as_ref() != Some(&binding.bootstrap) + || execution.intent.cluster_id != binding.bootstrap.cluster_id + || execution.intent.node_id != binding.node_id + { + continue; + } + let process = if execution.intent.kind == "kv" { + "kv" + } else { + &execution.intent.service_id + }; + if execution.pid.is_some() && execution.pid == live.get(process).copied() { + if operations.len() >= 4096 { + return Err("service process observation exceeds bound".into()); + } + operations.insert(process.to_owned(), execution.intent.operation_id.into()); + } + } + } + Ok(Value::Object(operations)) +} diff --git a/container/crowdb-monitor/src/node/runtime/services/execution.rs b/container/crowdb-monitor/src/node/runtime/services/execution.rs index a2db1442e..a489c62ac 100644 --- a/container/crowdb-monitor/src/node/runtime/services/execution.rs +++ b/container/crowdb-monitor/src/node/runtime/services/execution.rs @@ -33,25 +33,13 @@ pub(in crate::node::runtime) async fn execute( &intent.service_id }; std::fs::create_dir_all(&directory)?; - let mut attempts = 0; - let mut continuing = false; - if let Some(previous) = previous { - if previous.intent.operation_id == intent.operation_id { - if previous.intent != intent { - return Err("operation identity reused with other inputs".into()); - } - attempts = previous.attempts; - continuing = matches!( - intent.action, - NodeServiceAction::Start | NodeServiceAction::Restart - ) && processes.owns(process_id) - && processes.alive(process_id)?; - if previous.complete - && (previous.pid.is_none() || (processes.owns(process_id) && processes.alive(process_id)?)) - { - return Ok(json!({"pid": processes.pid(process_id), "operation_id": intent.operation_id})); - } - } + let Replay { + mut attempts, + continuing, + completed, + } = replay(previous.as_ref(), &binding, &intent, processes, process_id)?; + if completed { + return Ok(json!({"pid": processes.pid(process_id), "operation_id": intent.operation_id})); } if matches!( intent.action, @@ -109,6 +97,50 @@ pub(in crate::node::runtime) async fn execute( Ok(json!({"pid": pid, "operation_id": intent.operation_id})) } +#[derive(Default)] +struct Replay { + attempts: u32, + continuing: bool, + completed: bool, +} + +fn replay( + previous: Option<&Execution>, + binding: &NodeBinding, + intent: &NodeServiceIntent, + processes: &mut ProcessManager, + process_id: &str, +) -> Result { + let Some(previous) = previous.filter(|previous| previous.intent.operation_id == intent.operation_id) + else { + return Ok(Replay::default()); + }; + if previous.intent != *intent { + return Err("operation identity reused with other inputs".into()); + } + if previous + .bootstrap + .as_ref() + .is_some_and(|bootstrap| bootstrap != &binding.bootstrap) + { + return Err("service execution belongs to another bootstrap".into()); + } + let continuing = matches!( + intent.action, + NodeServiceAction::Start | NodeServiceAction::Restart + ) && processes.owns(process_id) + && processes.alive(process_id)?; + let completed = previous.complete + && previous.bootstrap.as_ref() == Some(&binding.bootstrap) + && previous.pid == processes.pid(process_id) + && (previous.pid.is_none() || (processes.owns(process_id) && processes.alive(process_id)?)); + Ok(Replay { + attempts: previous.attempts, + continuing, + completed, + }) +} + async fn apply( profile: &DeploymentProfile, processes: &mut ProcessManager, @@ -178,7 +210,11 @@ async fn apply( // A live process has already had its initial startup window. Later health // observations must yield after one probe so control requests can make progress. async fn observe_ready(profile: &DeploymentProfile, intent: &NodeServiceIntent) -> Result<()> { - let service = super::super::probes::for_intent(profile, intent)?; + let service = if intent.kind == "kv" { + super::super::service(profile, "kv")? + } else { + super::super::probes::for_intent(profile, intent)? + }; crate::ProbeExecutor::new(intent.kind == "diskio")? .probe_service(&service, &std::collections::BTreeMap::new()) .await diff --git a/container/crowdb-monitor/src/node/runtime/services/test_support.rs b/container/crowdb-monitor/src/node/runtime/services/test_support.rs index fec2ae88b..8eb61c74f 100644 --- a/container/crowdb-monitor/src/node/runtime/services/test_support.rs +++ b/container/crowdb-monitor/src/node/runtime/services/test_support.rs @@ -121,6 +121,27 @@ impl NodeServiceForTests { /// # Errors /// Propagates bounded failure observation errors. pub fn failures_for_tests(&mut self) -> Result { - failures(&self.profile.paths.data_root, &self.processes.live_pids()?) + let binding = super::durable::read::( + &self.profile.paths.data_root.join("node-binding.json"), + )? + .ok_or("node is unbound")?; + failures( + &self.profile.paths.data_root, + &self.processes.live_pids()?, + &binding, + ) + } + + /// Read the production bound namespace of this fixture's actual children. + /// + /// # Errors + /// Propagates binding, process and durable observation failures. + pub fn owned_processes_for_tests(&mut self, cluster: &str, node: u64) -> Result { + super::super::observation::processes( + &self.profile.paths.data_root, + &mut self.processes, + cluster, + node, + ) } } diff --git a/container/crowdb-monitor/tests/common/service_replacement.rs b/container/crowdb-monitor/tests/common/service_replacement.rs new file mode 100644 index 000000000..becccf06c --- /dev/null +++ b/container/crowdb-monitor/tests/common/service_replacement.rs @@ -0,0 +1,100 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::node_setup; +use crowdb_monitor::{DeploymentProfile, NodeIdentity, NodeServiceForTests}; +use crowdb_protocol::mgmt::node::{ + setup::{NodeSetupChoices, NodeSetupRequest}, + NodeServiceAction, NodeServiceIntent, +}; +use serde_json::json; +use std::{collections::BTreeMap, os::unix::fs::PermissionsExt}; + +pub struct TestReplacement { + pub directory: crowdb_test_harness::test_dirs::TestDir, + pub authority: node_setup::TestAuthority, + pub worker: NodeServiceForTests, + pub intent: NodeServiceIntent, + pub original: u32, + pub health: tokio::task::JoinHandle<()>, + pub state: std::path::PathBuf, +} + +impl TestReplacement { + pub async fn new() -> Self { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("service-replacement"); + let root = directory.path(); + let request = NodeSetupRequest { + version: 1, + discovery_id: NodeIdentity::load_or_create(root).unwrap().uuid().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + choices: NodeSetupChoices::default(), + }; + let authority = node_setup::TestAuthority::start(root, &request).await; + assert_eq!(authority.registry.nodes[0].node_id, authority.binding.node_id); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let health = tokio::spawn(async move { + axum::serve( + listener, + axum::Router::new().route("/health", axum::routing::get(|| async { "healthy" })), + ) + .await + .unwrap(); + }); + let mut profile = DeploymentProfile::load( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml"), + ) + .unwrap(); + profile.paths.data_root = root.to_path_buf(); + profile.paths.log_root = root.join("log"); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + let program = root.join("owned-kv"); + std::fs::write(&program, "#!/bin/sh\nexec sleep 60\n").unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap(); + let service = profile + .services + .iter_mut() + .find(|service| service.id == "kv") + .unwrap(); + service.program = program; + service.probe.target = format!("http://{address}/health"); + let intent = NodeServiceIntent { + cluster_id: authority.binding.bootstrap.cluster_id.clone(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: authority.binding.node_id, + service_id: "kv-2".into(), + kind: "kv".into(), + action: NodeServiceAction::Start, + configuration: "node_id = 2\n".into(), + environment: BTreeMap::new(), + }; + authority + .kv + .put( + 0, + 0, + b"/deployment/services/2/kv-2", + &serde_json::to_vec(&intent).unwrap(), + None, + ) + .await + .unwrap(); + let mut worker = NodeServiceForTests::new_for_tests(profile).await.unwrap(); + let original = worker.start_kv_child_for_tests().await.unwrap(); + let state = root.join("services/kv-2/execution.json"); + std::fs::create_dir_all(state.parent().unwrap()).unwrap(); + let record = json!({"intent":intent,"bootstrap":authority.binding.bootstrap,"complete":true,"pid":original,"attempts":0,"error":null}); + std::fs::write(&state, serde_json::to_vec(&record).unwrap()).unwrap(); + std::fs::set_permissions(&state, std::fs::Permissions::from_mode(0o600)).unwrap(); + Self { + directory, + authority, + worker, + intent, + original, + health, + state, + } + } +} diff --git a/container/crowdb-monitor/tests/node_process_operation_test.rs b/container/crowdb-monitor/tests/node_process_operation_test.rs new file mode 100644 index 000000000..9e8b8a6e9 --- /dev/null +++ b/container/crowdb-monitor/tests/node_process_operation_test.rs @@ -0,0 +1,95 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/runtime_observation.rs"] +mod fixture; +use crowdb_monitor::NodeServiceForTests; +use crowdb_protocol::mgmt::{node::NodeBinding, SystemBootstrapIdentity}; +use serde_json::{json, Value}; +use std::{os::unix::fs::PermissionsExt, time::Duration}; + +#[tokio::test] +async fn process_operations_and_failures_require_the_actual_pid_and_current_bound_execution() { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("process-operation"); + let mut test = fixture::TestObservation::start(directory.path()).await; + let binding = NodeBinding { + node_id: test.intent.node_id, + bootstrap: SystemBootstrapIdentity { + cluster_id: test.intent.cluster_id.clone(), + operation_id: "bootstrap".into(), + configuration_digest: "a".repeat(64), + }, + management_seeds: vec!["http://127.0.0.1:1".into()], + }; + std::fs::write( + directory.path().join("node-binding.json"), + serde_json::to_vec(&binding).unwrap(), + ) + .unwrap(); + std::fs::set_permissions( + directory.path().join("node-binding.json"), + std::fs::Permissions::from_mode(0o600), + ) + .unwrap(); + let state = directory.path().join("services/chunkdb-1/execution.json"); + let mut record: Value = serde_json::from_slice(&std::fs::read(&state).unwrap()).unwrap(); + record["bootstrap"] = json!(binding.bootstrap); + record["attempts"] = json!(5); + record["error"] = json!("retained diagnostic"); + std::fs::write(&state, serde_json::to_vec(&record).unwrap()).unwrap(); + let pid = test.manager.pid("chunkdb-1").unwrap(); + assert_eq!( + test.store.read(Duration::from_secs(3)).unwrap().services["chunkdb-1"].pid, + Some(pid) + ); + let read = |manager: &mut crowdb_monitor::ProcessManager| { + NodeServiceForTests::processes_for_tests( + directory.path(), + manager, + &binding.bootstrap.cluster_id, + binding.node_id, + ) + .unwrap() + }; + let observed = read(&mut test.manager); + assert_eq!(observed["bootstrap"], json!(binding.bootstrap)); + assert_eq!(observed["processes"]["chunkdb-1"], pid); + assert_eq!( + observed["process_operations"]["chunkdb-1"], + test.intent.operation_id + ); + assert_eq!(observed["service_failures"]["chunkdb-1"]["exhausted"], false); + for fault in ["legacy", "bootstrap", "cluster", "node", "pid"] { + let mut invalid = record.clone(); + match fault { + "legacy" => { + invalid.as_object_mut().unwrap().remove("bootstrap"); + } + "bootstrap" => invalid["bootstrap"]["operation_id"] = json!("old-bootstrap"), + "cluster" => invalid["intent"]["cluster_id"] = json!("other-cluster"), + "node" => invalid["intent"]["node_id"] = json!(99), + "pid" => invalid["pid"] = json!(pid + 100_000), + _ => unreachable!(), + } + std::fs::write(&state, serde_json::to_vec(&invalid).unwrap()).unwrap(); + let observed = read(&mut test.manager); + assert_eq!( + observed["processes"]["chunkdb-1"], pid, + "actual children remain observable: {fault}" + ); + assert!( + observed["process_operations"]["chunkdb-1"].is_null(), + "foreign execution labeled the live PID: {fault}" + ); + if fault != "pid" { + assert!( + observed["service_failures"]["chunkdb-1"].is_null(), + "foreign execution exhausted the current start: {fault}" + ); + } + } + test.manager + .stop("chunkdb-1", Duration::from_secs(1)) + .await + .unwrap(); +} diff --git a/container/crowdb-monitor/tests/node_service_replacement_test.rs b/container/crowdb-monitor/tests/node_service_replacement_test.rs new file mode 100644 index 000000000..3a908fa0f --- /dev/null +++ b/container/crowdb-monitor/tests/node_service_replacement_test.rs @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/service_replacement.rs"] +mod fixture; +#[path = "common/node_setup.rs"] +mod node_setup; +use fixture::TestReplacement; +use serde_json::{json, Value}; + +#[tokio::test] +async fn supervised_kv_replacements_and_legacy_receipts_refresh_the_pid_without_another_restart() { + let TestReplacement { + directory: _directory, + authority, + mut worker, + intent, + original, + health, + state, + } = TestReplacement::new().await; + worker.stop_kv_for_tests().await.unwrap(); + let replacement = worker.start_kv_child_for_tests().await.unwrap(); + assert_ne!(replacement, original); + assert!(worker.pending_for_tests(&intent).unwrap()); + assert!(worker + .owned_processes_for_tests(&intent.cluster_id, intent.node_id) + .unwrap()["process_operations"]["kv"] + .is_null()); + let result = worker.execute_for_tests(intent.clone()).await.unwrap(); + assert_eq!(result["pid"], replacement); + assert!(!worker.pending_for_tests(&intent).unwrap()); + let observed = worker + .owned_processes_for_tests(&intent.cluster_id, intent.node_id) + .unwrap(); + assert_eq!(observed["processes"]["kv"], replacement); + assert_eq!(observed["process_operations"]["kv"], intent.operation_id); + let mut record: Value = serde_json::from_slice(&std::fs::read(&state).unwrap()).unwrap(); + record.as_object_mut().unwrap().remove("bootstrap"); + std::fs::write(&state, serde_json::to_vec(&record).unwrap()).unwrap(); + assert!(worker.pending_for_tests(&intent).unwrap()); + assert_eq!( + worker.execute_for_tests(intent.clone()).await.unwrap()["pid"], + replacement + ); + let migrated: Value = serde_json::from_slice(&std::fs::read(&state).unwrap()).unwrap(); + assert_eq!(migrated["bootstrap"], json!(authority.binding.bootstrap)); + let mut foreign = migrated.clone(); + foreign["bootstrap"]["operation_id"] = json!("foreign-bootstrap"); + std::fs::write(&state, serde_json::to_vec(&foreign).unwrap()).unwrap(); + assert!(worker.pending_for_tests(&intent).unwrap()); + assert!(worker + .execute_for_tests(intent.clone()) + .await + .unwrap_err() + .to_string() + .contains("another bootstrap")); + assert_eq!( + worker + .owned_processes_for_tests(&intent.cluster_id, intent.node_id) + .unwrap()["processes"]["kv"], + replacement + ); + assert_eq!( + serde_json::from_slice::(&std::fs::read(&state).unwrap()).unwrap(), + foreign + ); + worker.stop_kv_for_tests().await.unwrap(); + health.abort(); +} diff --git a/doc/dev/physical-node-test-plan.md b/doc/dev/physical-node-test-plan.md index 7eb0bb78a..33969ca4b 100644 --- a/doc/dev/physical-node-test-plan.md +++ b/doc/dev/physical-node-test-plan.md @@ -32,6 +32,10 @@ Upstream: [deployment and recovery](../design/deploy/design-crowdb-deploy.md), or the Docker daemon. - A reduced three-node run must explicitly declare A1/A2/B1 and three voters. It cannot substitute for five-node coverage. +- Record each retained EC layout's fragment distribution and actual loss budget. + Three live KV voters do not establish payload availability. For example, + EC 8+4 with six fragments on B cannot reconstruct that data while B is offline. + Keep this limit separate from the acknowledged-data audit after B returns. ## 2. Execution contract @@ -156,7 +160,10 @@ acknowledged writes from the current case. Verify single-attempt KV writes/reads and catalog takeover onto surviving Serving owners within 180 seconds. While B stays offline, submit fresh S3 writes and Iceberg property commits through A, then cross-read. Use new - witnesses per fault to prove new writes occurred. Restore B, then recreate + witnesses per fault to prove new writes occurred, and verify those witnesses + through all three A endpoints while B remains offline. Record temporary + unavailability of older EC data when the stopped fragments exceed its actual + loss budget; do not claim rack protection for such a layout. Restore B, then recreate B1/B2 individually with retained volumes. Check original identities, all data through five endpoints, and all five replicas caught up. - [ ] **P13 All three A nodes offline**: stop A1/A2/A3; B loses quorum. @@ -209,7 +216,10 @@ prove individual deadlines. Preserve shorter production-client limits. with verified ownership. Check owned ports are closed on both hosts, no owned resources remain, and SSH key access still works. Retain run artifacts. Do not prune the daemon or clean other tasks' resources. Capture failures - before cleanup. + before cleanup. Before retiring failed retained data, privately archive the + stopped owned volumes and verify archive integrity and checksums. A Docker + archive helper streaming binary tar output must use `--log-driver none` so + the daemon does not duplicate the archive into container JSON logs. ## 7. Deliverables and the next run @@ -231,13 +241,6 @@ Save these artifacts and link them from the final summary: and sample verification summaries. Confirm implementation and acceptance of all three targets before use; mark unimplemented paths `NOT RUN`. -Open issue requiring retest (2026-10-31): five containers writing -512 records × 64 KiB reached the default five-second deadline during -split/cutover. The previous run acknowledged and recovered only 294 records, -not a full load. Reproduce, repair, and fully pass P04; then seed at least -4096 records and run 30 continuous minutes. Remove this open item only after -successful repair and retest, preserving results in its run directory. - Suggested request for a future run: > Execute the full five-node test in `doc/dev/physical-node-test-plan.md` with diff --git a/doc/working/plan-ci-stability-review.md b/doc/working/plan-ci-stability-review.md new file mode 100644 index 000000000..950990d78 --- /dev/null +++ b/doc/working/plan-ci-stability-review.md @@ -0,0 +1,504 @@ + + + +# CI Stability Review Plan + +Upstream: [first failed run](https://github.com/buzzcrow/crowdb/actions/runs/41003611925), +[second failed run](https://github.com/buzzcrow/crowdb/actions/runs/41128190276), +[newer failed run](https://github.com/buzzcrow/crowdb/actions/runs/41159636766), +[container acceptance](../../e2e-test/README.md), and +[CI workflow](../../.github/workflows/ci.yml). + +Goal: review all 16 failed jobs individually, verify their fixes and regression +coverage, and identify the remaining work needed to improve CI stability. + +用户要求保留这份 TODO，待本轮测试、提交、推送和新 CI 完成后逐项共同 review。 +测试通过只更新验证证据，不自动勾选 review。全部 review 完成后，归档结论， +将仍需实现的工作保留到对应任务，再删除这份临时清单。 + +2026-11-01 用户调整顺序：收尾当前 RPC 生命周期修复后，优先完成 CI 稳定性 +改动与验证，然后 commit/push 并检查新 CI；之后恢复 P13 和其余物理测试。 +提交时如实保留未完成的物理验证，不将其提前勾选。 + +2026-11-01 用户恢复执行。此前暂停于 GC 派发前；恢复后加强版 GC 三例、 +辅助服务/Journal 三轮组合、Go SDK 三轮和 Dataset 分阶段验证均已通过。 +本地补充验证控制器正常退出，下一步是提交、推送和核对新 SHA 的完整 CI。 +物理容器在恢复前已停止，原因未知；七个容器、镜像和卷身份均保持不变。 + +## 当前执行队列 + +- [x] **完成 CI 稳定性修复**：保留 auxiliary lifecycle 原有 combined 三秒 + predicate，将 `/api/servers` 的采集移出 HTTP 读路径；authority 与各节点 + 独立采样，新鲜度从采样开始计，绑定/端点/operation 与 mutation fence + 阻止旧 PID 复活。保留 Group 0 不可用时 authenticated monitor 的实际 + PID 观察，以及真实 Access readiness。补慢 peer、过期/错身份、乱序、 + 失败恢复及 sampler 退出的确定性回归。同时修复 browser batch 未派发 + peer 的错误归因，明确未执行，不能计为 PASS。 + 文件：`app/crowdb-web/src/services/observation/`、`services/remote.rs`、 + monitor 私有 process reply、`e2e-test/browser_batch.py`、`fixture.py` + 及对应测试。 +- [x] **恢复后完成最终本地验证**：受影响组件、fmt/clippy/C++ gate、共享 clients + 与新 OCI 镜像、原失败精确场景、396 个 canonical case 及并发隔离； + 每次首错停止后续派发，保留已通过项与完整错误。 + 当前 core 1165、access/monitor 2039、console 426、完整 fixture 301 项通过； + UI、C++/FFI、storage、fmt、full clippy 与 CI task mapping 通过。 + 新代码的 helper 拆分及 Rust 1.75 兼容修正后，受影响 focused 组再次通过； + 没有增加 lint allow、延长 timeout 或改变 UI/schema。新镜像已构建并核对 + production source；原 auxiliary 三秒断言通过，加强后的真实 PID/namespace、 + 周期前后 operation、注册唯一性和三节点 sibling 身份断言也通过， + Playwright 从原断言基线 9.7 秒到加强版 12.4 秒。native-operations 全组通过。 + 新镜像全量已收尾：396 个 canonical case 的最新最终结果全部 PASS， + 必需的四例并发隔离通过，七个保留物理容器及卷身份不变。加强版 CLI + 五例随后全部通过，用时 102.757 秒。恢复后正常刷新 clients：GC 三例 + 通过；浏览器组合曾在第二轮发现 Access 依赖恢复边界缺失，修正后三个 + 全新组合轮次全部通过，约 45–46 秒。Go 三轮普通和注入后 cleanup 均通过。 + Dataset browse client 278.32 秒，其中 seed 2.276 秒、默认 balance + 260.002 秒、512 条完整回读 0.273 秒；12 partitions、三个 owners。 + 最终 fmt/full clippy、受影响 C++ format/tree-lint、TypeScript 和 scope + guard 通过。新增断言均有独立实际运行证据，不能与旧 bundle 的全量混算。 +- [~] **提交、推送并核对远端 CI**：CROWDB `task-fix-build` 一个一致提交； + 记录精确 SHA、run/attempt 和全部展开 job，修复真实失败后推进。 +- [ ] **恢复物理验证**：远端 CI 核对后继续 P13 和 P14–P19，保留 native + 崩溃的新镜像验证及历史数据审计；当前物理资源和失败证据继续保留。 + +## 通过场景的稳定性改善 + +用户要求同时处理 PASS 但不稳定的场景。本节记录额外改善与待验证边界， +不增加历史失败 job 数，也不代替后面的 16 项人工 review。 + +- **GC 与真实 SDK 的重叠窗口**：前一镜像 matrix 虽通过，实际删除增长与 + 前台工作同时活跃的七个窗口合计仅 1.763 秒；冷启动 SDK 发生在已删除 + 一个文件后。现先在可执行 purge marker 发布前启动四个真实 SDK worker， + 等 namespace 请求成功并 parked，再执行原三次重启，直到实际删除边界 + 才放行请求与小写入。warmup、seed、run 共用原 720/3300 秒绝对期限。 + 保留九文件、三次重启、64 次小写入、六类各至少 48 次成功 SDK 操作和 + 成功请求完成时刻落在真实删除增长区间的断言。独占进程组正常清理失败 + 必须使 case 失败；取消与 Drop 保留首错及完整 stdout/stderr。 + 六项纯协议回归、full clippy 和独立只读复核通过；三个受影响真实 GC + case 在新 bundle 均已通过，队列用时 480.220 秒。convergence/matrix + 分别有 240/81 个 SDK 完成时刻落在 5/7 个真实删除区间，均保留 64 次 + 写入；matrix 三次重启先于放行。证据：`resume-gc-proof.json`。 +- **Journal 刷新响应归属**：原 pathname-only waiter 可被选择 partition + 自动产生的 GET，或同路径的旧 cursor/树页请求满足。现先完成选择时的 + 自动观察，再关联本次 Refresh 的具体 Request 和完整资源/cursor， + 等响应 body 完成并核对 Journal record。全部新增等待共用原三秒预算， + owner-loss、restart、stale-cursor 断言保留。四项事件回归和完整 E2E + TypeScript 检查通过；最终与 auxiliary 配对三个全新轮次均通过。 +- **辅助服务故障的依赖恢复边界**：恢复后的浏览器组合第二轮在 ChunkKV + 已返回新 PID、Access 尚未恢复时进入下一个周期，触发原即时 `up` 断言。 + GET inventory 为 0.3–12 毫秒，Access PID 未变化。精确单例和原组合串行 + 基线通过；原并发组合的只读采样确认依赖故障期间真实 readiness 503， + 后续恢复 200。现把同节点 Access 恢复加入已有三秒 replacement predicate， + 保留即时检查、真实 PID/namespace/operation、兄弟进程和注册唯一性。 + 没有额外等待窗口、sleep、超时放宽或 mutation 重试。随后三个全新配对 + 轮次均通过；首错和诊断保留在 `ACCESS-DEPENDENCY-CLOSURE.md` 及 + `first-resumed-browser-failure/`。这不是生产 false-down 根因修复的声明。 +- **CLI 生命周期快照**：本轮全量首错为 + `plain_docker_target_attachment` 的 `KeyError: 'pid'`；cluster Ready + 不承诺独立 `/api/servers` cache 瞬时填齐可选 PID。现核对 owned + 容器/镜像/binding，读取 21 个服务的 monitor PID/generation/operation， + 并在对应容器 `/proc` 拒绝已经退出的旧正 PID；显式要求 Web 加六类服务。 + 同时保留 18 个唯一注册、所有端点字段与已有 cached PID/namespace + 一致性检查。四项回归通过；最终加强版五例已在新 bundle 全部实际执行 + 并通过，用时 102.757 秒。没有对可选 PID 加等待或调用方重试。 +- **Dataset 准备阶段余量**：前一镜像五个相关 client 为 280.68–288.44 秒， + 其中 browser 仅 4.72–11.20 秒，主要耗时为正常 production balance + 收敛，外层 client 为 360 秒。不能声称并行 seed 能解决约 270 秒的主耗时。 + 保留 512 条记录、至少 12 partitions、三个 owners、默认 policy 和原期限， + 新增 seed/balance/逐条验证的独立阶段时间；本轮及新 SHA CI 继续核对余量。 + 当前新镜像的全量验证中，browse 为 281.284 秒，import 为 321.901 秒， + recovery 为 331.468 秒；后两者离 360 秒外层期限仅余 38.099/28.532 秒。 + 两者到 272 秒仍只有九个 partitions，302 秒才观察到十二个；尚无证据 + 表明条件已满足后存在可去掉的空等。30 秒为测试日志节流，生产 planner + 实际为两秒 heartbeat，不能把打印间隔当成决策周期。只读审计及下次复现 + 所需的有界 planner/transition/freshness 证据见 + `ci/stability-final/DATASET-HEADROOM-AUDIT.md`。 + 本轮新增计时的 browse 已通过，client 278.32 秒，默认 balance 占 + 260.002 秒；该项尚未证明有行为级性能改善，不能仅因一次通过关闭风险。 +- **有针对性的重复验证**：新的辅助服务/Journal 组合与 Go SDK cleanup + 各三个全新 fixture 轮次均通过；此前首错停止记录完整保留。 + Go 原始 cleanup 原因仍未知；重复通过只提供当前覆盖，不能声称已找到 + 历史根因或证明长期失败率。执行记录保留在 `ci/stability-final/`。 + +## Review 规则与证据 + +- 两次原始 run 分别有 27 个 job，失败数为 8 和 5；没有 cancelled job。 +- 用户追加的最新 run `41159636766` 同样有 27 个展开后的 job，24 成功、 + 3 失败、无 cancelled；attempt 为 1，head 为 + `8885d2adc351d9c99f0219e18ecf0c4782bc3079`，分支为 `task-fix-build`。 + 原始 13 项保留，新一轮逐项追加为 14–16；相同 job 名不代表同一个首错。 +- 原始测试版本分别为 `39440d023d7fb162dd61c377e8eb17eefa8c4c9a` 和 + `25059cf7d9a5f08d61479f4749659aa6f3fe119f`。 +- 已获取前两轮全部 13 个失败 job 的完整日志、7 份原始诊断 artifact，并逐项映射 + 32 条容器失败记录及 2 个非容器失败 case。 +- 第三轮另有 3 份完整失败 job 日志和 3 份诊断 artifact；下载档案的 SHA256 + 均与 GitHub metadata digest 相符。累计 16 个失败 job、35 条容器失败记录， + case 名称有跨 run 重复，不能说成 35 个不同 case。 +- 区分首个真实错误、同一故障的后续症状、同批启动失败导致的未执行项、 + 故意注入的失败和包装层 panic。不能把它们都算作独立根因。 +- 保持原有超时、断言和调用方重试策略；修复真实前置条件、竞态和资源隔离问题。 +- 单次 PASS 是覆盖证据，不是长期失败率结论。发布后的精确 SHA、所有子 job、 + attempt 和结果也必须记录。 +- 已有组件与前一镜像的完整 396-case 账本均通过。上一版物理 P12 修复镜像 + 全量 396 个 canonical case 及并发隔离均通过；共 400 个完整 fixture，0 失败。 +- 本地证据保存在 + `.crowdb-runtime/artifacts/physical-nodes/physical-20261031T194149Z/ci/prior-ci-audit/`： + `REPORT.md`、`job-matrix.json`、`case-coverage.json`、完整 job 日志及原始 artifact。 + 第三轮另见 `run-41159636766/REPORT.md`、`job-matrix.json`、 + `artifact-sha256.json` 及 `current-exact-case-validation.json`。 + 该版记录位于同一 run 的 `ci/repair-target-fixed/`；当前新验证队列位于 + `ci/stability-final/`，不能将旧镜像的 PASS 直接归给新增 RPC/观察路径修复。 +- 上一版 P12 修复镜像输入 tree 为 `76f794e03757249f37efa96cd0bfcad064be4388`； + 32 条历史容器失败记录所对应的 case 已全部在此镜像实际执行并通过， + 精确映射见 `new-image-case-progress.json`。全量结束后的 scope guard 确认 + 原有 7 个停止容器及卷身份完全不变。物理恢复与新提交的远端 CI 仍须完成。 + +## Run 41003611925：8 个失败 job + +- [ ] **01 / AccessTests**：review accepted policy 的测试前置条件。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123893403740)。 + - 首错：`retained_quorum_startup_does_not_block_discovery_or_confuse_liveness_with_health` + 在 retained quorum 启动时返回 `accepted node policy is absent`。 + - 根因已确认：fixture 未先接受启动所要求的 node policy。 + - 当前处理：启动 trust/profile 前先 `accept_node_policy`，明确 3 voters。 + 文件：`container/crowdb-monitor/tests/kv_resume_test.rs`。 + - 验证：后一次原始 run 的该 job 成功；本地 access 完整组及 exact case 通过。 + - Review：policy 接受顺序是否符合真实启动流程；是否仍分别验证 discovery、 + liveness 与 health；新 SHA 的 AccessTests 结果待补。 + +- [ ] **02 / ContainerDeployment (deployment-reference)**：review diskless + policy 对 Group 0 的要求。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123899615882)。 + - 首错：`direct_docker_three_nodes` 初始化 HTTP 500，明确报告 + `accepted node policy has no enabled Paxos KV listeners`。 + - 根因已确认：diskless/manual fixture 把初始化 Group 0 所需的 Paxos KV 也禁用了。 + - 当前处理：共享 setup choices 保留已接受的 KV listener 10000/10100。 + 文件：`e2e-test/profiles/setup_choices.py`、`e2e-test/console_native_policy.py`。 + - 验证：前一镜像 exact case 与 fresh reference 重跑通过；P12 修复镜像 + Docker 三节点完整通过，约 215 秒。 + - Review：diskless 与“无 Group 0”是否正确区分；单/三节点、保留卷、显式非 voter + 恢复及 Compose 覆盖是否完整；清理必须只作用于自己记录的资源。 + +- [ ] **03 / S3E2E**：review Go SDK 注入后的 cleanup 证据缺口。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123899615935)。 + - 首错：`s3.sdk::go` 普通流程先通过；`after-mpu` 故意失败本应返回 42， + 后续 `cleanup FAILED` 把退出码改为 1。 + - **旧根因未确定**：日志没有具体操作/API code，原始 artifact 也无法区分 + ListMultipartUploads、AbortMultipartUpload、ListObjectsV2、DeleteBucket 等失败点。 + 不能声称 cleanup 算法已被修复。 + - 当前处理：记录脱敏的 Smithy operation/API error code；保留原 cleanup 流程与期限。 + 文件：`e2e-test/clients/s3-go/main.go`、`multipart.go`、`e2e-test/clients/s3-sdk.sh`。 + - 验证：第二次原始 run、前一镜像完整验收及P12 修复镜像 exact case 均通过； + 本轮日志明确确认注入后的 bucket/MPU cleanup 完成。 + - Review：确认此项保留为历史原因未知；核对新 SHA 的 Go 场景。 + 若复现，用独立 artifact root 跑 `s3.sdk::go`，保留增强错误、Access 请求日志及 + bucket/MPU 只读状态后再修复；不增加重试、不放宽清理成功条件。 + +- [ ] **04 / ContainerConsole (console-native-data)**：review 9 个数据/拓扑场景。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123899615992)。 + - 首错：Dataset browse 初始化的 disabled-KV policy 错误；其余表现为 + No route、SSH 断开和 socket 错误。 + - 根因已确认：9 个实际失败 fixture 的节点原始日志均有 policy 错误和 exit 1， + 发生在各自网络症状之前，不能归因为跨 job 的共享 Docker 清理。 + - 当前处理：复用保留 Group 0 listener 的 diskless policy。 + 文件：`e2e-test/profiles/setup_choices.py`、`e2e-test/console_native_policy.py`。 + - 验证：前一镜像 9 个 exact case 全通过；第二次原始 run 的该 job 成功。 + P12 修复镜像 9 个 exact case 也已全部通过。 + - Review：逐一核对 Dataset browse/Split/capabilities、六节点往返、三种多 rack/ + 子集拓扑、mixed units 和 maintenance/status badges；启动失败与实际 UI 断言分开。 + 同时检查预算边界：`e2e-test/scenarios.py` 中 Dataset browser client 总期限为 + 360 秒，而 `e2e-test/tests/console/native_balance.rs` 的 + `wait_balanced` 内部上限为 600 秒；本轮 browse/import 的 + 总 client 时间约 287 秒。明确各阶段应达到的准备条件和计时，不直接延长期限。 + - 对应 case（均以 `console::` 开头）：`browser_dataset::browse`、 + `browser_dataset::split_revisions`、`browser_dataset::capabilities`、 + `browser_data::six_node_round_trips`、`browser_topology::multi_rack_leaders`、 + `browser_topology::independent_subsets`、`browser_topology::simple_and_complex`、 + `native_geometry::mixed_units_reject_consumers_without_starting_services`、 + `native_browser::disk_maintenance_and_status_badges`。 + +- [ ] **05 / ContainerConsole (console-browser)**：review 两个 Group 0 bootstrap 场景。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123899616023)。 + - 首错：replica navigation 和 logical create 的 public bootstrap 返回 No route。 + - 根因已确认：两个 fixture 的三个节点均先因 disabled-KV policy exit 1， + inspect 中 IP 已被清空。 + - 当前处理：共享 accepted diskless policy 保留 Paxos KV。 + 文件：`e2e-test/profiles/setup_choices.py`、`e2e-test/console_native_policy.py`。 + - 验证：两个 exact case 的前一镜像完整验收通过；第二次原始 run 该 job 成功。 + P12 修复镜像两个场景均已实际执行并通过。 + - Review：不把已退出进程的连接错误当成独立网络缺陷，核对 bootstrap 后的实际断言。 + +- [ ] **06 / ContainerConsole (console-inspection)**：review inspection 启动与 UI 边界。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123899616075)。 + - 首错：`page_layout_and_iceberg_inspection` 在 bootstrap 阶段 No route。 + - 根因已确认：三个节点原始日志有 disabled-KV policy 错误、exit 1、OOM false 和空 IP。 + 失败发生在 inspection 断言之前。 + - 当前处理：统一 accepted policy；文件同上。 + - 验证：前一镜像 exact case 通过；第二次原始 run 该 job 成功。 + P12 修复镜像进入全部 13 个 Playwright 断言并通过，总 client 约 326 秒。 + - Review：核对布局和 Iceberg inspection 的实际断言，区分 fixture 启动错误。 + +- [ ] **07 / ContainerConsole (console-transition)**：review journal transition 的真实前置条件。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123899616076)。 + - 首错：`production_split_and_transfer_journals` bootstrap 超时。 + - 根因已确认：node0 先因 disabled-KV policy exit 1，随后 bootstrap 等待超时； + 不能把这次失败说成 journal 状态转换断言失败。 + - 当前处理：统一 accepted policy；文件同上。 + - 验证：前一镜像 exact case 通过；第二次原始 run 该 job 成功。 + P12 修复镜像实际执行 split/transfer journal 断言并通过。 + - Review：检查 split/transfer journal 的状态转换，保留原有等待预算。 + +- [ ] **08 / ContainerConsole (console-native-operations)**：review 真实启动失败与同批级联。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123899616153)。 + - 首错：`diskdb_and_kv_independence` 初始化中的 SSH authentication failed； + 该 job 共打印 11 个失败 case。 + - 根因已确认：10 个实际失败 fixture 均有本地 disabled-KV policy exit 1。 + `binding_recommendations` 已准备但未派发浏览器：后一个 fixture 启动失败， + `ExitStack` 将同一异常传给已进入的 peer，后者按自己的 case 名打印失败。 + - 当前处理：修复实际 policy 前置条件；没有跨 fixture 凭据或 SSH target 共享的证据。 + 后续新增 `CaseNotExecuted`，同批未派发 peer 保留完整清理，并在 fixture、 + report 和 measurement 单独记为 `not_executed`，不能计作 PASS 或独立 case + 失败。真正的启动错误和 peer 自身的诊断/清理错误分别保留，batch 仍失败。 + 正常 client 后的清理失败打印自己的 case；已有首错时，后续清理错误单独 + 打印且保留原异常优先，不增加重复的 `FAILED CASE` 记录。 + 文件：`e2e-test/browser_batch.py`、`e2e-test/fixture.py`、 + `e2e-test/fixture_result.py`、`e2e-test/report.py`、 + `tools/test-metrics/measure.py` 及对应 Python 测试，以及共享 policy。 + - 验证：前一镜像 11 个 exact case 均通过；partial-startup 回归确认不派发 clients + 且清理已启动 peer。P12 修复镜像 11 个场景也全部通过。新增归因回归在旧实现 + 确定性失败，修复后完整 fixture suite 290 tests 通过；没有操作真实容器。 + - Review：逐项核对这 11 个场景及其实际派发状态，并核对新的未执行统计、 + startup/client/cleanup 各自归属及 fail-fast 清理；不将它扩大解释为凭据共享 bug。 + - 对应 case（均以 `console::` 开头）：`browser_lifecycle::diskdb_and_kv_independence`、 + `browser_capacity::binding_recommendations`（原 run 未执行）、 + `browser_capacity::mounted_crud_and_owner_binding`、`native_browser::auxiliary_process_menus`、 + `native_browser::interrupted_plan_reconciliation`、 + `native_browser::storage_windows_and_access_recovery`、 + `native_browser::large_journal_records_and_restart`、 + `native_load::production_load_keeps_heartbeats_and_durable_bytes_across_owner_restart`、 + `native_browser::chunk_ownership_scope`、 + `native_browser::multipart_upload_and_physical_navigation`、 + `native_binding::prebound_groups_acquire_live_owners_and_require_explicit_binding_repair`。 + +## Run 41128190276：5 个失败 job + +- [ ] **09 / UnitTests**：review leader role 与可提交状态的区别。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41128190276/job/124312015280)。 + - 首错：`g3_leader_change::leader_change_simulation` 的第二笔单次写入未通过新 leader 提交。 + - 已支持的原因：旧等待只看 leader role；focused regression 证明 role 可能先于 + 当前 proposing term 和有效 quorum lease。旧日志没有保留该次 RPC reply code， + 不应拿后续故意注入的 WAL 错误替代首错。 + - 当前处理：等待既有 proposing-term/quorum-lease readiness，并保护健康 leader + 在当前 quorum lease 内不投票离任。 + 文件：`lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs`、 + `lib/crowdb-kv/src/cluster/local_replica_election.rs`、 + `lib/crowdb-kv/tests/election_test/lease_test.rs`、 + `lib/crowdb-kv/tests/election_test/vote_test.rs`。 + - 验证：exact case、role-before-lease 和 quorum-lease vote 回归均通过； + 本地 core 1165 tests / 91 binaries 通过。 + - Review：准备条件不能替代提交断言；检查选举/恢复交互、无新增热路径锁， + 并核对新 SHA 的完整 UnitTests。 + +- [ ] **10 / ContainerDeployment (deployment-services)**：review KV-only readiness 与实际端口。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41128190276/job/124321492635)。 + - 首错：`monitor_creates_real_kv_groups_then_validates_after_restart` 与 + `selected_kv_services_recover_acknowledged_data` 一直不能 Ready。 + - 已确认问题：diskless KV-only policy 被要求提供普通 storage metadata；member + 收集还假定 Web 使用 9090，而未使用已注册端口。 + - 当前处理：按 `requires_default_preparation` 收集确实需要的依赖，使用 `node.web_port`。 + 文件：`lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs`、 + `lib/crowdb-console-shared/src/ops/cluster/readiness/collect/members.rs`、 + `lib/crowdb-console-shared/tests/cluster_readiness_test.rs`、 + `lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs`。 + - 验证：前一镜像 deployment-services 完整组、两个 exact case，以及 + KV-only offset-port/no-storage-preparation 回归通过。 + - Review：检查依赖来源符合 accepted policy，普通 storage readiness 仍严格验证； + P12 修复镜像两个 exact case 已通过；新 SHA 完整 job 仍须核对。 + +- [ ] **11 / ContainerConsole (console-api)**：review listener 来源和 terminal fault 的作用域。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41128190276/job/124321492786)。 + - 首错：plain Docker attachment 向旧 S3 端口 9091 请求；terminal writer + 检查旧 health 端口 9094，连接失败。 + - 当前处理：plain Docker public URI 改为此 fixture 使用的 S3/Iceberg/Dataset + 端口 16000/17000/18000；terminal health 从 accepted listener policy 读取。 + terminal 用例在 owned network namespace 注入 DiskIO outage。之后本地真实复现的 + ChunkKV 同步 native I/O 阻塞 executor/lease 问题另由 worker offload 修复。 + 文件：`e2e-test/console_cli_fixture.py`、`e2e-test/console_cli_recovery.py`、 + `e2e-test/console_cli_target.py`、`e2e-test/console_cli_io_fault.py`、 + `lib/crowdb-chunk-kv/src/partition/native.rs`、 + `app/crowdb-chunk-kv-server/src/storage.rs` 及相关回归。 + - 验证：accepted-listener 回归、console-api 完整组和两个 exact case 通过； + P12 修复镜像这两个 case 已完整通过。 + - Review：区分旧端口错误与后来复现的 liveness bug；故障确实作用于 I/O， + 原 lease/terminal replay/Stop fencing 断言和资源限制保留。 + +- [ ] **12 / IcebergE2E (iceberg-gc-cleanup)**：review 前台负载与实际删除窗口的重叠。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41128190276/job/124321492803)。 + - 首错：`restarted_cleaners_finish_large_and_sparse_backlogs_beside_an_invalid_catalog` + 未观察到成功 SDK request 的完成时刻落在观测到的 active-deletion 区间。 + - 根因证据：GC 在约 117 秒完成并执行了 3 次 restart；有限、同步步调的 + 48 cycles 没有保证请求与删除窗口相交，负载还先于 cleanup 结束。 + 这不证明 GC 自身未完成。 + - 当前处理：压力持续至 GC 和小写入均结束，保留每类最少 48 次操作， + 错开 4 个 workers，继续要求真实成功请求重叠；FileReader leaf cache 的 + 性能修复单独记录。 + 文件：`e2e-test/tests/access_iceberg/gc/sdk_pressure.py`、 + `e2e-test/tests/access_iceberg/gc/convergence/pressure.rs`、 + `e2e-test/tests/access_iceberg/gc/notifications/matrix.rs`、 + `e2e-test/tests/access_iceberg/gc/notifications/matrix/records.rs`、 + `lib/crowdb-access-iceberg/src/file/reader.rs`。 + - 验证：focused 重跑、前一镜像完整验收通过；新镜像 exact matrix 约 123 秒通过， + inline GC 前台并发及完整退休宽限期也通过。 + - Review：确认 overlap 是从实际事件/单次请求时间计算，原断言未放宽； + 不从该场景推断单独的生产 latency SLO。 + +- [ ] **13 / ContainerConsole (console-native-operations)**：review journal UI 的 coherent observation。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41128190276/job/124321492894)。 + - 首错：`journal_identity_and_owner_recovery` 导航后，current-owner 条件 + 在原 3000 ms 窗口内未选出 coherent journal observation。 + - 当前处理：Rust fixture 先创建 owned witness、等待已有分区的 ownership 稳定； + 浏览器验证 witness 并保留同一 identity/lifecycle 断言和 step timing。 + 之后本地 native executor/lease 复现由 offload 修复。 + 文件：`e2e-test/tests/console/native_browser.rs`、 + `e2e-test/tests/console/native_balance.rs`、 + `e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts`、 + `lib/crowdb-chunk-kv/tests/native_runtime_test.rs`。 + - 验证：focused native-journal 约 28 秒通过，前一镜像 exact case 通过； + P12 修复镜像对应场景约 16 秒通过，原 3 秒 UI 断言不变。 + - Review：检查 fixture 的真实前置条件、UI 的 current-owner/epoch/cursor 一致性， + 保留 3 秒断言以及 owner-loss/restart/stale-cursor 场景。 + +## Run 41159636766：3 个失败 job + +这轮来自更新的远端 head；三个失败都不能套用同名旧 job 的首错。完整日志、 +浏览器 trace、API 时间线和当前 exact-case 映射保存在证据目录的 +`run-41159636766/` 下。 + +- [ ] **14 / ContainerConsole (console-native-operations)**：review 目标进程的 + 生命周期与聚合观察延迟。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41159636766/job/124418859705)。 + - 唯一真实失败：`console::native_browser::auxiliary_process_menus`。 + DiskIO Start 响应成功，但 `diskio-1 new process` 的 3000 ms predicate 超时。 + - 已确认的首个分歧：Start 响应已有新 PID `1389`；该次 predicate 内的 + `GET /api/servers` 约 3012 ms 才返回，晚于 predicate 的失败时刻。 + 返回内容也已包含目标 PID 和 health up，不能解释为 DiskIO 没有启动。 + 同响应中另一节点的整组 PID 缺失，支持等待无关节点 SSH 的解释； + 但原记录没有各内部阶段的慢 span，不能确定 intent refresh、remote process、 + health probe 各耗时多少，**该节点观察为什么不可用仍未确定**。 + - 当前修复：authority、每个节点的实际进程和每个服务的 endpoint health + 独立采样发布；GET 读取带身份/新鲜度 fence 的 remote facts，不等待 KV、 + SSH 或 endpoint 探测。仍读取本地 binding 以立即拒绝旧身份。 + 三秒从采样开始计，读取不刷新 age；mutation 期间及失败返回后的迟到结果 + 也必须失效。monitor 私有 Processes 将 live PID 与实际执行 operation/ + bootstrap 绑定，保留 Group 0 故障时的 actual PID 观察。 + 新校验同时发现 KV 自动恢复后旧 receipt PID 未更新的边界，补只更新 + 已验证当前执行回执的恢复路径，不再次 restart，不接纳 foreign bootstrap。 + 不将上述新发现说成原 CI 的已证实现场根因。 + 文件：`e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts`、 + `e2e-test/clients/console-ui/e2e/fixtures/nativeProcess.ts`，以及 + `app/crowdb-web/src/services/observation.rs`、 + `app/crowdb-web/src/services/remote.rs`、`services/observation/`、 + `container/crowdb-monitor/src/node/runtime/services/` 及其 process observation。 + - 验证：前一镜像 exact case 实际通过，Playwright 单 test 约 13.3 秒。 + 新生产观察路径的九个 focused 回归和两个真实 monitor 进程回归通过：慢 peer/同节点慢 sibling、 + concurrent read、expiry/identity、same-op late sample、unknown mutation、 + Group 0 故障、pending/exhaustion 和删除不复活；完整组件与 lint 已通过， + 新镜像和 E2E 门禁仍待完成。原 combined 三秒 predicate 保留，不通过移动 deadline 修复。 + - Review：分别计时 Start response、目标 typed process、聚合 inventory 和 DOM。 + 通过 owned monitor PID 与同 namespace `/proc` 核对 typed target，继续保留 + inventory 的唯一 identity、18 条记录、DOM 和健康检查及 combined 三秒。 + 完成原 test 的当前精确计时后，再补 Start body identity/data-preserved 与 + siblings PID/generation/operation 快照；不能把一次 PASS 说成长期稳定率。 + +- [ ] **15 / ContainerDeployment (deployment-services)**：review retained KV + restart 后读取 readiness 的顺序。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41159636766/job/124418859709)。 + - 唯一真实失败： + `deployment::kv_bootstrap::monitor_creates_real_kv_groups_then_validates_after_restart`。 + verify 的首个 `/stores/0/groups/1/ready` 请求返回 503，原因是 + `no leader elected`；预期为 200。不是上一轮 KV-only storage dependency 错误。 + - 已确认问题：verify 在重启后直接断言已有 Group 1 ready，没有先等待选举完成。 + - 当前处理：`cdf023d6` 让 write/verify 两个阶段都先调用现有 + `wait_for_witness_group`，再执行单次 200 断言和 acknowledged-data 验证； + 保留原有四秒准备期限，不重试写入。 + 文件：`e2e-test/tests/deployment/kv_bootstrap.rs`。 + - 验证：P12 修复镜像 exact case 通过；新 SHA 的完整 deployment-services 待核对。 + - Review：ready 必须包含已选 leader 和可提交条件，不能只看进程存活；核对 + 重启后的 retained group/data 验证确实执行，且 readiness 期限没有延长。 + +- [ ] **16 / ContainerDeployment (deployment-reference)**：review 自动加入与 + 显式 admit 的排序。 + [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41159636766/job/124418859712)。 + - 唯一真实失败：`deployment::reference::direct_docker_three_nodes`。 + Docker 三节点主体验证已经 PASS，后续手工 admit 一个 nonvoter 得到 409， + 原始响应说明 `service operation already running`。 + - 已确认问题：手工 admit 与已有 service operation 冲突；不是第一轮的 + disabled-KV policy 启动错误。旧 fixture 未先停 AutoJoin，源码与针对性修复 + 支持 AutoJoin/manual admission 竞争的解释；原 artifact 没有记录持有 operation + 的 actor/id，确切竞争者身份仍属有依据的推断。 + - 当前处理：`cdf023d6` 先停 AutoJoin 并等待 disabled 状态，再执行一次显式 admit。 + 文件：`e2e-test/oci_reference_recovery.py`。 + - 验证：P12 修复镜像 exact case 完整通过；原先的主体验证、nonvoter 行为及 + scoped cleanup 都保留。 + - Review：确认自动操作已结束后再手工操作；保留单次 admit 及 nonvoter 断言， + 不用重试 409 掩盖竞争；核对新 SHA 的完整 deployment-reference。 + +## 最后逐项 review 的记录 + +- [ ] **补齐最终验收证据**：更新新镜像 396-case 账本、受影响的物理恢复， + 以及本次新 SHA 的全部 27 个远端 job；保留各自输入身份和覆盖边界。 +- [ ] **逐项共同 review 01–16**：每项明确“原因确认程度、修复是否充分、 + 回归是否能捕获原问题、还需做什么”。保留 Go 旧原因未知的事实， + 核对 browser batch 新增的未执行/级联错误归因及独立统计。 +- [ ] **归档结论与后续工作**：将确定需要实现的稳定性工作拆成具体任务； + 不以反复重跑得到一次绿色、延长时间或删除断言代替修复。 +- [ ] **Review 物理 P12 的 EC 可用性边界**：此项不是新增失败 CI job。 + 新镜像的 catalog 接管在 40.184 秒通过；新 S3 witness 两份镜像均在 A， + B 停止时三个 A 入口都完整读回。旧 baseline witness 已转为 EC 8+4， + B 上有 6 个分片，超出 4 个分片的恢复预算；旧 harness 在 fault witness + 验证中额外遍历所有历史 witness，因旧对象暂时不可读而失败。 + 只读复现已区分新/旧对象，B 返回后两者五入口均通过，未重放 mutation。 + Review 既有 P12 合同的两阶段验证：停机时严格验证本次新写入；恢复后 + 保留所有历史数据的完整审计。若需要旧 EC 数据也抗整机故障，须另行确定 + 拓扑、放置和可用性合同；KV quorum 不能替代该承诺。 +- [ ] **复核 P12 幸存者的 native 崩溃修复及物理验证**：此项也不是新增失败 CI job。 + 完整区间核对发现 A3 ChunkKV 从 PID 451/generation 2 变为 + PID 66135/generation 3，其余 20 个 A 服务身份稳定。此前 stderr 为空， + 新增 `std::system_error: Resource deadlock avoided`；monitor 记录了停止与重启， + 尚无原进程 core。已通过真实 Rust FFI 最后 owner eviction 复现同样异常， + gdb 证实 `RpcServer` 析构在 response completion 内触发 `Worker::join` 错误 35。 + 共享路由 owner 已改为外部生命周期线程释放，覆盖 response/close/timeout、 + 递归退休、阻塞析构的 backlog 和进程退出顺序；修复后的完整 FFI、storage、 + fmt/clippy 已通过。旧物理进程的精确栈仍未知，不能声称已直接取得它的栈。 + Review 另发现 C ABI send/chain/slab 在同步 completion 释放 wrapper 后仍会 + 读它；四个确定性原生用例在修复前失败，保活 native client/connection/transport + 至返回后均通过。完整 C++、owned tree transport、storage、fmt/tree-lint 已通过。 + 按新顺序先收尾此修复与 CI，再回到物理验证；P14 门禁继续保留。 + 文件：`lib/crowdb-tree/ffi/src/chunk/transport.rs`、 + `lib/crowdb-rpc/ffi/src/owned_route.rs`、 + `lib/crowdb-rpc/src/client/client.cpp`、 + `lib/crowdb-rpc/src/transport/socket_transport.cpp`。 + 不以跳过 join、捕获后忽略异常、扩大健康预算或容忍静默重启代替生命周期修复。 +- [ ] **定位 P13 恢复后的 Iceberg 完整读取超时**：此项不是新增失败 CI job。 + 五个入口的 S3 SF1 原始文件均完整读回；并发 Iceberg 审计先后打印 + `NETWORK_CONNECTION during GetObject`、curl 28、三秒低速超时。 + 三个 verifier 真实失败，另两个在保留首错后收到定向 SIGTERM；后两项计为 + 中止，Dataset/history 尚未执行，不能把包装层的 exit -15 当成最初根因。 + 并发 stdout 没有入口标签，三次错误不能仅按打印顺序归属到 A1。 + 不能因原始 S3 校验通过就忽略 Iceberg scan 的失败，也不延长客户端期限掩盖原因。 + 保留本次并发、镜像、服务日志和原始错误，先确认进程身份与恢复状态， + 再复现首错并检查其与 native 崩溃的关系。 + B1/B2 此次 ChunkKV 重启已有 lease 到期、主动退出、正常 join 的日志闭环， + stderr 为空；不是已证实的同一 C++ 异常。下次每个入口单独保存 stdout、 + endpoint、起止时间与退出码，再聚合状态，避免并发错误归属缺失。 + +## 文件与验证入口 + +- 文件按上面 01–16 各项归属列出；跨项共同检查 policy、fixture 生命周期与 + `e2e-test/oci_shortcuts.py` 的资源归属清理，不将独立问题混为一个根因。 +- 单元/组件：KV election/readiness、console cluster readiness、accepted-listener、 + partial browser startup、OCI shortcut cleanup、native worker 和 FileReader 回归。 +- 集成：完整 core/access/console/storage 组，以及精确的原始非容器失败 case。 +- E2E：396 个 canonical scope/case、并发隔离、原失败 case 的实际执行、Go + 故意注入后的 cleanup、GC 真实 overlap，以及新镜像上的物理恢复。 +- 远端：推送后关联本次 SHA 的新 run，核对 27 个展开后的 job，而非仅看汇总状态。 diff --git a/doc/working/plan-physical-node-ci-handoff.md b/doc/working/plan-physical-node-ci-handoff.md index b22f8c25e..351ee1b4d 100644 --- a/doc/working/plan-physical-node-ci-handoff.md +++ b/doc/working/plan-physical-node-ci-handoff.md @@ -4,362 +4,396 @@ # Physical Node and CI Handoff Plan Upstream: [Physical Node Test Plan](../dev/physical-node-test-plan.md), -[OCI image operations](../../container/oci-image/README.md), and -[container acceptance](../../e2e-test/README.md). - -Goal: validate the committed recovery fixes with a fresh five-container run -across two physical hosts, including 30 continuous minutes of active load, -the recovery matrix, and the remaining CI cases. - -This is a temporary handoff, updated on 2026-10-31. Delete it after the remaining -work is complete and the final report is retained in the new run directory. - -## 1. Read this first - -- The user deliberately deleted `.crowdb-runtime` and instructed us to retest - next time. The old physical run is interrupted, not an acceptance pass. -- **The full 30-minute workload has not completed.** The formal final workload - had not started at the last recorded checkpoint. Earlier short runs do not - count toward the requested duration. -- **A complete Iceberg SF1 load has not passed.** The latest election fix still - needs physical load verification. Passing unit tests and retained-volume - startup do not establish that the original load failure is fixed. -- The deleted directory contained harness scripts, wheels, logs, ledgers, - image receipts, and controller state. Historical paths below are identifiers, - not available evidence. Reconstruct a fresh harness from current interfaces; - do not launch commands that depend on those deleted files. -- The loader's S3/Dataset implementation is committed and pushed. Its later - 0.1.2 filename changes and all CROWDB changes are committed locally. - **Do not push CROWDB without new user authorization.** No further push was - requested when committing the remaining changes. -- No fresh run was started for this handoff. Keep new artifacts under - `.crowdb-runtime`, as requested by the user. - -## 2. Repositories and publication boundaries - -- **CROWDB**: `/cjdata/cpp/crowdb2`, branch `task-fix-build`. - The recovery checkpoint ends at - `d396a2207e5a13ef61ee1fe04e68ee40b0f8622c`. - These three local commits contain the recovery work: - - `cdf023d642da40f6b2db03c54b3f1a4fdc5fd049`: - `Reduce recovery memory and harden storage liveness`. - - `81d07f03398aaa144e8dcec1187bff0a0d18f91b`: - `Prefer nearby mirrors during partition recovery`. - - `d396a2207e5a13ef61ee1fe04e68ee40b0f8622c`: - `Protect leader votes with the current quorum lease`. - The follow-up commit containing this handoff also records the remaining - OCI cleanup changes. Do not push this repository. -- **TPC loader**: `/cjdata/cpp/crowdb-tpc-loader`, branch `main`. - Current HEAD is `447a1e619550bbdc0150139976e0c442675f92b0`, - `Use readable table-scoped Parquet names for version 0.1.2`. - This follow-up is local only. The local `origin/main` tracking ref remains at - `4cafdc9227871f55942a2b215ccb5a3078ade70d`, - `Add standalone S3 and synthetic Dataset loading`. - This was pushed, including its ancestor `ead5449`, which publishes native - Iceberg upload locations before manifest registration. - [Loader CI run 41194754281](https://github.com/buzzcrow/crowdb-tpc-loader/actions/runs/41194754281) - was recorded successful for Python 3.10, 3.11, and 3.12. -- **Website**: `/cjdata/cpp/crowdb-web`, branch `main`. - `c9306c51c71dd3b71a4daf9255692da53f0f3544`, - `Document standalone S3 and Dataset loader commands`, is local only. - Do not publish it without authorization. Never read or stage its untracked - `.aws` directory. - -The remaining changes were committed at the user's request; no additional -repository was pushed. - -## 3. Physical environment and remaining resources - -- Host A: `192.168.31.129`, previously using interface `wlp37s0`. - Nodes A1/A2/A3 use port offsets 0/1/2. -- Host B: `192.168.31.184`, previously using interface `wlp4s0`. - Nodes B1/B2 use port offsets 0/1. -- Verify addresses, interfaces, host identity, clocks, and ownership again. - These are two physical failure domains and five container voters. -- Existing Ed25519 SSH access works noninteractively. A fresh `BatchMode` - connection to B succeeded during handoff preparation. Use private local - SSH configuration; do not put login accounts or credentials in documents. -- The previous containers used host networking, `1 CPU / 2048 MiB` each, - one Tokio worker per service, five voters, Groups 0/1/3, and 35 managed - services. Test media was 16 GiB per node with a 1 GiB reserve. -- Old run ID and ownership label: - `physical-20261031T024347Z` and - `org.crowdb.e2e.run=physical-20261031T024347Z`. -- Deleting `.crowdb-runtime` did not remove Docker resources. At handoff, - the three A containers still existed and were running, with the expected - ownership label, CPU/memory limits, and no container-level `OOMKilled` flag. - B containers were not freshly inventoried during this handoff; their last - recorded state was running the same upgraded image. -- The old physical controller exited with status 1. The orphaned CI controller, - which was waiting for workload ledgers that no longer exist, was stopped with - `SIGTERM`; its exit was verified. Do not resume either controller or rely on - old session IDs. -- No persistent volumes were removed for this handoff. Cleanup is unfinished. - Inventory both hosts before stopping, replacing, or deleting any resource. - Do not use Docker-wide prune or remove another task's containers/volumes. - -Read-only starting commands, from the CROWDB root: - -```sh -git status --short -git log -4 --oneline -pixi run docker ps -a --filter label=org.crowdb.e2e.run=physical-20261031T024347Z -pixi run ssh -o BatchMode=yes -o IdentitiesOnly=yes -o ConnectTimeout=5 192.168.31.184 true -``` - -For remote executable/test commands, use remote Pixi with an explicitly verified -project manifest or working directory. A bare remote `pixi run` from the login -directory failed because there was no Pixi manifest there; this was not an SSH -authentication failure. Do not guess the old remote staging directory, whose -local receipt was deleted. - -## 4. Latest root cause: healthy leader votes away its quorum - -The last failed physical Iceberg SF1 run did not report OOM. Six tables committed: -`region`, `nation`, `supplier`, `customer`, `part`, and `partsupp`. -The `orders` data upload completed, but a manifest `UploadPart` returned -`INTERNAL_FAILURE`, leaving its outcome uncertain. `lineitem` failed during -native upload with HTTP 500. These outcomes must not be counted as successful -table commits or blindly replayed. - -The first divergence was in Root Group 0 election handling: - -- A3 was the leader with a current quorum lease and a responsive A1/A2 majority. - The two B voters lagged under load and repeatedly attempted elections. -- Around 11:37 UTC, a B challenger obtained a vote from the healthy leader. - Leadership changed, recovery scanned a backlog of roughly 7,000 slots, - process leases expired, and a native file-block intent reached its ChunkKV - operation deadline. -- `handle_pre_vote` and `handle_request_vote` checked the old self-vote lockout. - That lockout can expire during a long leader tenure even while heartbeat - quorum acknowledgments keep the read lease current. -- Commit `d396a220` rejects both vote paths while the replica is Leader and - `lease_read_until() > now`. Normal voting resumes when that lease expires. - It uses the existing lease and vote-handler synchronization; no new hot-path - lock, caller retry, or larger deadline was introduced. - -Relevant files: - -- `lib/crowdb-kv/src/cluster/local_replica_election.rs`. -- `lib/crowdb-kv/tests/election_test/vote_test.rs`. -- `doc/design/kv/design-crowdb-kv-leader-election.md`, section 6.1. - -Recorded verification before runtime deletion: - -- Both new leased-leader vote tests failed before the fix and passed afterward. -- The final election suite passed 79 cases, including the five-voter case where - two lagging voters cannot obtain the healthy leader's third pre-vote. -- Affected KV/server tests passed 733 cases across 31 test binaries. -- Rust formatting and clippy with warnings denied passed. -- The OCI build passed. All five retained-volume nodes were upgraded, and - 35-service readiness passed in 47.582 seconds under the original 180-second - gate; identity restoration took 2.355 seconds. -- The controller had started read-only reconciliation of prior Iceberg failures - when last observed. No subsequent fresh SF1 success was recorded. - -The locally inspected running A image ID was -`sha256:439f85489106fc8a7d01c3ee265e31445c0879bd95b2578eaea7e573fed838d5`. -The recorded OCI config digest was -`sha256:a1f93e79d6fe463733e7a1b789a746c5fe35811dcd53cac26286dce737054519`. -These identify different image objects; do not equate a config digest with a -Docker image ID. Rebuild and verify both hosts for the next run. - -## 5. Previous fixes and what they establish - -- **Mirror locality, commit `81d07f03`**: partition recovery replayed an - approximately 88 MB WAL very slowly. In the diagnostic sample, 761 of 775 - slow reads selected a remote-rack mirror first. Optional - `ChunkReadPolicy.preferred_node_id` orders complete mirror hints by local - node, then local rack, preserving fallback and the original deadline. - Unknown or incomplete placement leaves the original order intact. - Production ChunkKV supplies its node ID; cache contents and replica membership - are unchanged. Files are under `lib/crowdb-chunk-client/src/chunk/`, its - `tests/`, and `app/crowdb-chunk-kv-server/src/storage.rs`. - Recorded results: 427 affected tests passed, one opt-in RDMA test ignored, - clippy passed, and retained-volume readiness completed in 46.796 seconds. - The later SF1 run exposed the separate election issue above. -- **Recovery memory, commit `cdf023d6`**: preserve the replay-built WAL index, - stream replay while retaining independent highest accepted/promise records, - and preserve current term/vote records during WAL garbage collection. - Other changes bound ChunkKV replay buffers, release encoded snapshot pages, - and avoid prefaulting unused frame-cache capacity. Focused memory experiments - and C++/FFI/tree checks passed. An isolated exact-volume Root replay improved - readiness from 97.756 to 1.791 seconds with matching recovered state. - This isolated diagnostic does not replace five-node acceptance. -- **Storage liveness in the same checkpoint**: honor the independently renewed - WAL process lease during ownership transfer; preserve staged strip state - through empty liveness refreshes; renew DiskDB ownership by CAS; refresh - Monitor route authority before optional old-owner confirmation; and retain - bounded recovery progress and mirror-rack consistency. -- **CI fixes in the same checkpoint**: disable initial auto-join before the - explicit fourth-node admission case, and accept only the specified bootstrap - readiness responses under the original limits. Earlier local reruns passed - the three previously failing cases, but closing verification on current HEAD - is still required. - -Do not interpret a previous image's pass as a pass for current HEAD. The old -raw receipts were deleted, so retain fresh evidence for any final claim. - -## 6. TPC loader implementation and acceptance limits - -- The committed implementation supports native Iceberg loading, standalone - S3 generation/upload/verification, and generated Dataset samples. Shared - journaling records acknowledged, failed, and uncertain outcomes. -- Standalone S3 explicitly uses zero SDK mutation retries. A regression checks - that HTTP 500 produces exactly one PUT. Failed/uncertain candidates are - retained for read-only reconciliation instead of automatic mutation replay - or generic deletion. -- Recorded exact-commit checks for the pushed 0.1.1 implementation: 252 tests - passed, six opt-in cases skipped; Ruff, wheel/sdist build, and Twine checks - passed. Its pushed CI was green. -- The subsequent 0.1.2 commit records readable table-scoped Parquet filenames, - the version bump, and matching usage/changelog/test changes. Fresh local - checks also passed: 252 tests, six opt-in skips, Ruff lint/format for 63 files, - wheel/sdist build, and Twine. These are local checks, not fresh remote - acceptance or a published release. New receipts are under - `.crowdb-runtime/artifacts/commit-remaining-20261031T120321Z/`. -- Older physical runs passed Iceberg SF0.01 for eight tables/86,805 rows, - standalone S3 SF1 for eight files/8,661,245 rows/340,201,375 bytes, and Dataset - readback for eight samples. These used older backend states and do not close - the current physical acceptance. The S3 result also predates the final - zero-retry acceptance setup. -- For the next run, build a wheel from the exact committed loader tree, save - its SHA-256 and source identity, and install it into a dedicated run-local - environment. Inspect any new worktree changes first. Recheck all three paths - against the new CROWDB image, including the 0.1.2 filename behavior. - -## 7. Remaining changes now captured - -The user subsequently requested committing the edits previously kept separate -from recovery work. They are now captured without overwriting their content: - -- CROWDB: `container/oci-image/README.md`, `clean.py`, - `docker_local_cleanup.py`, `host_cleanup.py`, `tests/cleanup_cli_test.py`, - `tests/docker_local_test.py`, `tests/host_cleanup_test.py`, untracked - `docker_garbage.py` and `tests/docker_garbage_test.py` under that directory; - `container/single-node-container/README.md`; - `doc/design/deploy/design-crowdb-oci-image.md`; and `pixi.toml`. -- Loader: `CHANGELOG.md`, `README.md`, `pyproject.toml`, - `src/crowdb_tpc_loader/__init__.py`, `src/crowdb_tpc_loader/loader.py`, and - `tests/unit/test_commits.py`. -- CROWDB's cleanup change preserves running recorded clusters by default; - explicit `--cluster`/`--all` can stop selected clusters. Unscoped cleanup also - collects unused Docker resources, while retaining protected receipts and data - according to `--keep-data`. Its affected OCI tooling suite passed all 150 - tests before commit. No cleanup command was run against the physical fixtures - as part of this commit validation. -- Both `/cjdata/cpp/crowdb-web` and `/cjdata/cpp/buzzcrow.github.io` still have - an untracked `.aws` path. Neither was read or staged. There were no other - pending website/journal changes when inspected. - -Capture a fresh `git status` before further edits and preserve any new user work. -The original physical-run hash receipts and other evidence remain deleted. - -## 8. Remaining execution, in dependency order - -- [x] **Implementation checkpoint**: commit the recovery and election changes - locally; push the completed loader implementation. Files: the commit scopes - in sections 2, 4, 5, and 6. -- [x] **Handoff and interruption record**: preserve this state outside the - deleted runtime directory; defer fresh testing to the next session as the - user requested. Files: this document. -- [x] **Remaining changes checkpoint**: commit the OCI cleanup behavior and - handoff in CROWDB, and the 0.1.2 filename changes in the loader, after local - affected checks. Files: the paths in section 7 and this document. -- [ ] **Fresh preparation**: inventory old owned resources on both hosts, - preserve user edits, create a new run ID, reconstruct the harness, build one - frozen-source OCI serially, import/verify it on both hosts, and establish - fresh baseline data and identities. Files: a new run's `run.json`, harness - source, image receipts, and P00-P03 results. -- [ ] **First-divergence regression**: complete a fresh native Iceberg SF1 load - with full file checksums and bidirectional SQL verification. Observe Root - leases, elections, backlog, process epochs, and memory. On failure, print the - exact error, capture the first divergence, and stop dependent work. - Files: new loader reports, operation ledger, and service logs. -- [ ] **Scale and protocol acceptance**: pass isolated native 512-record and - 4096-record cases across all five owners with full readback; verify fresh S3 - SF1 with zero mutation retries and generated Dataset publication/readback. - Files: P04/P05 ledgers and loader reports. -- [ ] **Thirty-minute workload**: run P06-P08 for 1,800 continuous active - seconds after preparation passes. Files: durable native/protocol ledgers, - complete metrics, and final data audit. -- [ ] **Recovery matrix**: execute P09-P16 using the new soak data, including - retained-volume recreation, the unresponsive child, follower/leader lifecycle, - two-node B outage, three-node A quorum loss, allocation safe retry, backlog, - full cold restart, and final occupancy audit. Files: per-case receipts and - fixed-checkpoint/data verification. -- [ ] **Closing CI**: rerun the original failures and affected protocol suites - against current inputs. Files: CI artifacts and final result matrix. -- [ ] **Close the run**: capture failures or the final pass, verify owned-only - cleanup, commit coherent follow-up fixes/results as authorized, and report - unfinished coverage honestly. Do not push CROWDB. Files: `results.json`, - `cleanup.json`, and `summary.md` in the new run directory. - -## 9. Workload and failure contracts to retain - -- Native scale: 512 then at least 4096 records, each 64 KiB, batches of 16, - with the original five-second operation deadline. Cover all five Serving - owners using actual catalog ranges. -- Exclude native key first-byte namespaces reserved by S3, Iceberg, and - Dataset (`S`, `I`, and `D`). A former harness used these prefixes and - contaminated protocol scans. Exact owned-key retirement was recorded before - deletion; do not reuse that harness or delete whole protocol prefixes. -- Intended native soak: one batch in flight, 80/20 put/delete mix with 16 exact - reads per cycle, durable intent/acknowledgment records, bounded live data, and - a final audit of all 4096 keys. Declare actual pacing and concurrency before - starting. Build/deploy/idle time does not count toward 30 minutes. -- Concurrent protocols: bounded S3 put/get/delete plus absence verification; - Iceberg property commit/load/cleanup; Dataset sample publication and complete - field verification. Keep mutation attempts single where required. An unknown - outcome is audited read-only, never retried merely to obtain success. -- Observe all 35 service PIDs and OOM state; collect metrics at least every - five seconds. Readiness observations must postdate the current process start - and be at most three seconds old. Require at least 30% free test media. -- Preserve independent 180-second service-recovery, catalog-takeover, and - per-group fixed-checkpoint catch-up gates. Do not replace them with one - whole-phase duration, increase timeouts, or weaken assertions. -- Any source/harness repair during the soak requires a complete new 30-minute - run after the repair. Earlier 70-second and 148.9-second runs stopped on - harness freshness errors and do not count. -- Run builds, tests, lint, and executables through `pixi run`, including remote - execution. Serialize shared-output builds and freeze production source during - OCI build. Add no new hot-path lock or unsafe exception without following - project rules. - -## 10. Closing CI scope and useful entry points - -Original user-reported runs: - -- [CROWDB run 41003611925](https://github.com/buzzcrow/crowdb/actions/runs/41003611925). -- [CROWDB run 41128190276](https://github.com/buzzcrow/crowdb/actions/runs/41128190276). - -The pending closing queue covered console native operations, deployment -services, deployment reference, Iceberg GC cleanup, and full S3 acceptance. -It did not dispatch on the latest election-fix image before interruption. -Reconstruct commands from current Pixi tasks and `e2e-test/README.md`; old -controller files are gone. Set `CROWDB_E2E_PRINT_SERVICE_ERRORS=1` for diagnosis -and retain complete client/service errors. Container CI does not replace the -physical run. - -Useful files for selecting focused validation: - -- Unit/integration: `lib/crowdb-kv/tests/election_test/vote_test.rs`, - `lib/crowdb-kv/tests/group_test/`, - `lib/crowdb-chunk-client/tests/chunk_reader_test/`, and - `lib/crowdb-chunk-client/tests/common/mirror_locality.rs`. -- Service/transport: `app/crowdb-kv-server/`, - `app/crowdb-chunk-kv-server/`, `app/crowdb-chunkdb/tests/`, and - `container/crowdb-monitor/tests/`. -- Container/protocol: `e2e-test/tests/console/`, - `e2e-test/tests/deployment/`, `e2e-test/tests/access_iceberg/`, and the - S3 acceptance entries documented in the E2E README. -- Design contracts: `doc/design/kv/design-crowdb-kv-leader-election.md`, - `doc/design/chunkio/design-crowdb-chunkio-reader.md`, and - `doc/dev/physical-node-test-plan.md`. - -Historical artifact root, now deleted: -`.crowdb-runtime/artifacts/physical-nodes/physical-20261031T024347Z/`. -Its former controller progression, `CURRENT-STATE.json`, `loader/` reports, -`mirror-locality-*` evidence, and `leader-vote-lease-*` evidence cannot be used -as executable inputs or linked as surviving proof. The commits and this handoff -are the starting point for the next test session. +[CI Stability Review Plan](plan-ci-stability-review.md), +[container acceptance](../../e2e-test/README.md), and +[OCI image operations](../../container/oci-image/README.md). + +Goal: finish CI stability verification and publish CROWDB, then resolve P13 and +complete the two-host physical recovery matrix without losing data or evidence. + +## 1. Current checkpoint: local verification complete, publication next + +Updated 2026-11-01, 17:56 Asia/Shanghai. The user explicitly resumed this plan. +Continue the agreed CI refinement, commit/push, exact-head CI, then physical +recovery sequence. No physical fault or retirement has been dispatched on resume. + +- The first resume scope check failed because all five current physical nodes + are stopped. All seven container/image/volume identities remain unchanged. + A stopped at 14:44 with exit 0; B stopped at 15:14 with exit 137 and no Docker + OOM flag. Both hosts subsequently rebooted. The stop cause is not established. + Preserve `ci/stability-final/resume-host-reboot-scope.json` and + `resume-host-states.json`; do not count this interruption as a physical PASS. +- The original `scope-guard-before.json` remains immutable. The resume guard + uses `scope-guard-resumed-before.json`, preserving exact identities and the + observed stopped state throughout CI. Physical restart remains deferred. + +- The final-image container acceptance has completed: all 396 canonical cases + have a finalized latest PASS; required concurrent isolation also passed. +- The strengthened CLI five cases, real GC three cases, three fresh paired + auxiliary/Journal browser rounds, three fresh Go SDK rounds and instrumented + Dataset browse all passed. The resumed controller exited 0; its closing + scope guard preserved all seven physical containers and volumes exactly. +- The resumed browser queue first failed an immediate Access-health assertion + after ChunkKV replacement. The test now closes dependency readiness within + its existing three-second predicate. Three fresh paired rounds passed; the + earlier failure, traces and read-only diagnosis remain preserved. +- Session 44834, the all-domain controller, also exited 0. Its resumed command + took 5574.963 seconds; the earlier actual CLI failure remains in its attempts. +- The CI pause marker was removed after the normal client refresh and resumed + scope guard passed. The separate physical investigation marker remains. +- No test/workload/fault controller is left running. Physical work remains + paused; the seven retained physical containers and their volumes remain owned + and preserved. Do not restart old session IDs. +- No new commit or push has been made. No new CI run has been triggered. +- Two physical hosts have not completed final acceptance: P13 is still failed, + P14–P19 are unrun, and the latest native lifecycle fix needs physical validation. + +This is a temporary execution handoff. Delete it only after physical completion +and retention of the final report. Keep the separate 16-item CI checklist until +the user finishes joint review; a PASS must not check a human review box. + +## 2. Repository and publication boundary + +- CROWDB root: /cjdata/cpp/crowdb2. +- Branch: task-fix-build; HEAD: + 7a93c4e5fdca9bc39455331bfae9f3082e167a43. +- Four existing local commits are ahead of the remote branch: + cdf023d6 (recovery/liveness), 81d07f03 (nearby mirrors), + d396a220 (healthy leader quorum lease), and 7a93c4e5 (cleanup/handoff). +- The substantial current production/test/documentation changes are still + uncommitted. Preserve them and any new user edits; do not reset or restore. +- Remote task-fix-build was last verified through GitHub during this session at + 8885d2adc351d9c99f0219e18ecf0c4782bc3079. + [PR 55](https://github.com/buzzcrow/crowdb/pull/55) is open. + Fetch and verify remote state again immediately before the eventual push. +- After explicit resume, the agreed order remains: finish local CI stability + gates; commit/push CROWDB; inspect the new exact-head-SHA CI and every expanded + job; then resume P13 and the physical matrix. Physical completion is not a + prerequisite for that CROWDB push. +- Loader and website publication are outside scope. Last recorded loader HEAD + was local 447a1e619550bbdc0150139976e0c442675f92b0, with origin/main at + 4cafdc9227871f55942a2b215ccb5a3078ade70d; website had local c9306c51. + These are historical identities, not freshly checked remote status. + Do not read or stage any .aws directory in any workspace. + +## 3. Evidence and frozen inputs + +All abbreviated paths below are relative to: + + /cjdata/cpp/crowdb2/.crowdb-runtime/artifacts/physical-nodes/physical-20261031T194149Z + +- CURRENT-STATE.json: current pause and next actions. +- ci/stability-final/: final local gates, queues, complete logs, current-image + coverage, independent reviews and pending refinement evidence. +- ci/stability-final/final-current-image-evidence.json: + finalized 396-case coverage and current-image mappings for all original + 32 container failure occurrences plus the newer run's three cases. + Occurrences include repeated case names; they are not 35 distinct cases. +- ci/stability-final/container-coverage.json: latest finalized case per scope/name. +- ci/stability-final/all-queue.json: final all-domain PASS plus preserved failed + attempt. container-all/run-72c5f00c04874d38ae2b97bb70822cd6 is the completed + resumed run. Do not label every historical attempt successful. +- ci/stability-final/refined-e2e-queue.json: all ten queue rows PASS, with + earlier browser attempts preserved. The per-round finalized fixture/client + identities and Dataset timings are in resume-refinement-closure.json. +- ci/stability-final/resume-publication-gates.json: Rust fmt/full clippy, + affected C++ format/tree-lint and diff check PASS. Full E2E TypeScript also + passed after the final test-only dependency-recovery predicate change. +- ci/stability-final/scope-guard-before.json and scope-guard-latest.json: + exact retained A3/B4 container, image, running-state and volume identities. +- The earlier long handoff is preserved at + handoff-history/plan-physical-node-ci-handoff-before-rewrite-20261101T062936Z.md. + It contains superseded historical instructions, not current authorization. + +Final local server image: + +- Tag: crowdb-node:ci-stable. +- Production source tree: 376fdc26a0e96d7810313ea670bf043c4f0dfcae. +- Docker image ID: + sha256:c2bc8bfd6228cf61fa5a73e8ab296766cd880f8eb4fb100efe247fa23c0a55e9. +- OCI config digest: + sha256:c208f00b8c04b085053010227a755dfe114d1e5a756a13f30019422eb91d1bc0. + This is a different object from the Docker image ID. +- Runtime SHA-256: + 7e17ab9c83a2d4c0411114c39377cd4cabb28d2330791dd5d09d3dce3ac98cfd. +- OCI archive: image/stability-final/crowdb.oci.tar; SHA-256: + 565ffcde6e39f3286d3a671dea8310e646c467cac8d0a9e6af4e598a662100d8. +- Image revision label is the old HEAD 7a93c4e5; the source-tree label captures + the uncommitted production fixes. The image is built and locally verified. + It has not replaced the older image on the physical hosts. +- Latest compiled client tree used by all three fresh paired browser rounds, + all three Go rounds and Dataset browse: + 9f2a4a339b6b02852e39cffa16b5cae0f6f9d8a5. + The CLI used 65819b6aa8cdacf284941faf932681219d01cd0a; resumed GC used + d862cc4d3238afd6e9ccb39e4996ab415d10c5bc. Later changes affect only the + auxiliary browser helper and documentation, so those prior passes remain valid. +- ci/stability-final/refined-source-verification.json confirms current server + sources match the final image. Later changes are client/test/documentation + changes; the full all-domain run used an earlier copied client bundle. + Its PASS cannot validate the newer GC/Journal assertions; their separate + strengthened runs now pass. resume-server-source-verification.json confirms + current production sources still match this image. +- This handoff update changes the repository source identity again. Before + another E2E run, refresh compiled clients through the normal builder. + Do not bypass manifest validation or blindly rerun all-queue, which is complete. + +## 4. Work already verified + +- [x] **Core production fixes and component gates**: core 1165, + access/monitor 2039, console 426 and full fixture 301 tests passed, with the + existing opt-in/environment ignores retained. UI, full TypeScript, + Rust fmt/full clippy, affected C++ format/tree-lint, full C++/FFI, storage, + ChunkDB and CI task mapping passed. Exact queues/logs are retained. +- [x] **RPC/native lifecycle**: final shared route ownership is retired outside + RPC callbacks, including recursive retirement and process-exit draining. + C ABI send/chain/slab retain native owners before synchronous completion. + Deterministic Rust FFI and four native red/green cases, full gates and + independent review passed. See ci/route-retirement-fixed/ and + first-failures/P12-A3-survivor-restart/crash/FINAL-READONLY-REVIEW.md. + The physical P12 abort has no stack; do not call it the same proven stack. +- [x] **Service observation**: HTTP reads no longer perform authority/SSH/health + network collection. Independent samplers use sample-start freshness and + binding/endpoint/operation/mutation fences; real monitor identity survives + Group 0 unavailability. Actual readiness remains independently checked. + Nine focused Web tests, two real monitor tests, ten replay tests and full + component gates passed. Final source review has no blocking finding. +- [x] **Browser batch failure provenance and scoped cleanup**: undispatched peers + are explicitly not_executed, never PASS or an independent root failure. + Original errors and case-specific cleanup errors remain distinct. Shortcut + cleanup is scoped to recorded owned resources. Full fixture tests passed; + actual all-domain and scope-guard results passed. +- [x] **Auxiliary lifecycle coverage**: original combined three-second predicate + passed on the new image (Playwright 9.7 seconds). Strengthened actual PID, + namespace, old-process exit, generation/operation, sibling identity and + registration uniqueness coverage passed (12.4 seconds). All 19 native + operations cases passed. Monitor operation change is checked; the HTTP API + does not expose an exact accepted operation ID, so that is not claimed. +- [x] **CLI cold observation and actual process identity**: the real first + all-domain failure was KeyError: 'pid' in plain_docker_target_attachment. + A ready cluster need not have every optional cached PID immediately. + The fix checks owned containers/bindings, 21 monitor services and real /proc + liveness while preserving 18 unique full endpoint registrations. + Four regressions and all five strengthened actual CLI cases now pass. + This does not add waits or caller retries. +- [x] **Final image container acceptance**: 396/396 latest canonical PASS and + required four-case concurrent isolation PASS. The progress helper shows eight + isolation passes because two all-domain attempts each ran four; do not count + them as eight distinct isolation cases. All seven physical resources retained + their exact identities. +- [x] **CI failure audit**: all 16 failed jobs from the three user-specified runs + are in the linked stability TODO. Complete job logs, ten original artifact + ZIPs with matching hashes, and case mappings are retained in ci/prior-ci-audit/. + The runs are 41003611925 (8/27 failed), 41128190276 (5/27), and + 41159636766 (3/27 at head 8885d2a). All 16 joint review boxes remain unchecked. + +Main uncommitted source ownership: + +- RPC/C ABI: lib/crowdb-rpc/ffi/src/owned_route* and logging.rs, + lib/crowdb-rpc/src/c_api.cpp, lib/crowdb-tree/ffi/src/tree.rs, and tests. +- Repair/recovery/read path: app/crowdb-chunkdb/src/repair*, + app/crowdb-chunk-kv-server/src/storage.rs, + lib/crowdb-chunk-kv/src/partition* and lib/crowdb-access-iceberg/src/file/reader.rs. +- Observation: app/crowdb-web/src/services/observation* and remote/deployment/state + integration; container/crowdb-monitor/src/node/runtime/{observation,services}* + and corresponding crate tests. +- Test stability: e2e-test/browser_batch.py, fixture*.py, report.py, + oci_shortcuts.py, console_cli_target.py, GC convergence/pressure/sdk_pressure, + console/native_balance.rs, console-ui lifecycle/Journal flows and helpers, + their regressions, and tools/test-metrics/measure.py. +- Documentation: this handoff, plan-ci-stability-review.md and + doc/dev/physical-node-test-plan.md. Inspect git status before staging. + +## 5. Remaining CI stability and publication tasks + +The user has resumed; proceed in the order below. + +- [x] **Validate the GC ready barrier with the new bundle**: run the three exact + convergence, notification matrix and foreground SDK cases in the saved queue. + Four real SDK workers now warm up and park before executable purge markers; + actual deletion releases work after the original faults. Preserve nine files, + three restarts, six operation classes, 64 writes, true completion/deletion + overlap and original 720/3300-second absolute deadlines. Six pure regressions, + compiled clients, clippy and independent review passed. All three strengthened + real cases passed on resume; queue duration 480.220 seconds. Inline convergence + and matrix retain 240/81 SDK completions inside 5/7 deletion intervals, with + 64 writes each and all three matrix restarts before release. Evidence: + `ci/stability-final/resume-gc-proof.json`; see GC-READY-BARRIER.md and + GC-READY-FINAL-REVIEW.md. +- [x] **Validate Journal request ownership and repeat the browser pair**: exact + request/cursor identity plus Response.request() association and body completion + replace pathname-only response matching. All waits share the original three + seconds; owner-loss/restart/stale-cursor assertions remain. Four event + regressions and full TypeScript pass. Run auxiliary plus + journal_identity_and_owner_recovery together, three fresh fixture rounds, + all required to pass. See JOURNAL-REQUEST-STABILITY.md. + Resume round one passed; round two failed the auxiliary Access initial `up` + assertion after Chunk-KV replacement, while Journal passed both rounds. + Exact isolated and paired serial baselines passed. Read-only sampling of the + unchanged concurrent pair confirms real Access 503 during dependency faults, + followed by recovery. The existing three-second replacement predicate now + additionally requires same-node Access recovery before advancing. No assertion, + timeout, or mutation retry was removed or relaxed. Three fresh paired rounds + on this strengthened source all passed in 45.303, 44.938 and 46.144 seconds. + Earlier attempts are preserved under + `ci/stability-final/first-resumed-browser-failure/`; diagnosis is in + ACCESS-DEPENDENCY-CLOSURE.md. +- [x] **Repeat real Go SDK cleanup three times**: normal and injected cleanup + assertions stay unchanged. Old CI cleanup's operation/root cause is unknown; + enhanced Smithy operation/API-code errors are retained. Repeats provide + current coverage, not a claim that an unknown historical algorithm was fixed. + Three fresh rounds passed in 11.143, 10.611 and 10.139 seconds. Each log + confirms both ordinary and injected-failure cleanup completed. +- [x] **Run instrumented Dataset browse and retain headroom risk**: phase timing + now separates seed, default production balance and exact 512-record readback. + In the completed all-domain run browse was 281.284 seconds, import 321.901, + recovery 331.468; the latter two leave only 38.099/28.532 seconds under 360. + Late partition formation is observed; no safe behavior fix is proven. + Keep policy, cooldown, three owners, at least 12 partitions, data and deadlines. + DATASET-HEADROOM-AUDIT.md records causal limits and bounded follow-up evidence. + Instrumented browse passed: seed 2.276 seconds, balance 260.002, exact readback + 0.273; 12 partitions and three owners. Client 278.32 seconds, full queue case + 291.230 seconds. This risk remains open even after a PASS. +- [x] **Close local evidence**: all refinement rows and finalized fixture + timings checked; fresh scope guard and production source verification pass. + Final-current-image evidence confirms 396 latest PASS, mapping all 32 older + failure occurrences and three newer cases. Client binary hashes match the + frozen build manifest; browser/Go/Dataset source digests agree. Stability TODO + updated; all human review boxes remain unchecked. +- [~] **Commit and push CROWDB only**: run final relevant gates/diff check, + inspect staged contents, and make one coherent commit with a single-line + subject, no body/trailers/doc references/requirement numbers. Fetch and verify + the remote before the authorized fast-forward push; do not force-push. + No Sirius-specific pre-push rules apply to this repository. +- [ ] **Inspect new exact-head CI**: PR synchronization should trigger CI. + Match the actual new head SHA/run/attempt, inspect every expanded job + (the three historical runs had 27), preserve every first failure and artifact, + fix real errors and rerun the appropriate new-head job/run. Do not rerun an old + failing SHA as validation of the new code. +- [ ] **Joint review**: retain all 16 job-level entries for the user. Passing + tests cannot close the historical Go root-cause unknown, Dataset headroom, + or unfinished physical validation. + +The refinement queue is complete. Do not replay it merely because this document +changed. If a later code fix needs E2E, refresh clients normally and select the +affected group while retaining valid completed evidence. Reuse the final image +while production sources still match. Every command and original error is durable. + +## 6. Physical checkpoint and remaining work + +Physical work is deferred until the CI/publication stage above is complete. + +- Hosts: A 192.168.31.129 (A1/A2/A3) and B 192.168.31.184 (B1/B2). + Existing SSH uses BatchMode/IdentitiesOnly. Verify host identity/clocks on resume. +- Remote project commands use /home/cj/.pixi/bin/pixi run with explicit manifest + /home/cj/crowdb-runs/pixi.toml. Remote artifact/harness root: + /home/cj/.crowdb-runtime/artifacts/physical-nodes/physical-20261031T194149Z. +- Current owned nodes: a1/a2/a3/b1/b2 labelled with this run. Two older stopped + B containers are additionally retained: seven physical containers total, A3/B4. +- Five voters, two physical racks, Groups 0/1/3, 35 managed services; + each node has hard 1 CPU, 2 GiB, 512 PIDs, one Tokio worker, + 16 GiB media with 1 GiB reserve. Preserve port offsets A=0/1/2 and B=0/1. + Port bases: Web9090, KV10000/10100, DiskDB11100/11200, DiskIO13000, + ChunkDB12100/12200, ChunkKV15100/15200, S316000, Iceberg17000, + Dataset18000, health19000, container SSH2222. Preserve host SSH22. +- P00–P08 passed on the initial image, including complete eight-table Iceberg + SF1 file/checksum/row and bidirectional SQL checks, independent zero-retry S3 + SF1, Dataset full fields/multipart, 512/4096 native records across five safe + owners, acknowledged-deletion audits, and the first complete 30-minute soak. +- Actual mixed-workload overlap was 1818.511 seconds: 1826 native cycles, + 1829 S3 cycles, 183 Iceberg and 183 Dataset cycles. All 35 identities unchanged; + 909 complete observations, max gap 2.617 seconds, no OOM. + Do not repeat/resume that timed workload merely to manufacture a new pass. + Production changes during a new soak would require restarting that new soak. +- P09–P11 passed on the older repair image with full data/history and sibling + identity closure. P12 original repair defect was fixed: expired unpublished + Allocated targets are replaced under the claim, copied/published targets and + normal owner-grace reclamation remain intact. Original takeover then passed + in 40.184 seconds. Old physical image input tree is + 76f794e03757249f37efa96cd0bfcad064be4388. +- A P12 harness witness exceeded its EC 8+4 outage budget when B removed six + fragments; the corrected fresh mirrored outage witness passes, and complete + history returns after B recovery. Keep this harness boundary distinct from + production repair and retain the complete post-recovery historical audit. + +- [ ] **Archive current physical data before image upgrade**: existing private + archives are complete/tested/hashed but predate later P12/P13 data. Make fresh + private archives of every still-existing volume before any new upgrade or + retirement. Archive helper containers must use --log-driver none so binary + tar data cannot fill Docker JSON logs. Keep incomplete old streams invalid. + Transfer/verify the final production image and preserve exact mounts/limits. + Do not blindly replay resume_repaired_image.py or old witness mutation IDs. +- [ ] **Close unexpected P12 A3 identity change**: A3 ChunkKV changed PID451/gen2 + to PID66135/gen3, stderr Resource deadlock avoided, no usable core. + Local Rust-FFI gdb proves callback-final-owner destruction can join its own + RPC worker; the physical stack is unknown. Validate the new lifecycle fix on + physical hosts and close all collateral process identities through full audits. + Preserve first-failures/P12-A3-survivor-restart/ and + faults/P12-full-identity-closure.json. +- [ ] **Resolve P13 Iceberg read timeout**: after A quorum loss, recovery/native + and five complete raw S3 SF1 audits passed. Three Iceberg readers returned + curl 28/GetObject low-speed timeout; two remaining owned read-only readers + were stopped after preserving errors and are interrupted, not PASS. + Dataset/history checks were not dispatched. Exact endpoint/object/range cause + is unknown; unlabeled old stdout cannot assign failures by output order. + B1/B2 had controlled process-lease-expiry restarts with empty stderr, distinct + from the P12 A3 abort. Read first-failures/P13-protocol-read/, + owned-reader-stop.json, READONLY-REVIEW.md and ci/physical-after-witness-scope.log. + Add per-endpoint/object-stage read-only diagnostics before retrying verification; + keep original I/O semantics, limits and credential redaction. + The minority write returned OutcomeUnknown; later value presence is not an + acknowledgment and must not cause automatic mutation replay. +- [ ] **P14 allocation admission fault**: preserve bounded Group1 step-down, + unchanged Group0 lease/term observations, explicit allocation Unavailable, + same-ID safe retry under the original 10-second/five-retry client policy, + all 35 process identities, and complete subsequent data audit. +- [ ] **P15 backlog and cold restart**: stop a real Group3 follower, acknowledge + 1100 single writes and prove gap greater than 1024; restore within the original + independent 180-second checkpoint gates, audit all writes, then cold-restart + all five and audit complete retained history. Do not claim reclaimed-WAL or + other untriggered physical sub-boundaries independently exercised. +- [ ] **P16 final occupancy/live overlap**: validate 17 total KVs including the + fresh batch, allocation, full data/history, capacity and process identity + fences. Follow the actual contracts in harness/recovery_final.py and the + reusable physical plan; do not replace them with a weak readiness-only pass. +- [ ] **P17–P19 close only after all prerequisites pass**: freeze complete + evidence/results/summary, inspect identities across the entire fault and audit + intervals, privately preserve remaining data, then exact owned-only retirement. + Review harness/close_run.py before dispatch; no global cleanup. + +The separate investigate-unexpected-P12-restart marker prevents recovery_final.py +from dispatching P14. It is not the CI pause marker. Do not remove it merely to +make the controller advance; P12/P13 evidence and gates must actually be closed. +Old controllers already submitted mutations: never run all_physical.py or an +earlier resume script blindly. + +## 7. Non-negotiable evidence and data boundaries + +- Run all builds/tests/lints/project executables through pixi run, including + remote commands with the verified manifest. Serialize shared-output builders. +- Print the real failing case/error; use CROWDB_E2E_PRINT_SERVICE_ERRORS=1 while + diagnosing and retain complete logs. Only finalized timing records containing + total and stop count as results; provisional diagnostic status is not failure. +- Keep existing assertions, operation budgets and caller retry policies. + Add no hot-path lock or unsafe exception. Unknown mutations are reconciled + read-only; acknowledgment cannot be inferred from a later successful read. +- Native keys exclude reserved S/I/D prefixes. Keep the original 64-KiB values, + batches of 16, five-second operation deadline and actual safe-owner coverage. +- Observe all 35 service identities and OOM state with at most five seconds + between complete samples. Readiness must postdate the current process start + and be at most three seconds old; retain at least 30% free test media. +- Recovery service/catalog/fixed-checkpoint gates remain independently bounded + at 180 seconds; PID checks include the entire subsequent data-audit interval. +- Preserve the earlier destructive cleanup incident honestly in the final report: + the old unscoped shortcut cleanup removed seven containers, 89 volumes and one + network on A, including six physical A data volumes before archival. Those six + contents cannot be recovered from retained logs. Incident evidence is under + the older physical-20261031T145356Z run's + ci/native-worker-fixed/unscoped-cleanup-incident.json. +- Cleanup has since been scoped and verified, but it does not undo that loss. + All still-existing failed-run volumes were privately archived before previous + retirement. Never use Docker-wide prune, generic --all cleanup, prefix-wide + protocol deletion, or remove unrelated data. +- Current authoritative state is this checkpoint plus current runtime receipts. + Historical 024347 runtime was deleted; its old paths and sessions are not + surviving proof or valid executable inputs. The archived earlier handoff is + historical context only. diff --git a/e2e-test/browser_batch.py b/e2e-test/browser_batch.py index 0fb434663..e972fe00e 100644 --- a/e2e-test/browser_batch.py +++ b/e2e-test/browser_batch.py @@ -4,23 +4,46 @@ import concurrent.futures from contextlib import ExitStack +from fixture_result import CaseNotExecuted from resource_dispatch import resource_batches class BatchFixture: """Retain each client's failure until its own deferred lifecycle teardown.""" - def __init__(self, cluster): + def __init__(self, cluster, case_name, errors): self.cluster = cluster + self.case_name = getattr(cluster, 'case_name', case_name) + self.errors = errors self.failure = None def __enter__(self): return self.cluster.__enter__() def __exit__(self, kind, error, traceback): - if error is None and self.failure is not None: + if self.failure is not None: error = self.failure kind, traceback = type(error), error.__traceback__ - return self.cluster.__exit__(kind, error, traceback) + try: + return self.cluster.__exit__(kind, error, traceback) + except Exception as cleanup_error: + # An unwind error belongs to this fixture, not the next peer. + self.errors.append(cleanup_error) + return False + + +def _enter_fixtures(stack, owners, errors): + clusters = [] + for owner in owners: + try: + clusters.append(stack.enter_context(owner)) + except BaseException as error: + # Keep startup first even when peer cleanup also fails. + if isinstance(error, Exception): + errors.append(error) + for entered in owners[:len(clusters)]: + entered.failure = CaseNotExecuted(owner.case_name) + raise + return clusters def run_browser_batches(cases, slots, prepare, execute, fail_fast=False, @@ -31,8 +54,9 @@ def run_browser_batches(cases, slots, prepare, execute, fail_fast=False, # Keep all fixtures in this bounded batch until every client has ended. try: with ExitStack() as stack: - owners = [BatchFixture(prepare(case)) for case in batch] - clusters = [stack.enter_context(owner) for owner in owners] + owners = [BatchFixture(prepare(case), getattr(case, 'name', str(case)), errors) + for case in batch] + clusters = _enter_fixtures(stack, owners, errors) with concurrent.futures.ThreadPoolExecutor(max_workers=slots) as pool: futures = {pool.submit(execute, case, cluster): owner for case, cluster, owner in zip(batch, clusters, owners)} @@ -43,7 +67,8 @@ def run_browser_batches(cases, slots, prepare, execute, fail_fast=False, futures[future].failure = error errors.append(error) except Exception as error: - errors.append(error) + if not any(error is previous for previous in errors): + errors.append(error) if fail_fast and errors: break return errors diff --git a/e2e-test/clients/console-ui/e2e/fixtures/journalRuntimeRequest.ts b/e2e-test/clients/console-ui/e2e/fixtures/journalRuntimeRequest.ts new file mode 100644 index 000000000..e693638fc --- /dev/null +++ b/e2e-test/clients/console-ui/e2e/fixtures/journalRuntimeRequest.ts @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +import type { Page, Request, Response } from '@playwright/test'; + +export interface JournalRuntimeTarget { + origin: string; id: string; epoch: string; generation: string; page: number; offset: number; +} + +export function matchesJournalRuntimeRequest(request: Pick, target: JournalRuntimeTarget): boolean { + if (request.method() !== 'GET') return false; + const url = new URL(request.url()); + const query = { id: target.id, epoch: target.epoch, generation: target.generation, + page: String(target.page), offset: String(target.offset), stream_offset: '0' }; + return url.origin === target.origin && url.pathname === '/api/chunk-kv/runtime' + && [...url.searchParams.keys()].length === Object.keys(query).length + && Object.entries(query).every(([name, value]) => url.searchParams.get(name) === value); +} + +/** Observe the GET started by this action, never a late response to an earlier selection. */ +export async function journalRuntimeResponse(page: Pick, target: JournalRuntimeTarget, + action: () => Promise, remaining: () => number): Promise { + const budget = remaining(); + if (budget <= 0) throw new Error('Journal observation exceeded its three-second deadline'); + let owned: Request | null = null; + let receive!: (response: Response) => void; + let fail!: (error: Error) => void; + const observed = new Promise((resolve, reject) => { receive = resolve; fail = reject; }); + const onRequest = (request: Request) => { + if (!owned && matchesJournalRuntimeRequest(request, target)) owned = request; + }; + const onResponse = (response: Response) => { + if (response.request() !== owned) return; + void response.finished().then(error => { if (error) fail(error); else receive(response); }, fail); + }; + const onFailure = (request: Request) => { + if (request === owned) fail(new Error(request.failure()?.errorText ?? 'Journal runtime request failed')); + }; + const timeout = setTimeout(() => fail(new Error(`Journal runtime response timed out for ${target.id}, epoch ${target.epoch}, catalog ${target.generation}`)), budget); + page.on('request', onRequest); + page.on('response', onResponse); + page.on('requestfailed', onFailure); + try { + const [response] = await Promise.all([observed, Promise.resolve().then(action)]); + return response; + } finally { + clearTimeout(timeout); + page.off('request', onRequest); + page.off('response', onResponse); + page.off('requestfailed', onFailure); + } +} diff --git a/e2e-test/clients/console-ui/e2e/fixtures/nativeProcess.ts b/e2e-test/clients/console-ui/e2e/fixtures/nativeProcess.ts index bda727381..8b3e0d3ed 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/nativeProcess.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/nativeProcess.ts @@ -7,17 +7,35 @@ import { containerControl } from './containerControl'; const known = new Set(); const observation = (body: object) => containerControl(process.env.CROWDB_NATIVE_OBSERVATION_CONTROL, body); +export interface NativeServiceState { + pid: number | null; + generation: number; + operation_id: string | null; +} + +/** Read typed service identities from this fixture's owned monitor. */ +export async function nativeServices(node: number): Promise> { + const status = await observation({ action: 'observe-native-services', node }); + for (const service of Object.values(status.services) as NativeServiceState[]) { + if (service.pid) known.add(`${node}/${service.pid}`); + } + return status.services; +} + +/** One OS observation after the owning monitor has supplied the PID. */ +export async function observedNativeProcessAlive(node: number, pid: number): Promise { + if (!known.has(`${node}/${pid}`)) throw new Error('Native PID has not been observed in this owned node'); + return (await observation({ action: 'observe-native-process', node, pid })).alive; +} + /** Observe only a PID recorded by this fixture's actual monitor. */ export async function nativeProcessAlive(node: number, pid: number): Promise { const key = `${node}/${pid}`; if (!known.has(key)) { await expect.poll(async () => { - const status = await observation({ action: 'observe-native-services', node }); - for (const service of Object.values(status.services) as { pid?: number }[]) { - if (service.pid) known.add(`${node}/${service.pid}`); - } + await nativeServices(node); return known.has(key); }, { intervals: [100] }).toBe(true); } - return (await observation({ action: 'observe-native-process', node, pid })).alive; + return observedNativeProcessAlive(node, pid); } diff --git a/e2e-test/clients/console-ui/e2e/fixtures/nativeServiceLifecycle.ts b/e2e-test/clients/console-ui/e2e/fixtures/nativeServiceLifecycle.ts new file mode 100644 index 000000000..ade16770e --- /dev/null +++ b/e2e-test/clients/console-ui/e2e/fixtures/nativeServiceLifecycle.ts @@ -0,0 +1,122 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +import type { APIRequestContext, Page } from '@playwright/test'; +import { expect } from './realBackend'; +import { nativeProcessAlive, nativeServices, observedNativeProcessAlive } from './nativeProcess'; +import { step } from './stepTimer'; + +interface NativeServer { + id: string; + node_id: number; + pid?: number; + pid_node_id?: number; + health: string; + service_type: string; + start_pending: boolean; +} + +export async function nativeServerInventory(request: APIRequestContext): Promise { + const response = await request.get('/api/servers'); + expect(response.ok(), await response.text()).toBe(true); + return response.json(); +} + +async function siblingIdentities(target: string) { + const nodes = await Promise.all([0, 1, 2].map(node => nativeServices(node))); + return nodes.flatMap((services, node) => Object.entries(services) + .filter(([id]) => node !== 0 || id !== target) + .map(([id, { pid, generation, operation_id }]) => ({ node, id, pid, generation, operation_id }))) + .sort((left, right) => `${left.node}/${left.id}`.localeCompare(`${right.node}/${right.id}`)); +} + +const registration = ({ pid: _pid, pid_node_id: _namespace, health: _health, start_pending: _pending, ...row }: NativeServer) => row; + +async function stopAuxiliary(page: Page, request: APIRequestContext, original: NativeServer, kind: string, label: string, prefix: string) { + const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); + await step(`native lifecycle: ${original.id} Stop response`, async () => { + await sidebar.getByRole('button', { name: 'N-1', exact: true }).click({ button: 'right' }); + await page.getByRole('menuitem', { name: `${prefix}-1 Running`, exact: true }).hover(); + const stopped = page.waitForResponse(response => + new URL(response.url()).pathname === `/api/services/${original.id}/stop` + && response.request().method() === 'POST'); + await page.getByRole('menuitem', { name: `Stop ${label}`, exact: true }).click(); + const response = await stopped; + expect(response.status(), await response.text()).toBe(200); + expect(await response.json()).toMatchObject({ id: original.id, pid: null, removed: false, data_preserved: true }); + }); + await expect(page.locator('header').getByRole('button', { name: 'Refresh', exact: true }).locator('svg')).not.toHaveClass(/tw-animate-spin/); + await step(`native lifecycle: ${original.id} stopped process and inventory`, async () => { + await expect.poll(() => nativeProcessAlive(0, original.pid), { intervals: [100] }).toBe(false); + await expect.poll(async () => { + const current = (await nativeServerInventory(request)).find(row => row.id === original.id); + return { id: current?.id, pid: current?.pid ?? null, ...(kind === 'access-server' ? { health: current?.health } : {}) }; + }, { intervals: [100] }).toEqual({ id: original.id, pid: null, ...(kind === 'access-server' ? { health: 'down' } : {}) }); + }); +} + +async function startAuxiliary(page: Page, request: APIRequestContext, original: NativeServer, kind: string, label: string, prefix: string, oldOperation: string) { + const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); + let submitted = false; + try { + await sidebar.getByRole('button', { name: 'N-1', exact: true }).click({ button: 'right' }); + await page.getByRole('menuitem', { name: `${prefix}-1 Stopped`, exact: true }).hover(); + // Preserve the existing response budget; the combined process assertion stays 3s. + const started = page.waitForResponse(response => + new URL(response.url()).pathname === `/api/services/${original.id}/restart` + && response.request().method() === 'POST', { timeout: 5_000 }); + const startAt = performance.now(); + submitted = true; + const response = await step(`native lifecycle: ${original.id} Start response`, async () => { + await page.getByRole('menuitem', { name: `Start ${label}`, exact: true }).click(); + return started; + }); + console.log(`[START] ${original.id}: ${(performance.now() - startAt).toFixed(0)}ms`); + expect(response.status(), await response.text()).toBe(200); + const accepted = await response.json(); + expect(accepted).toMatchObject({ id: original.id, removed: false, data_preserved: true }); + expect(Number.isInteger(accepted.pid)).toBe(true); + expect(accepted.pid).toBeGreaterThan(1); + expect(accepted.pid).not.toBe(original.pid); + await step(`native lifecycle: ${original.id} new process, inventory and Access recovery`, () => expect.poll(async () => { + const inventory = await nativeServerInventory(request); + const current = inventory.find(row => row.id === original.id); + const access = inventory.find(row => row.node_id === original.node_id && row.service_type === 'access-server'); + const actual = (await nativeServices(0))[original.id]; + // A new backend PID can precede dependent Access readiness. Close that + // outage inside this same three-second assertion before the next cycle. + return current?.pid === accepted.pid && current.pid_node_id === original.node_id + && actual?.pid === accepted.pid && typeof actual.operation_id === 'string' + && actual.operation_id !== oldOperation && await observedNativeProcessAlive(0, accepted.pid) + && (kind !== 'access-server' || current.health === 'up') && access?.health === 'up'; + }, { intervals: [100] }).toBe(true)); + const current = (await nativeServerInventory(request)).filter(row => row.id === original.id); + expect(current).toHaveLength(1); + expect(registration(current[0])).toEqual(registration(original)); + } finally { + if (!submitted) { + const current = (await nativeServerInventory(request)).find(row => row.id === original.id); + if (!current?.pid) expect((await request.post(`/api/services/${original.id}/restart`, { data: {} })).ok()).toBe(true); + } + } +} + +export async function cycleAuxiliaryService(page: Page, request: APIRequestContext, kind: string, label: string, prefix: string) { + const original = (await nativeServerInventory(request)).find(row => row.node_id === 1 && row.service_type === kind); + expect(original).toBeDefined(); + expect(original.pid_node_id).toBe(original.node_id); + if (kind === 'access-server') expect(original.health).toBe('up'); + expect(original.pid).toBeGreaterThan(1); + const actual = (await nativeServices(0))[original.id]; + expect(actual?.pid).toBe(original.pid); + expect(actual.operation_id).toEqual(expect.any(String)); + expect(await observedNativeProcessAlive(0, original.pid)).toBe(true); + const siblings = await siblingIdentities(original.id); + const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); + await expect(sidebar.getByText(`${prefix}-1`, { exact: true })).toBeVisible(); + if (kind === 'access-server') await expect(sidebar.getByRole('button', { name: 'AS-access-1', exact: true })).toHaveCount(0); + await stopAuxiliary(page, request, original, kind, label, prefix); + await step(`native lifecycle: ${original.id} stopped siblings`, async () => expect(await siblingIdentities(original.id)).toEqual(siblings)); + await startAuxiliary(page, request, original, kind, label, prefix, actual.operation_id); + await step(`native lifecycle: ${original.id} restarted siblings`, async () => expect(await siblingIdentities(original.id)).toEqual(siblings)); +} diff --git a/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts b/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts index 79b3bfefe..f13a31845 100644 --- a/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts @@ -6,67 +6,17 @@ import { observeDeployingSteps } from '../fixtures/servicePlanObservation'; import { step } from '../fixtures/stepTimer'; import { observeDefaultDeployments } from '../fixtures/defaultDeployments'; import { nativeProcessAlive } from '../fixtures/nativeProcess'; +import { cycleAuxiliaryService, nativeServerInventory } from '../fixtures/nativeServiceLifecycle'; -// Baseline: 20.7s (2026-10-29), including external stop/start observation without Refresh. +// Baseline: 9.7s (2026-11-01), original combined 3s assertions on the independent-observation image. +// Previous: 20.7s (2026-10-29), including external stop/start observation without Refresh. test('native diagnostics: auxiliary menus stop and restart the actual typed process', async ({ page, request }) => { const alive = (pid: number) => nativeProcessAlive(0, pid); - const list = async () => { - const response = await request.get('/api/servers'); - expect(response.ok(), await response.text()).toBe(true); - return response.json(); - }; + const list = () => nativeServerInventory(request); await page.goto('/?domain=Cluster'); - const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); await expect(page.locator('header').getByTitle('Cluster health: Healthy', { exact: true })).toBeVisible(); for (const [kind, label, prefix] of [['chunkdb', 'crowdb-chunk-db', 'CDB'], ['diskio', 'crowdb-disk-io', 'DIO'], ['chunk-kv', 'crowdb-chunk-kv', 'CKV'], ['access-server', 'crowdb-access-server', 'AS']]) { - const original = (await list()).find((row: { node_id: number; service_type: string }) => row.node_id === 1 && row.service_type === kind); - expect(original.pid_node_id).toBe(original.node_id); - if (kind === 'access-server') expect(original.health).toBe('up'); - expect(original.pid).toBeGreaterThan(0); expect(await alive(original.pid)).toBe(true); - const node = sidebar.getByRole('button', { name: 'N-1', exact: true }); - const item = sidebar.getByText(`${prefix}-1`, { exact: true }); - await expect(item).toBeVisible(); - if (kind === 'access-server') await expect(sidebar.getByRole('button', { name: 'AS-access-1', exact: true })).toHaveCount(0); - await node.click({ button: 'right' }); - await page.getByRole('menuitem', { name: `${prefix}-1 Running`, exact: true }).hover(); - const stopped = page.waitForResponse(response => - new URL(response.url()).pathname === `/api/services/${original.id}/stop` - && response.request().method() === 'POST'); - await page.getByRole('menuitem', { name: `Stop ${label}`, exact: true }).click(); - const stopResponse = await stopped; - expect(stopResponse.status(), await stopResponse.text()).toBe(200); - await expect(page.locator('header').getByRole('button', { name: 'Refresh', exact: true }).locator('svg')).not.toHaveClass(/tw-animate-spin/); - await expect.poll(() => alive(original.pid), { intervals: [100] }).toBe(false); - await expect.poll(async () => { - const stopped = (await list()).find((row: { id: string }) => row.id === original.id); - return { id: stopped?.id, pid: stopped?.pid ?? null, ...(kind === 'access-server' ? { health: stopped?.health } : {}) }; - }, { intervals: [100] }).toEqual({ id: original.id, pid: null, ...(kind === 'access-server' ? { health: 'down' } : {}) }); - let startSubmitted = false; - try { - await node.click({ button: 'right' }); - await page.getByRole('menuitem', { name: `${prefix}-1 Stopped`, exact: true }).hover(); - // Native recovery previously reached its listener after 3.15s; keep a - // modest response budget while process and DOM assertions remain 3s. - const started = page.waitForResponse(response => - new URL(response.url()).pathname === `/api/services/${original.id}/restart` - && response.request().method() === 'POST', { timeout: 5_000 }); - const startAt = performance.now(); - startSubmitted = true; - const response = await step(`native lifecycle: ${original.id} Start response`, async () => { - await page.getByRole('menuitem', { name: `Start ${label}`, exact: true }).click(); - return started; - }); - console.log(`[START] ${original.id}: ${(performance.now() - startAt).toFixed(0)}ms`); - expect(response.status(), await response.text()).toBe(200); - await step(`native lifecycle: ${original.id} new process`, () => expect.poll(async () => { - const current = (await list()).find((row: { id: string }) => row.id === original.id); - return current?.pid && current.pid !== original.pid && await alive(current.pid) && (kind !== 'access-server' || current.health === 'up'); - }, { intervals: [100] }).toBe(true)); - expect((await list()).filter((row: { id: string }) => row.id === original.id)).toHaveLength(1); - } finally { - const current = (await list()).find((row: { id: string }) => row.id === original.id); - if (!startSubmitted && !current?.pid) expect((await request.post(`/api/services/${original.id}/restart`, { data: {} })).ok()).toBe(true); - } + await cycleAuxiliaryService(page, request, kind, label, prefix); } await step('native lifecycle: observe external service changes without browser refresh', async () => { const access = (await list()).find((row: { node_id: number; service_type: string }) => row.node_id === 1 && row.service_type === 'access-server'); diff --git a/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts b/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts index bbef568a2..58f7731ad 100644 --- a/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts @@ -3,8 +3,9 @@ // Baseline: journal 4.6s (2026-10-31); partition/overlay 1.4s, unavailable 0.208s, placement 0.321s (2026-10-03). import { test, expect } from '../fixtures/realBackend'; import { step } from '../fixtures/stepTimer'; +import { journalRuntimeResponse, type JournalRuntimeTarget } from '../fixtures/journalRuntimeRequest'; import type { APIRequestContext, APIResponse, Page, Response } from '@playwright/test'; -import type { CatalogPage, Partition } from '../../src/chunk-kv/catalog'; +import { identity, type CatalogPage, type Partition } from '../../src/chunk-kv/catalog'; test('native diagnostics: journal identity survives Chunk navigation and owner interruption', async ({ page, request }) => { const bucket = process.env.CROWDB_NATIVE_JOURNAL_BUCKET; @@ -180,8 +181,31 @@ test('native diagnostics: Chunk-KV graph survives tab changes, refresh and resiz }); }); +async function refreshSelectedJournal(page: Page, target: JournalRuntimeTarget, remaining: () => number) { + const properties = page.getByRole('region', { name: 'Split properties', exact: true }); + const property = (name: string) => properties.locator('dt').filter({ hasText: new RegExp(`^${name}$`) }).locator('..').locator('dd').textContent({ timeout: remaining() }); + const sameSelection = await properties.count() === 1 && (await property('Split ID')) === target.id + && (await property('Owner epoch')) === target.epoch && (await property('Generation')) === target.generation; + const select = () => page.getByLabel('Partition range map').getByRole('button', { name: `Partition ${target.id}`, exact: true }).click({ timeout: remaining() }); + if (sameSelection) await select(); + else await step('journal automatic selection response', () => journalRuntimeResponse(page, target, select, remaining)); + const refreshRuntime = page.getByRole('region', { name: 'Partition runtime', exact: true }).getByRole('button', { name: 'Refresh runtime', exact: true }); + await expect(refreshRuntime).toBeEnabled({ timeout: remaining() }); + // Selecting the same partition leaves its identity unchanged. Explicitly + // refresh runtime so owner restart is observed even without a new catalog. + const observed = await step('journal runtime DOM refresh', () => journalRuntimeResponse(page, target, + () => refreshRuntime.click({ timeout: remaining() }), remaining)); + return { observed, refreshRuntime }; +} + async function selectActiveJournal(page: Page, request: APIRequestContext, initial: CatalogPage, partitionId?: string) { let catalog = initial; + const deadline = performance.now() + 3_000; + const remaining = () => { + const budget = deadline - performance.now(); + if (budget <= 0) throw new Error('Journal observation exceeded its three-second deadline'); + return budget; + }; let selected: { partition: Partition; observed: Response } | null = null; const refresh = async (action: () => Promise) => { const observation = page.waitForResponse(response => new URL(response.url()).pathname === '/api/chunk-kv/catalog'); @@ -215,16 +239,19 @@ async function selectActiveJournal(page: Page, request: APIRequestContext, initi if (await changed(response)) return false; expect(response.ok(), await response.text()).toBe(true); if (!(await response.json()).journal.chunk_record) continue; - await page.getByLabel('Partition range map').getByRole('button', { name: `Partition ${entry.id}`, exact: true }).click(); - // Selecting the same partition leaves its identity unchanged. Explicitly - // refresh runtime so owner restart is observed even without a new catalog. - const observation = page.waitForResponse(response => new URL(response.url()).pathname === '/api/chunk-kv/runtime'); - const observed = await step('journal runtime DOM refresh', async () => { - await page.getByRole('region', { name: 'Partition runtime', exact: true }).getByRole('button', { name: 'Refresh runtime', exact: true }).click(); - return observation; - }); + const target = { origin: new URL(page.url()).origin, id: entry.id, epoch: entry.epoch, + generation: catalog.generation, page: catalog.page, offset: catalog.offset }; + const { observed, refreshRuntime } = await refreshSelectedJournal(page, target, remaining); if (await changed(observed)) return false; expect(observed.ok(), await observed.text()).toBe(true); + const runtime = await observed.json(); + expect(runtime.partition_id).toBe(entry.id); + expect(runtime.owner_epoch).toBe(entry.epoch); + expect(runtime.catalog_generation).toBe(catalog.generation); + expect(runtime.stream_id).toBe(identity(entry.artifact.stream_name)); + expect(runtime.journal?.offset).toBe(0); + expect(runtime.journal?.chunk_record).toBeTruthy(); + await expect(refreshRuntime).toBeEnabled({ timeout: remaining() }); selected = { partition: entry, observed }; return true; } diff --git a/e2e-test/console_cli_target.py b/e2e-test/console_cli_target.py index 9f6e95ede..851854369 100644 --- a/e2e-test/console_cli_target.py +++ b/e2e-test/console_cli_target.py @@ -67,10 +67,54 @@ def access_listeners(self, index): def inventory(self): containers = [json.loads(self.command("inspect", name))[0] for name in self.servers] - assert all(item["Config"]["Labels"].get(LABEL) == self.run_id for item in containers) + assert all(item["Config"]["Labels"].get(LABEL) == self.run_id + and item["Image"] == self.image and item["State"]["Running"] + for item in containers), "CLI inventory container ownership or image changed" + processes = {} + nodes = set() + for container in containers: + identity = container["Id"] + binding = json.loads(self.command("exec", identity, "cat", "/opt/crowdb/data/node-binding.json")) + node = binding["node_id"] + assert type(node) is int and node > 0 and node not in nodes, binding + assert binding["bootstrap"]["cluster_id"] == self.cluster_id, binding + nodes.add(node) + status = json.loads(self.command("exec", identity, "cat", "/opt/crowdb/run/status/monitor.json")) + for service, current in status["services"].items(): + assert type(current.get("pid")) is int and current["pid"] > 1, (node, service, "invalid monitor PID", current) + processes[node, service] = (current["pid"], current["generation"], current["operation_id"]) + self.command("exec", identity, "sh", "-c", """ + for pid do + state=$(awk '{print $3}' "/proc/$pid/stat") || exit 1 + case "$state" in ''|Z|X) + echo "owned service PID $pid is not live ($state)" >&2 + exit 1;; + esac + done + """, "owned-service-pids", *(str(current["pid"]) for current in status["services"].values())) + # Aggregate PID fields are optional fresh samples. Lifecycle invariance + # uses each owning monitor, including services absent from that cache. services = request(self.endpoint(0), "/api/servers") + assert len({item["id"] for item in services}) == len(services), services + kinds = {"paxos-kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access-server"} + for node in nodes: + registered = [item for item in services if item["node_id"] == node] + assert len(registered) == len(kinds) and {item["service_type"] for item in registered} == kinds, registered + required = {"web"} | {"kv" if item["service_type"] == "paxos-kv" else item["id"] for item in registered} + observed = {service for owner, service in processes if owner == node} + assert required <= observed, ("owned monitor service is absent", node, required, observed) + assert {item["node_id"] for item in services} == nodes, services + for item in services: + service = "kv" if item["service_type"] == "paxos-kv" else item["id"] + actual = processes[item["node_id"], service] + if item.get("pid") is not None: + assert item.get("pid_node_id") == item["node_id"], ("cached PID namespace changed", item) + assert item["pid"] == actual[0], ("cached PID differs from owning monitor", item, actual) + volatile = {"pid", "pid_node_id", "health", "start_pending"} + registrations = [{key: value for key, value in item.items() if key not in volatile} for item in services] return {"containers": [(item["Id"], item["Image"]) for item in containers], - "processes": sorted((item["id"], item["pid"]) for item in services)} + "processes": sorted((node, service, *identity) for (node, service), identity in processes.items()), + "registrations": sorted(registrations, key=lambda item: item["id"])} def replay(self, checkpoint): if checkpoint != "attachment": diff --git a/e2e-test/fixture.py b/e2e-test/fixture.py index a6107a440..11ee2aa50 100644 --- a/e2e-test/fixture.py +++ b/e2e-test/fixture.py @@ -16,6 +16,7 @@ from client import run_client from kv_node import arguments as kv_arguments from control import Control +from fixture_result import CaseNotExecuted from kv_topology import ready_nodes, wire_nodes HTTP = urllib.request.build_opener(urllib.request.ProxyHandler({})) @@ -250,26 +251,49 @@ def __enter__(self): def __exit__(self, _kind, error, _traceback): from failure_logs import print_failure + not_executed = isinstance(error, CaseNotExecuted) + if not_executed: + self.timings["not_executed_reason"] = redact(str(error)) if self.closed: - self.timings["status"] = "pass" if error is None else "failed" + self.timings["status"] = ("not_executed" if not_executed + else "pass" if error is None else "failed") self.write_timings() return + if not_executed: + print(redact(f"[NOT EXECUTED CASE] {getattr(self, 'case_name', self.run_id)}: " + f"{error}\nArtifacts: {self.artifacts}"), flush=True) collection_error = None try: self.diagnostics() except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as diagnostic_error: collection_error = diagnostic_error - if error is not None: + if error is not None and not not_executed: error.add_note(str(diagnostic_error)) - if error is not None or collection_error is not None: + reported_error = collection_error if not_executed else error or collection_error + if reported_error is not None: print_failure(self.artifacts, self.data_paths, getattr(self, "case_name", self.run_id), - error if error is not None else collection_error, redact) + reported_error, redact) try: - self.timings["status"] = "pass" if error is None and collection_error is None else "failed" + self.timings["status"] = ("failed" if collection_error is not None + else "not_executed" if not_executed + else "pass" if error is None else "failed") self.close() except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as cleanup_error: + if not_executed: + self.timings["status"] = "failed" + self.write_timings() + print_failure(self.artifacts, self.data_paths, getattr(self, "case_name", self.run_id), + cleanup_error, redact) + if collection_error is not None: + collection_error.add_note(str(cleanup_error)) + raise collection_error from cleanup_error + raise if error is None: + print_failure(self.artifacts, self.data_paths, getattr(self, "case_name", self.run_id), + cleanup_error, redact) raise error.add_note(str(cleanup_error)) - if error is None and collection_error is not None: + print(redact(f"[CLEANUP ERROR] {getattr(self, 'case_name', self.run_id)}: " + f"{cleanup_error}\nArtifacts: {self.artifacts}"), flush=True) + if (error is None or not_executed) and collection_error is not None: raise collection_error diff --git a/e2e-test/fixture_result.py b/e2e-test/fixture_result.py new file mode 100644 index 000000000..55c51686a --- /dev/null +++ b/e2e-test/fixture_result.py @@ -0,0 +1,10 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Distinguish an undispatched client from a failed fixture operation.""" + + +class CaseNotExecuted(RuntimeError): + """An entered fixture whose client was blocked by another startup.""" + + def __init__(self, blocked_by): + super().__init__(f'browser client not executed: batch startup failed in {blocked_by}') diff --git a/e2e-test/oci_shortcuts.py b/e2e-test/oci_shortcuts.py index 21de03cfe..b495f4aa3 100644 --- a/e2e-test/oci_shortcuts.py +++ b/e2e-test/oci_shortcuts.py @@ -103,7 +103,11 @@ def credentials(offset): retained_credentials=True, retained_management_write=True, unrelated_container_ids=unrelated, actions=actions) finally: - invoke('oci-clean', '--all', '--delete-data' if success else '--keep-data') + for name in (one, other, three): + receipt = root / 'persistent/oci-clusters' / (name + '.json') + if receipt.exists(): + invoke('oci-clean', '--cluster', name, + '--delete-data' if success else '--keep-data') assert all(not container_ids(name) for name in (one,other,three,missing)) (cluster.artifacts/'shortcuts-proof.json').write_text(json.dumps(proof,indent=2)) print('PASS real Pixi one/three-node shortcuts, retained data and independent scoped cleanup without CLI',flush=True) diff --git a/e2e-test/report.py b/e2e-test/report.py index da132ec0f..5ca273f81 100644 --- a/e2e-test/report.py +++ b/e2e-test/report.py @@ -17,8 +17,10 @@ def write_report(root, timings): phase.get("exit_code") == phase.get("expected_exit_code", 0) for phase in phases) actions = [json.loads(action.read_text()) for action in sorted( (directory / "control").glob("*.response.json"))] + status = ("not_executed" if fixture.get("status") == "not_executed" + else "pass" if completed else "failed") cases.append({**case, "actions": actions, "artifacts": str(directory), "phases": phases, - "fixture": fixture, "status": "pass" if completed else "failed"}) + "fixture": fixture, "status": status}) cases.sort(key=lambda case: case["fixture"].get("total", 0), reverse=True) slowest_clients = sorted( ({"case": case["case"], "scope": case["scope"], **phase} diff --git a/e2e-test/tests/access_iceberg/gc/convergence.rs b/e2e-test/tests/access_iceberg/gc/convergence.rs index 99f3d3d03..273ad0e72 100644 --- a/e2e-test/tests/access_iceberg/gc/convergence.rs +++ b/e2e-test/tests/access_iceberg/gc/convergence.rs @@ -36,26 +36,36 @@ struct TestGcConvergence { async fn expired_inline_backlog_completes_with_foreground_progress() { let stack = TestNativeStorage::new(); let (store, context, table) = seed_table(&stack).await; - let backlog = seed_expired_backlog(store.as_ref(), context, table, 128).await; + let started = Instant::now(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(3300); + let pressure = TestGcPressure::prepare_before(deadline).await.unwrap(); + let backlog = tokio::time::timeout_at( + deadline, + seed_expired_backlog(store.as_ref(), context, table, 128), + ) + .await + .expect("inline GC backlog seed exceeded the existing case deadline"); assert_eq!(backlog.files.len(), 128); assert!(backlog.files.len() > usize::from(GcLimits::default().page_items)); assert!(backlog.dropped_ms + GcTask::PURGE_DELAY_MS < now_ms()); - let server = TestFrontend::start(12).await; let mut check = TestGcConvergence { store, context, table, backlog, - started: Instant::now(), + started, task_gets: 0, verification_gets: 0, observations: Vec::new(), last: None, - pressure: None, + pressure: Some(pressure), }; check.report("seeded"); - let result = - tokio::time::timeout(Duration::from_secs(3300), check.run(&server.origin, &stack.seeds)).await; + let result = tokio::time::timeout_at(deadline, async { + TestFrontend::start(12).await; + check.run(&stack.seeds).await + }) + .await; let result = result.unwrap_or_else(|_| Err("inline GC convergence exceeded 3300 seconds".into())); if let Err(error) = result { check.report("failed"); @@ -68,15 +78,15 @@ async fn expired_inline_backlog_completes_with_foreground_progress() { if let Some(pressure) = &mut check.pressure { pressure.stop().await; } - server.stop().await; + crate::common::control::node("stop", 12).await; panic!("inline GC convergence failed: {error}"); } check.report("complete"); - server.stop().await; + crate::common::control::node("stop", 12).await; } impl TestGcConvergence { - async fn run(&mut self, origin: &str, seeds: &[String]) -> Result<(), String> { + async fn run(&mut self, seeds: &[String]) -> Result<(), String> { let identity = OperationId::from_bytes(self.table.as_bytes()).map_err(|error| error.to_string())?; let repository = GcRepository::new(self.store.clone()); loop { @@ -87,7 +97,11 @@ impl TestGcConvergence { .map_err(|error| error.to_string())? { self.observe(&task)?; - if let Some(pressure) = &mut self.pressure { + let pressure = self + .pressure + .as_mut() + .ok_or("prepared foreground pressure is missing")?; + if pressure.released() { pressure .observe(task.deleted, task.phase == GcPhase::Complete) .await?; @@ -102,7 +116,7 @@ impl TestGcConvergence { "revision":task.revision,"phase":format!("{:?}",task.phase), }) ); - self.pressure = Some(TestGcPressure::start(origin, seeds.to_vec())?); + pressure.release(seeds.to_vec())?; } if task.phase == GcPhase::Complete && self.pressure.as_ref().is_some_and(TestGcPressure::complete) diff --git a/e2e-test/tests/access_iceberg/gc/convergence/pressure.rs b/e2e-test/tests/access_iceberg/gc/convergence/pressure.rs index d66b5069f..543f4afb3 100644 --- a/e2e-test/tests/access_iceberg/gc/convergence/pressure.rs +++ b/e2e-test/tests/access_iceberg/gc/convergence/pressure.rs @@ -1,10 +1,13 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::super::foreground::{write_s3_with_delay, SDK_PRESSURE_SCRIPT}; -use crowdb_access_iceberg::key::OperationId; +mod sdk; + +use super::super::foreground::write_s3_with_delay; +use crate::{access_iceberg::frontend::TestFrontend, common::control}; +use futures::FutureExt; +use sdk::TestSdkWorker; use std::{ - path::PathBuf, sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -13,9 +16,7 @@ use std::{ }; pub(in crate::access_iceberg::gc) struct TestGcPressure { - sdk: std::process::Child, - stdout: PathBuf, - control: PathBuf, + sdk: TestSdkWorker, writes: Option>, progress: Arc, previous_active: Option<(u64, u64)>, @@ -24,51 +25,75 @@ pub(in crate::access_iceberg::gc) struct TestGcPressure { } impl TestGcPressure { - pub(in crate::access_iceberg::gc) fn start(origin: &str, seeds: Vec) -> Result { - let stdout = PathBuf::from(std::env::var("CROWDB_E2E_ARTIFACTS").map_err(|error| error.to_string())?) - .join(format!("gc-inline-sdk-{}.log", OperationId::random())); - let output = std::fs::File::create(&stdout).map_err(|error| error.to_string())?; - let control = stdout.with_extension("control"); - std::fs::create_dir(&control).map_err(|error| error.to_string())?; - std::fs::write(control.join("run"), b"").map_err(|error| error.to_string())?; - let sdk = std::process::Command::new("pixi") - .args(["run", "-e", "iceberg-e2e", "python", "-c", SDK_PRESSURE_SCRIPT]) - .args([origin, "12", "0.5", "gc-convergence"]) - .arg(&control) - .stdout(output) - .spawn() - .map_err(|error| error.to_string())?; - let progress = Arc::new(AtomicUsize::new(0)); - let writes = tokio::spawn(write_s3_with_delay( - seeds, - progress.clone(), - Duration::from_millis(250), - )); + pub(in crate::access_iceberg::gc) async fn prepare_before( + deadline: tokio::time::Instant, + ) -> Result { + // Call before publishing an executable purge marker. This ordinary + // frontend warms the real SDK while no seeded deletion can race it. + let result = tokio::time::timeout_at(deadline, async { + let server = TestFrontend::start(12).await; + let mut sdk = TestSdkWorker::prepare(&server.origin)?; + sdk.wait_ready().await?; + // Restart after publishing the marker so its first scan sees the + // backlog, without waiting for the production idle scan interval. + server.stop().await; + Ok::<_, String>(sdk) + }) + .await; + let sdk = match result + .unwrap_or_else(|_| Err("SDK warmup exceeded the existing GC case deadline".into())) + { + Ok(sdk) => sdk, + Err(error) => { + let cleanup = std::panic::AssertUnwindSafe(control::node("stop", 12)) + .catch_unwind() + .await; + if cleanup.is_err() { + eprintln!("SDK warmup frontend cleanup also failed; retaining the original warmup error"); + } + return Err(error); + } + }; Ok(Self { sdk, - stdout, - control, - writes: Some(writes), - progress, + writes: None, + progress: Arc::new(AtomicUsize::new(0)), previous_active: None, deletion_intervals: Vec::new(), sdk_complete: false, }) } + pub(in crate::access_iceberg::gc) fn released(&self) -> bool { + self.sdk.released() + } + + pub(in crate::access_iceberg::gc) fn release(&mut self, seeds: Vec) -> Result<(), String> { + self.sdk.release()?; + self.writes = Some(tokio::spawn(write_s3_with_delay( + seeds, + self.progress.clone(), + Duration::from_millis(250), + ))); + Ok(()) + } + pub(in crate::access_iceberg::gc) async fn observe( &mut self, deleted: u64, gc_complete: bool, ) -> Result<(), String> { - if let Some(status) = self.sdk.try_wait().map_err(|error| error.to_string())? { + if !self.released() { + return Err("foreground observation requires the deletion-boundary release".into()); + } + if let Some(status) = self.sdk.try_wait()? { if !status.success() { return Err(format!("official SDK foreground failed: {status}")); } self.sdk_complete = true; } if gc_complete && self.writes.is_none() { - std::fs::write(self.control.join("stop"), b"").map_err(|error| error.to_string())?; + self.sdk.stop()?; } if self .writes @@ -99,31 +124,31 @@ impl TestGcPressure { } pub(in crate::access_iceberg::gc) fn complete(&self) -> bool { - self.sdk_complete && self.writes.is_none() + self.released() && self.sdk_complete && self.writes.is_none() } pub(in crate::access_iceberg::gc) async fn stop(&mut self) { - let _ = std::fs::write(self.control.join("stop"), b""); + if let Err(error) = self.sdk.stop() { + eprintln!("SDK cleanup also failed: {error}"); + } if let Some(writes) = self.writes.take() { writes.abort(); let _ = writes.await; } - if !self.sdk_complete { - let _ = self.sdk.kill(); - let _ = self.sdk.wait(); + if let Err(error) = self.sdk.terminate() { + eprintln!("SDK cleanup also failed: {error}"); } - match std::fs::read_to_string(&self.stdout) { - Ok(output) => print!("{output}"), - Err(error) => eprintln!("SDK stdout artifact {}: {error}", self.stdout.display()), + if let Err(error) = self.sdk.output() { + eprintln!("SDK output: {error}"); } } - pub(in crate::access_iceberg::gc) fn finish(self) -> Result<(), String> { + pub(in crate::access_iceberg::gc) fn finish(mut self) -> Result<(), String> { if !self.complete() { return Err("foreground workloads did not finish".into()); } - let stdout = std::fs::read_to_string(&self.stdout).map_err(|error| error.to_string())?; - print!("{stdout}"); + self.sdk.terminate()?; + let stdout = self.sdk.output()?; if self.deletion_intervals.is_empty() { return Err("no actual deletion advanced while both foreground workloads were active".into()); } @@ -176,10 +201,18 @@ impl TestGcPressure { "deletion_intervals_unix_ns":self.deletion_intervals,"small_writes":64, "sdk_workers":4,"minimum_tables_per_worker":12,"inter_table_sleep_ms":500, "small_write_interval_ms":250,"sleeps_in_latency_samples":false, - "sdk_stdout_artifact":self.stdout, + "sdk_stdout_artifact":self.sdk.stdout(), "scope":"continuous foreground through cleanup and small-writer completion; latency is not a fixed-load baseline", }) ); Ok(()) } } + +impl Drop for TestGcPressure { + fn drop(&mut self) { + if let Some(writes) = self.writes.take() { + writes.abort(); + } + } +} diff --git a/e2e-test/tests/access_iceberg/gc/convergence/pressure/sdk.rs b/e2e-test/tests/access_iceberg/gc/convergence/pressure/sdk.rs new file mode 100644 index 000000000..85243af52 --- /dev/null +++ b/e2e-test/tests/access_iceberg/gc/convergence/pressure/sdk.rs @@ -0,0 +1,234 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Own the real SDK process across warmup, endpoint restarts and release. + +use super::super::super::foreground::SDK_PRESSURE_SCRIPT; +use crowdb_access_iceberg::key::OperationId; +use std::os::unix::process::CommandExt; +use std::{ + path::PathBuf, + process::{Child, ExitStatus}, + time::{Duration, Instant}, +}; + +pub(super) struct TestSdkWorker { + child: Child, + stdout: PathBuf, + stderr: PathBuf, + control: PathBuf, + released: bool, + reported: bool, + cleanup: Option>, +} + +impl TestSdkWorker { + pub(super) fn prepare(origin: &str) -> Result { + let stdout = PathBuf::from(std::env::var("CROWDB_E2E_ARTIFACTS").map_err(|error| error.to_string())?) + .join(format!("gc-inline-sdk-{}.log", OperationId::random())); + let stderr = stdout.with_extension("stderr.log"); + let control = stdout.with_extension("control"); + std::fs::create_dir(&control).map_err(|error| error.to_string())?; + let output = std::fs::File::create(&stdout).map_err(|error| error.to_string())?; + let errors = std::fs::File::create(&stderr).map_err(|error| error.to_string())?; + let mut command = std::process::Command::new("pixi"); + command + .args(["run", "-e", "iceberg-e2e", "python", "-c", SDK_PRESSURE_SCRIPT]) + .args([origin, "12", "0.5", "gc-convergence"]) + .arg(&control) + .stdout(output) + .stderr(errors) + .process_group(0); + let child = command.spawn().map_err(|error| error.to_string())?; + Ok(Self { + child, + stdout, + stderr, + control, + released: false, + reported: false, + cleanup: None, + }) + } + + pub(super) async fn wait_ready(&mut self) -> Result<(), String> { + loop { + if self.ready()? { + println!( + "gc_foreground_ready {}", + serde_json::json!({"workers":4,"run_exists":false,"stdout":self.stdout,"stderr":self.stderr}) + ); + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + + fn ready(&mut self) -> Result { + if let Some(status) = self.try_wait()? { + return Err(format!("official SDK exited before release: {status}")); + } + if self + .control + .join("run") + .try_exists() + .map_err(|error| error.to_string())? + { + return Err("official SDK was released before warmup completed".into()); + } + for worker in 0..4 { + if !self + .control + .join(format!("ready-{worker}")) + .try_exists() + .map_err(|error| error.to_string())? + { + return Ok(false); + } + } + Ok(true) + } + + fn check_failure(&self) -> Result<(), String> { + for worker in 0..4 { + let failed = self.control.join(format!("failed-{worker}")); + if failed.try_exists().map_err(|error| error.to_string())? { + return Err(format!( + "SDK worker {worker} failed: {}", + std::fs::read_to_string(failed).map_err(|error| error.to_string())? + )); + } + } + Ok(()) + } + + pub(super) fn release(&mut self) -> Result<(), String> { + if self.released || !self.ready()? { + return Err("official SDK release requires four parked ready workers".into()); + } + std::fs::write(self.control.join("run"), b"").map_err(|error| error.to_string())?; + self.released = true; + Ok(()) + } + + pub(super) fn released(&self) -> bool { + self.released + } + + pub(super) fn try_wait(&mut self) -> Result, String> { + self.check_failure()?; + self.child.try_wait().map_err(|error| error.to_string()) + } + + pub(super) fn stop(&self) -> Result<(), String> { + std::fs::write(self.control.join("stop"), b"") + .map_err(|error| format!("SDK stop signal {}: {error}", self.control.display())) + } + + pub(super) fn stdout(&self) -> &std::path::Path { + &self.stdout + } + + pub(super) fn output(&mut self) -> Result { + let stdout = std::fs::read_to_string(&self.stdout).map_err(|error| error.to_string())?; + if !self.reported { + print!("{stdout}"); + match std::fs::read_to_string(&self.stderr) { + Ok(stderr) => eprint!("{stderr}"), + Err(error) => eprintln!("SDK stderr artifact {}: {error}", self.stderr.display()), + } + self.reported = true; + } + Ok(stdout) + } + + pub(super) fn terminate(&mut self) -> Result<(), String> { + if let Some(result) = &self.cleanup { + return result.clone(); + } + let mut errors = self.stop().err().into_iter().collect::>(); + let started = Instant::now(); + let deadline = started + Duration::from_secs(2); + match self.wait_until(started + Duration::from_millis(1500)) { + Ok(true) => {} + result => { + if let Err(error) = result { + errors.push(error); + } + // A reaped launcher does not imply that its Python descendants + // exited. Kill only the new group created by this spawn. + if let Err(error) = self.signal_group("-KILL") { + errors.push(error); + } + match self.wait_until(deadline) { + Ok(true) => {} + Ok(false) => errors.push(format!( + "SDK owned group {} did not exit and reap within cleanup's two seconds", + self.child.id() + )), + Err(error) => errors.push(error), + } + } + } + let result = if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("\n")) + }; + self.cleanup = Some(result.clone()); + result + } + + fn wait_until(&mut self, deadline: Instant) -> Result { + loop { + let reaped = self + .child + .try_wait() + .map_err(|error| format!("SDK child status: {error}"))? + .is_some(); + if !self.signal_group("-0")? && reaped { + // try_wait has cached the exit status, so wait cannot block. + self.child + .wait() + .map_err(|error| format!("SDK child wait: {error}"))?; + return Ok(true); + } + if Instant::now() >= deadline { + return Ok(false); + } + std::thread::sleep(Duration::from_millis(10)); + } + } + + fn signal_group(&self, signal: &str) -> Result { + let output = std::process::Command::new("/bin/kill") + .env("LC_ALL", "C") + .args([signal, "--", &format!("-{}", self.child.id())]) + .output() + .map_err(|error| format!("SDK owned group {} {signal}: {error}", self.child.id()))?; + if output.status.success() { + Ok(true) + } else if String::from_utf8_lossy(&output.stderr).contains("No such process") { + Ok(false) + } else { + Err(format!( + "SDK owned group {} {signal}: {}: {}{}", + self.child.id(), + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )) + } + } +} + +impl Drop for TestSdkWorker { + fn drop(&mut self) { + if let Err(error) = self.terminate() { + eprintln!("SDK cleanup also failed: {error}"); + } + if let Err(error) = self.output() { + eprintln!("SDK stdout artifact {}: {error}", self.stdout.display()); + } + } +} diff --git a/e2e-test/tests/access_iceberg/gc/notifications/matrix.rs b/e2e-test/tests/access_iceberg/gc/notifications/matrix.rs index e969916e9..0d152f5a9 100644 --- a/e2e-test/tests/access_iceberg/gc/notifications/matrix.rs +++ b/e2e-test/tests/access_iceberg/gc/notifications/matrix.rs @@ -33,20 +33,33 @@ struct TestMatrix { async fn restarted_cleaners_finish_large_and_sparse_backlogs_beside_an_invalid_catalog() { let stack = TestNativeStorage::new(); let (store, context, table) = seed_table(&stack).await; - let large = TestLargeRange::write(&stack, store.clone(), context, table).await; - let records = TestMatrixRecords::seed(store, context, table, large.file.clone()).await; + let started = Instant::now(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(720); + let pressure = TestGcPressure::prepare_before(deadline).await.unwrap(); + let large = tokio::time::timeout_at( + deadline, + TestLargeRange::write(&stack, store.clone(), context, table), + ) + .await + .expect("GC matrix large range seed exceeded the existing case deadline"); + let records = tokio::time::timeout_at( + deadline, + TestMatrixRecords::seed(store, context, table, large.file.clone()), + ) + .await + .expect("GC matrix record seed exceeded the existing case deadline"); let mut check = TestMatrix { records, large, servers: Vec::new(), - pressure: None, - started: Instant::now(), + pressure: Some(pressure), + started, observations: Vec::new(), last_revision: None, restarts: 0, }; - let result = tokio::time::timeout( - Duration::from_secs(720), + let result = tokio::time::timeout_at( + deadline, std::panic::AssertUnwindSafe(check.run(&stack.seeds)).catch_unwind(), ) .await; @@ -124,7 +137,11 @@ impl TestMatrix { let multipart = self.records.multipart().await?; let deleted = task.deleted; self.observe(&task, multipart.as_ref()); - if let Some(pressure) = &mut self.pressure { + let pressure = self + .pressure + .as_mut() + .ok_or("prepared foreground pressure is missing")?; + if pressure.released() { pressure.observe(deleted, task.phase == GcPhase::Complete).await?; } else if deleted > 0 { let remaining = self.records.remaining().await?; @@ -135,7 +152,7 @@ impl TestMatrix { "gc_matrix_foreground_start {}", serde_json::json!({"elapsed_ms":self.started.elapsed().as_millis(),"deleted":deleted,"remaining_canonical":remaining,"revision":task.revision,"phase":format!("{:?}",task.phase)}) ); - self.pressure = Some(TestGcPressure::start(&self.servers[0].origin, seeds.to_vec())?); + pressure.release(seeds.to_vec())?; } if task.phase == GcPhase::Complete && self.records.sparse_complete().await? diff --git a/e2e-test/tests/access_iceberg/gc/sdk_pressure.py b/e2e-test/tests/access_iceberg/gc/sdk_pressure.py index f9571512e..1020221c2 100644 --- a/e2e-test/tests/access_iceberg/gc/sdk_pressure.py +++ b/e2e-test/tests/access_iceberg/gc/sdk_pressure.py @@ -1,17 +1,40 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. -import sys, json, time +import sys, json, time, traceback from concurrent.futures import ThreadPoolExecutor -import requests from pathlib import Path -from pyiceberg.catalog import load_catalog -from pyiceberg.schema import Schema -from pyiceberg.types import LongType, NestedField -control = Path(sys.argv[5]) if len(sys.argv) > 5 else None +control = None + +def wait_for_release(directory, worker): + (directory / f"ready-{worker}").touch() + while True: + failures = sorted(directory.glob("failed-*")) + if failures: + raise RuntimeError(f"foreground peer failed: {failures[0].read_text()}") + if (directory / "stop").exists(): + raise RuntimeError("foreground cancelled before release") + if (directory / "run").exists(): + return + time.sleep(0.01) + +def guarded_run(worker): + try: + return run(worker) + except BaseException: + if control and not any(control.glob("failed-*")): + temporary = control / f"failure-{worker}.tmp" + temporary.write_text(traceback.format_exc()) + temporary.replace(control / f"failed-{worker}") + raise def run(worker): + import requests + from pyiceberg.catalog import load_catalog + from pyiceberg.schema import Schema + from pyiceberg.types import LongType, NestedField + samples = {} windows = {} def measure(operation, call): @@ -27,11 +50,7 @@ def measure(operation, call): namespace = (f"{prefix}-{worker}",) measure("create_namespace", lambda: catalog.create_namespace(namespace)) if control: - (control / f"ready-{worker}").touch() - while not (control / "run").exists(): - if (control / "stop").exists(): - raise RuntimeError("foreground cancelled before release") - time.sleep(0.01) + wait_for_release(control, worker) count = int(sys.argv[2]) if len(sys.argv) > 2 else 3 interval = float(sys.argv[3]) if len(sys.argv) > 3 else 0 # Keep paced workers spread across a cycle so a short deletion interval @@ -68,16 +87,22 @@ def read_metadata(): measure("drop_namespace", lambda: catalog.drop_namespace(namespace)) return samples, windows -with ThreadPoolExecutor(max_workers=4) as executor: - results = list(executor.map(run, range(4))) -combined = {} -combined_windows = {} -for result, windows in results: - for operation, samples in result.items(): - combined.setdefault(operation, []).extend(samples) - combined_windows.setdefault(operation, []).extend(windows.get(operation, [])) -report = {} -for operation, samples in combined.items(): - ordered = sorted(samples) - report[operation] = {"raw_ns":samples,"count":len(samples),"p50_ns":ordered[(len(ordered)-1)//2],"p99_ns":ordered[(99*len(ordered)+99)//100-1],"windows_unix_ns":combined_windows[operation]} -print("gc_foreground_official_sdk " + json.dumps(report), flush=True) +def main(): + global control + control = Path(sys.argv[5]) if len(sys.argv) > 5 else None + with ThreadPoolExecutor(max_workers=4) as executor: + results = list(executor.map(guarded_run, range(4))) + combined = {} + combined_windows = {} + for result, windows in results: + for operation, samples in result.items(): + combined.setdefault(operation, []).extend(samples) + combined_windows.setdefault(operation, []).extend(windows.get(operation, [])) + report = {} + for operation, samples in combined.items(): + ordered = sorted(samples) + report[operation] = {"raw_ns":samples,"count":len(samples),"p50_ns":ordered[(len(ordered)-1)//2],"p99_ns":ordered[(99*len(ordered)+99)//100-1],"windows_unix_ns":combined_windows[operation]} + print("gc_foreground_official_sdk " + json.dumps(report), flush=True) + +if __name__ == "__main__": + main() diff --git a/e2e-test/tests/browser_batch_provenance_test.py b/e2e-test/tests/browser_batch_provenance_test.py new file mode 100644 index 000000000..32782d887 --- /dev/null +++ b/e2e-test/tests/browser_batch_provenance_test.py @@ -0,0 +1,140 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Batch setup failure does not impersonate a dispatched browser failure.""" + +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path + +from browser_batch import run_browser_batches +from fixture import Cluster +from report import write_report + + +class TestCluster(Cluster): + def __init__(self, directory, case): + super().__init__('image', directory) + self.case_name = case + self.start_error = None + self.collection_error = None + self.cleanup_error = None + self.collections = self.cleanups = 0 + (self.artifacts / 'case.json').write_text(json.dumps({ + 'case': case, 'scope': 'console.native.browser.lifecycle'})) + + def start(self): + if self.start_error: + raise self.start_error + return self + + def diagnostics(self): + self.collections += 1 + if self.collection_error: + raise self.collection_error + + def close(self): + self.cleanups += 1 + if self.cleanup_error: + raise self.cleanup_error + self.closed = True + self.write_timings() + + +class BrowserBatchProvenanceTest(unittest.TestCase): + def run_failed_startup(self, directory, *, cleanup_error=None, collection_error=None): + peer = TestCluster(directory, 'prepared-peer') + failed = TestCluster(directory, 'failed-startup') + failed.start_error = RuntimeError('original startup rejected') + peer.cleanup_error, peer.collection_error = cleanup_error, collection_error + output = io.StringIO() + with contextlib.redirect_stdout(output): + errors = run_browser_batches([peer, failed], 2, lambda case: case, + lambda *_: self.fail('client was dispatched')) + self.assertIs(errors[0], failed.start_error) + self.assertEqual((peer.collections, peer.cleanups), (1, 1)) + self.assertEqual((failed.collections, failed.cleanups), (1, 1)) + self.assertTrue(failed.closed) + self.assertEqual(failed.timings['status'], 'failed') + self.assertEqual(output.getvalue().count('[FAILED CASE] failed-startup:'), 1) + self.assertIn('original startup rejected', output.getvalue()) + self.assertIn('[NOT EXECUTED CASE] prepared-peer:', output.getvalue()) + return peer, errors, output.getvalue() + + def test_prepared_peer_is_not_executed_and_cannot_be_reported_as_pass(self): + with tempfile.TemporaryDirectory() as directory: + peer, errors, output = self.run_failed_startup(directory) + self.assertEqual(len(errors), 1) + self.assertTrue(peer.closed) + self.assertNotIn('[FAILED CASE] prepared-peer:', output) + self.assertEqual(peer.timings['status'], 'not_executed') + self.assertIn('failed-startup', peer.timings['not_executed_reason']) + timing = json.loads((peer.artifacts / 'timing.json').read_text()) + self.assertEqual(timing['status'], 'not_executed') + write_report(Path(directory), {'status': 'failed'}) + report = json.loads((Path(directory) / 'results.json').read_text()) + self.assertTrue(all(case['status'] != 'pass' for case in report['cases'])) + self.assertEqual({case['case']: case['status'] for case in report['cases']}, { + 'prepared-peer': 'not_executed', 'failed-startup': 'failed'}) + self.assertEqual(report['timings']['status'], 'failed') + + def test_peer_cleanup_failure_retains_original_startup_and_its_own_error(self): + with tempfile.TemporaryDirectory() as directory: + cleanup = RuntimeError('peer cleanup rejected') + peer, errors, output = self.run_failed_startup(directory, cleanup_error=cleanup) + self.assertEqual(errors[1:], [cleanup]) + self.assertEqual(peer.timings['status'], 'failed') + self.assertIn('[FAILED CASE] prepared-peer: peer cleanup rejected', output) + self.assertNotIn('[FAILED CASE] prepared-peer: original startup rejected', output) + + def test_peer_collection_failure_still_cleans_and_retains_its_own_error(self): + with tempfile.TemporaryDirectory() as directory: + collection = OSError('peer artifact disk full') + peer, errors, output = self.run_failed_startup(directory, collection_error=collection) + self.assertEqual(errors[1:], [collection]) + self.assertTrue(peer.closed) + self.assertEqual(peer.timings['status'], 'failed') + self.assertIn('[FAILED CASE] prepared-peer: peer artifact disk full', output) + self.assertNotIn('[FAILED CASE] prepared-peer: original startup rejected', output) + + def test_peer_collection_and_cleanup_failures_are_both_retained(self): + with tempfile.TemporaryDirectory() as directory: + collection = OSError('peer artifact disk full') + cleanup = RuntimeError('peer cleanup rejected') + peer, errors, output = self.run_failed_startup( + directory, collection_error=collection, cleanup_error=cleanup) + self.assertEqual(errors[1:], [collection]) + self.assertIn(str(cleanup), collection.__notes__) + self.assertIs(collection.__cause__, cleanup) + self.assertEqual(peer.timings['status'], 'failed') + self.assertIn('[FAILED CASE] prepared-peer: peer artifact disk full', output) + self.assertIn('[FAILED CASE] prepared-peer: peer cleanup rejected', output) + + def test_real_client_and_cleanup_failures_remain_independent_of_successful_peer(self): + with tempfile.TemporaryDirectory() as directory: + failed, peer = TestCluster(directory, 'failed-client'), TestCluster(directory, 'passed-client') + failure = RuntimeError('browser assertion failed') + failed.cleanup_error = RuntimeError('owned browser cleanup failed') + executed = [] + + def execute(case, cluster): + executed.append(case) + if cluster is failed: + raise failure + + output = io.StringIO() + with contextlib.redirect_stdout(output): + errors = run_browser_batches([failed, peer], 2, lambda case: case, execute) + self.assertCountEqual(executed, [failed, peer]) + self.assertEqual(errors, [failure]) + self.assertEqual(failed.timings['status'], 'failed') + self.assertEqual(peer.timings['status'], 'pass') + self.assertIn('[FAILED CASE] failed-client: browser assertion failed', output.getvalue()) + self.assertEqual(output.getvalue().count('[FAILED CASE] failed-client:'), 1) + self.assertIn('[CLEANUP ERROR] failed-client: owned browser cleanup failed', output.getvalue()) + self.assertIn('owned browser cleanup failed', failure.__notes__) + self.assertNotIn('[FAILED CASE] passed-client:', output.getvalue()) + self.assertNotIn('[CLEANUP ERROR] passed-client:', output.getvalue()) + self.assertNotIn('[NOT EXECUTED CASE]', output.getvalue()) diff --git a/e2e-test/tests/browser_batch_test.py b/e2e-test/tests/browser_batch_test.py index 50334eed0..9d744f187 100644 --- a/e2e-test/tests/browser_batch_test.py +++ b/e2e-test/tests/browser_batch_test.py @@ -21,6 +21,8 @@ def __exit__(self, kind, error, traceback): owner.events.append(('close', case)) owner.failures[case] = error owner.active.remove(case) + if case == owner.cleanup_failure: + raise RuntimeError('cleanup rejected') return Fixture() @@ -28,6 +30,7 @@ def setUp(self): self.events = [] self.active = set() self.start_failure = None + self.cleanup_failure = None self.failures = {} def test_clients_share_stable_bounded_batch_and_teardown_precedes_next_batch(self): @@ -62,6 +65,17 @@ def test_partial_startup_closes_started_peer_without_dispatching_clients(self): self.assertEqual([str(error) for error in errors], ['startup rejected']) self.assertEqual(self.events, [('start', 0), ('start', 1), ('close', 0)]) self.assertEqual(self.active, set()) + self.assertEqual(str(self.failures[0]), + 'browser client not executed: batch startup failed in 1') + + def test_cleanup_failure_is_not_attributed_to_a_successful_peer(self): + self.cleanup_failure = 1 + errors = run_browser_batches([0, 1], 2, self.fixture, lambda *_: None) + self.assertEqual([str(error) for error in errors], ['cleanup rejected']) + self.assertEqual(self.events[-2:], [('close', 1), ('close', 0)]) + self.assertEqual(self.active, set()) + self.assertIsNone(self.failures[0]) + self.assertIsNone(self.failures[1]) def test_fail_fast_closes_current_batch_before_skipping_later_fixtures(self): def execute(case, fixture): @@ -73,6 +87,14 @@ def execute(case, fixture): self.assertEqual(self.events, [('start', 0), ('close', 0)]) self.assertEqual(self.active, set()) + def test_fail_fast_startup_closes_peer_before_skipping_later_batch(self): + self.start_failure = 1 + errors = run_browser_batches([0, 1, 2], 2, self.fixture, + lambda *_: self.fail('client was dispatched'), fail_fast=True) + self.assertEqual([str(error) for error in errors], ['startup rejected']) + self.assertEqual(self.events, [('start', 0), ('start', 1), ('close', 0)]) + self.assertEqual(self.active, set()) + def test_large_fixture_does_not_serialize_later_small_browser_batches(self): def execute(case, fixture): self.assertEqual(self.active, {0} if case == 0 else {1, 2}) diff --git a/e2e-test/tests/console/native_balance.rs b/e2e-test/tests/console/native_balance.rs index 277a8b0ac..cb8fef8e5 100644 --- a/e2e-test/tests/console/native_balance.rs +++ b/e2e-test/tests/console/native_balance.rs @@ -20,6 +20,7 @@ use std::{ }; pub(super) async fn settle(context: &TestNativeContext) { + let started = Instant::now(); let kv = context.kv.clone(); let source = Arc::new(Group0ChunkKvRangeCatalogSource::from_shared(kv.clone())); let registry = ServiceRegistryClient::from_shared(kv.clone()); @@ -29,9 +30,30 @@ pub(super) async fn settle(context: &TestNativeContext) { Arc::new(ChunkKvRpcTransport::new(64, 1, 2)), ) .unwrap(); + let phase = Instant::now(); + eprintln!("[PHASE] native production balance seed started: 512 records"); seed_values(&client).await; - wait_balanced(&source, ®istry, &kv, Instant::now()).await; + eprintln!( + "[PHASE] native production balance seed completed: phase_ms={}, total_ms={}", + phase.elapsed().as_millis(), + started.elapsed().as_millis() + ); + let phase = Instant::now(); + let catalog = wait_balanced(&source, ®istry, &kv, phase).await; + eprintln!( + "[PHASE] native production balance converged: phase_ms={}, total_ms={}, partitions={}, owners={}", + phase.elapsed().as_millis(), + started.elapsed().as_millis(), + catalog.entries().len(), + counts(&catalog).len() + ); + let phase = Instant::now(); verify_values(&client).await; + eprintln!( + "[PHASE] native production balance verified 512 records: phase_ms={}, total_ms={}", + phase.elapsed().as_millis(), + started.elapsed().as_millis() + ); } pub(super) async fn settle_existing(context: &TestNativeContext) { diff --git a/e2e-test/tests/console_cli_inventory_test.py b/e2e-test/tests/console_cli_inventory_test.py new file mode 100644 index 000000000..f2ac76b72 --- /dev/null +++ b/e2e-test/tests/console_cli_inventory_test.py @@ -0,0 +1,91 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +import copy +import json +import unittest +from unittest.mock import Mock, patch + +from console_cli_target import CliAttachedCluster +from fixture import LABEL + + +class AttachedInventoryTest(unittest.TestCase): + def setUp(self): + self.cluster = object.__new__(CliAttachedCluster) + self.cluster.servers = ["owned-1", "owned-2", "owned-3"] + self.cluster.addresses = ["192.0.2.1", "192.0.2.2", "192.0.2.3"] + self.cluster.run_id = "fixture" + self.cluster.cluster_id = "cluster" + self.cluster.image = "image" + self.exited = set() + self.containers = {name: {"Id": name, "Image": "image", "State": {"Running": True}, + "Config": {"Labels": {LABEL: "fixture"}}} for name in self.cluster.servers} + self.status, self.services = {}, [] + kinds = ("paxos-kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access-server") + for node, name in enumerate(self.cluster.servers, 1): + processes = {"web": {"pid": 20, "generation": 1, "operation_id": None}} + for number, kind in enumerate(kinds, 1): + identity = f"{kind}-{node}" + processes["kv" if kind == "paxos-kv" else identity] = { + "pid": 100 + number, "generation": 1, "operation_id": f"operation-{node}-{number}"} + self.services.append({"id": identity, "node_id": node, "service_type": kind, + "endpoint": f"192.0.2.{node}:{10000 + number}", "health": "unknown", "start_pending": False}) + self.status[name] = {"services": processes} + self.cluster.command = Mock(side_effect=self.command) + self.request = patch("console_cli_target.request", side_effect=lambda *_: copy.deepcopy(self.services)).start() + self.addCleanup(patch.stopall) + + def command(self, action, name, *arguments): + if action == "inspect": + return json.dumps([self.containers[name]]) + if action == "exec" and arguments[0] == "sh": + self.assertEqual(arguments[3], "owned-service-pids") + if any((name, int(pid)) in self.exited for pid in arguments[4:]): + raise RuntimeError("owned service PID is not live") + return "" + self.assertEqual((action, arguments[0]), ("exec", "cat")) + if arguments[1] == "/opt/crowdb/data/node-binding.json": + return json.dumps({"node_id": self.cluster.servers.index(name) + 1, + "bootstrap": {"cluster_id": "cluster"}}) + self.assertEqual(arguments[1], "/opt/crowdb/run/status/monitor.json") + return json.dumps(self.status[name]) + + def test_cold_cache_does_not_hide_process_or_registration_changes(self): + before = self.cluster.inventory() + self.assertEqual(len(before["processes"]), 21) + self.services[0].update(pid=101, pid_node_id=1, health="up") + self.assertEqual(self.cluster.inventory(), before) + self.status["owned-1"]["services"]["chunk-kv-1"]["pid"] += 1 + self.assertNotEqual(self.cluster.inventory(), before) + self.status["owned-1"]["services"]["chunk-kv-1"]["pid"] -= 1 + self.services[0]["endpoint"] = "192.0.2.1:29999" + self.assertNotEqual(self.cluster.inventory(), before) + + def test_foreign_container_is_rejected_before_monitor_access(self): + self.containers["owned-1"]["Config"]["Labels"][LABEL] = "foreign" + with self.assertRaisesRegex(AssertionError, "ownership"): + self.cluster.inventory() + self.assertFalse(any(call.args[0] == "exec" for call in self.cluster.command.call_args_list)) + + def test_invalid_real_pid_and_wrong_cached_namespace_are_rejected(self): + self.status["owned-1"]["services"]["web"]["pid"] = None + with self.assertRaisesRegex(AssertionError, "PID"): + self.cluster.inventory() + self.status["owned-1"]["services"]["web"]["pid"] = 20 + self.services[0].update(pid=101, pid_node_id=2) + with self.assertRaisesRegex(AssertionError, "namespace"): + self.cluster.inventory() + + def test_old_positive_status_does_not_hide_process_exit_or_missing_web(self): + self.exited.add(("owned-1", 20)) + with self.assertRaisesRegex(RuntimeError, "not live"): + self.cluster.inventory() + self.exited.clear() + del self.status["owned-1"]["services"]["web"] + with self.assertRaisesRegex(AssertionError, "service is absent"): + self.cluster.inventory() + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e-test/tests/fixture_test.py b/e2e-test/tests/fixture_test.py index 3f3ddf739..5eea1294e 100644 --- a/e2e-test/tests/fixture_test.py +++ b/e2e-test/tests/fixture_test.py @@ -2,6 +2,8 @@ # Licensed under the Apache License, Version 2.0. """Failure ownership and resource contracts without a container daemon.""" +import contextlib +import io import json import tempfile import subprocess @@ -107,9 +109,35 @@ def test_cleanup_failure_preserves_the_original_error(self): with tempfile.TemporaryDirectory() as directory: daemon = Daemon() daemon.fail_cleanup = True - with self.assertRaisesRegex(RuntimeError, "first server failed") as caught, Cluster("image", directory, daemon): + cluster = Cluster("image", directory, daemon) + cluster.case_name = "owned-startup" + output = io.StringIO() + with contextlib.redirect_stdout(output), \ + self.assertRaisesRegex(RuntimeError, "first server failed") as caught, cluster: pass self.assertIn("owned cleanup failed", " ".join(caught.exception.__notes__)) + self.assertEqual(output.getvalue().count("[FAILED CASE] owned-startup:"), 1) + self.assertIn("[CLEANUP ERROR] owned-startup: owned cleanup failed", output.getvalue()) + self.assertIn("daemon cleanup unavailable", output.getvalue()) + + def test_cleanup_failure_after_client_success_prints_the_actual_case_and_error(self): + with tempfile.TemporaryDirectory() as directory: + daemon = Daemon() + daemon.fail_cleanup = True + cluster = Cluster("image", directory, daemon) + cluster.case_name = "owned-successful-client" + cluster.network_created = True + daemon.resources[("network", cluster.run_id)] = cluster.run_id + output = io.StringIO() + with patch.object(cluster, "start", return_value=cluster), \ + contextlib.redirect_stdout(output), \ + self.assertRaisesRegex(RuntimeError, "owned cleanup failed"), cluster: + pass + self.assertIn("[FAILED CASE] owned-successful-client: owned cleanup failed", output.getvalue()) + self.assertIn("daemon cleanup unavailable", output.getvalue()) + self.assertIn(str(cluster.artifacts), output.getvalue()) + self.assertEqual(cluster.timings["status"], "failed") + self.assertEqual(json.loads((cluster.artifacts / "timing.json").read_text())["status"], "failed") def test_interruption_tears_down_and_keeps_other_fixture(self): with tempfile.TemporaryDirectory() as directory: diff --git a/e2e-test/tests/gc_sdk_pressure_test.py b/e2e-test/tests/gc_sdk_pressure_test.py new file mode 100644 index 000000000..de41ea50b --- /dev/null +++ b/e2e-test/tests/gc_sdk_pressure_test.py @@ -0,0 +1,121 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Control-protocol regressions only; no SDK, endpoint, process or container.""" + +import ast +from concurrent.futures import ThreadPoolExecutor +import os +from pathlib import Path +import tempfile +import threading +import types +import unittest +from unittest.mock import patch + + +def pressure_module(): + source = Path(os.environ.get("CROWDB_GC_PRESSURE_TEST_SOURCE", Path(__file__).parent / + "access_iceberg/gc/sdk_pressure.py")) + tree = ast.parse(source.read_text(), filename=str(source)) + # Load the real control functions without importing or executing an SDK. + declarations = [node for node in tree.body if isinstance(node, ast.FunctionDef) + or isinstance(node, (ast.Import, ast.ImportFrom)) and not ( + isinstance(node, ast.ImportFrom) and node.module.startswith("pyiceberg") + or isinstance(node, ast.Import) and any(alias.name == "requests" for alias in node.names))] + module = types.ModuleType("gc_sdk_pressure") + exec(compile(ast.Module(body=declarations, type_ignores=[]), str(source), "exec"), module.__dict__) + module.control = None + return module + + +class GcSdkPressureControlTest(unittest.TestCase): + def setUp(self): + self.pressure = pressure_module() + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + + def test_all_four_workers_park_without_doing_post_ready_work_until_release(self): + parked = threading.Barrier(5) + release_check = threading.Event() + completed = [] + + def observed_sleep(_seconds): + parked.wait(timeout=2) + if not release_check.wait(timeout=2): + raise TimeoutError("test did not release parked control workers") + + def worker(index): + self.pressure.wait_for_release(self.root, index) + completed.append(index) + + with ThreadPoolExecutor(max_workers=4) as executor: + futures = [] + try: + with patch.object(self.pressure.time, "sleep", side_effect=observed_sleep): + futures = [executor.submit(worker, index) for index in range(4)] + parked.wait(timeout=2) + self.assertEqual(sorted(path.name for path in self.root.glob("ready-*")), + ["ready-0", "ready-1", "ready-2", "ready-3"]) + self.assertFalse((self.root / "run").exists()) + self.assertEqual(completed, []) + (self.root / "run").touch() + release_check.set() + for future in futures: + future.result(timeout=2) + finally: + (self.root / "stop").touch() + release_check.set() + self.assertEqual(sorted(completed), [0, 1, 2, 3]) + + def test_stop_prevents_release_even_if_run_marker_also_exists(self): + (self.root / "stop").touch() + (self.root / "run").touch() + with self.assertRaisesRegex(RuntimeError, "cancelled before release"): + self.pressure.wait_for_release(self.root, 1) + + def test_original_worker_error_is_published_and_unblocks_parked_peer(self): + self.pressure.control = self.root + with patch.object(self.pressure, "run", side_effect=ValueError("original catalog error")): + with self.assertRaisesRegex(ValueError, "original catalog error"): + self.pressure.guarded_run(2) + failure = (self.root / "failed-2").read_text() + self.assertIn("ValueError: original catalog error", failure) + self.assertIn("Traceback", failure) + with self.assertRaisesRegex(RuntimeError, "foreground peer failed:.*"): + self.pressure.wait_for_release(self.root, 0) + + def test_failure_prevents_release_even_when_run_marker_arrives(self): + (self.root / "failed-3").write_text("original SDK HTTP failure") + (self.root / "run").touch() + with self.assertRaisesRegex(RuntimeError, "original SDK HTTP failure"): + self.pressure.wait_for_release(self.root, 2) + + def test_failure_is_not_visible_until_the_complete_traceback_is_published(self): + self.pressure.control = self.root + real_write = Path.write_text + + def interrupted_write(path, text): + real_write(path, text[:len(text) // 2]) + self.assertEqual(list(self.root.glob("failed-*")), []) + return real_write(path, text) + + with patch.object(Path, "write_text", new=interrupted_write), \ + patch.object(self.pressure, "run", side_effect=ValueError("whole original error")): + with self.assertRaisesRegex(ValueError, "whole original error"): + self.pressure.guarded_run(1) + self.assertTrue((self.root / "failed-1").read_text().endswith("ValueError: whole original error\n")) + + def test_peer_cancellation_keeps_the_original_failure_record(self): + self.pressure.control = self.root + original = "ValueError: actual SDK request failed" + (self.root / "failed-3").write_text(original) + with patch.object(self.pressure, "run", side_effect=RuntimeError("foreground peer failed")): + with self.assertRaisesRegex(RuntimeError, "foreground peer failed"): + self.pressure.guarded_run(0) + self.assertEqual(sorted(path.name for path in self.root.glob("failed-*")), ["failed-3"]) + self.assertEqual((self.root / "failed-3").read_text(), original) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e-test/tests/journal_request_stability_test.py b/e2e-test/tests/journal_request_stability_test.py new file mode 100644 index 000000000..81420e5ac --- /dev/null +++ b/e2e-test/tests/journal_request_stability_test.py @@ -0,0 +1,20 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Exercise exact journal request/response ownership without intercepting E2E APIs.""" + +from pathlib import Path +import subprocess +import unittest + + +class JournalRequestStabilityTest(unittest.TestCase): + def test_journal_runtime_request_identity_and_refresh_order(self): + script = Path(__file__).with_name("journal_runtime_request_test.mjs") + result = subprocess.run(["pixi", "run", "node", "--experimental-strip-types", "--test", str(script)], + capture_output=True, text=True, timeout=60) + print(result.stdout + result.stderr, end="", flush=True) + self.assertEqual(result.returncode, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e-test/tests/journal_runtime_request_test.mjs b/e2e-test/tests/journal_runtime_request_test.mjs new file mode 100644 index 000000000..c6d623953 --- /dev/null +++ b/e2e-test/tests/journal_runtime_request_test.mjs @@ -0,0 +1,74 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { test } from 'node:test'; +import { journalRuntimeResponse, matchesJournalRuntimeRequest } from '../clients/console-ui/e2e/fixtures/journalRuntimeRequest.ts'; + +const target = { origin: 'http://owned', id: 'a'.repeat(32), epoch: '9007199254740993', generation: '9007199254740997', page: 2, offset: 17 }; +const url = () => new URL('/api/chunk-kv/runtime?' + new URLSearchParams({ id: target.id, epoch: target.epoch, + generation: target.generation, page: String(target.page), offset: String(target.offset), stream_offset: '0' }), target.origin); +const request = (address = url(), method = 'GET') => ({ url: () => String(address), method: () => method, + failure: () => ({ errorText: 'owned request aborted' }) }); +const response = (owner, finished = async () => null) => ({ request: () => owner, finished }); +const remaining = () => 1_000; + +test('journal matcher rejects another partition and every old resource or cursor component', () => { + assert.equal(matchesJournalRuntimeRequest(request(), target), true); + for (const [name, value] of [['id', 'b'.repeat(32)], ['epoch', '1'], ['generation', '2'], + ['page', '0'], ['offset', '0'], ['stream_offset', '20'], ['stream_generation', 'old'], + ['page_path', ''], ['entry_offset', '0']]) { + const changed = url(); changed.searchParams.set(name, value); + assert.equal(matchesJournalRuntimeRequest(request(changed), target), false, name); + } + const duplicate = url(); duplicate.searchParams.append('id', target.id); + assert.equal(matchesJournalRuntimeRequest(request(duplicate), target), false); + const foreign = url(); foreign.hostname = 'foreign'; + assert.equal(matchesJournalRuntimeRequest(request(foreign), target), false); + assert.equal(matchesJournalRuntimeRequest(request(url(), 'POST'), target), false); +}); + +test('an automatic response started before Refresh cannot satisfy its request observation', async () => { + const page = new EventEmitter(); + const automatic = request(); + page.emit('request', automatic); + const refreshed = request(); + const wrong = url(); wrong.searchParams.set('id', 'b'.repeat(32)); + const unrelated = request(wrong); + const observed = await journalRuntimeResponse(page, target, async () => { + page.emit('response', response(automatic)); + page.emit('request', unrelated); page.emit('response', response(unrelated)); + page.emit('request', refreshed); page.emit('response', response(refreshed)); + }, remaining); + assert.equal(observed.request(), refreshed); + assert.equal(page.listenerCount('request'), 0); + assert.equal(page.listenerCount('response'), 0); + assert.equal(page.listenerCount('requestfailed'), 0); +}); + +test('Refresh ignores an old cursor and waits for its exact response body to finish', async () => { + const page = new EventEmitter(); + const oldUrl = url(); oldUrl.searchParams.set('stream_generation', 'old'); + const old = request(oldUrl), refreshed = request(); + let finish; + const body = new Promise(resolve => { finish = resolve; }); + let completed = false; + const pending = journalRuntimeResponse(page, target, async () => { + page.emit('request', old); page.emit('response', response(old)); + page.emit('request', refreshed); page.emit('response', response(refreshed, () => body)); + }, remaining).then(value => { completed = true; return value; }); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(completed, false); + finish(null); + assert.equal((await pending).request(), refreshed); +}); + +test('request failure or action failure removes all listeners and keeps the real error', async () => { + const page = new EventEmitter(), failed = request(); + await assert.rejects(journalRuntimeResponse(page, target, async () => { + page.emit('request', failed); page.emit('requestfailed', failed); + }, remaining), /owned request aborted/); + await assert.rejects(journalRuntimeResponse(page, target, async () => { throw new Error('selection click failed'); }, remaining), /selection click failed/); + for (const name of ['request', 'response', 'requestfailed']) assert.equal(page.listenerCount(name), 0); +}); diff --git a/e2e-test/tests/oci_shortcuts_test.py b/e2e-test/tests/oci_shortcuts_test.py index 21811bb66..7929cf0ee 100644 --- a/e2e-test/tests/oci_shortcuts_test.py +++ b/e2e-test/tests/oci_shortcuts_test.py @@ -2,6 +2,9 @@ # Licensed under the Apache License, Version 2.0. from types import SimpleNamespace +from pathlib import Path +import subprocess +import tempfile import unittest from unittest.mock import patch @@ -9,6 +12,65 @@ class OciShortcutRetentionTest(unittest.TestCase): + def test_cleanup_only_selects_owned_receipts_after_success_or_failure(self): + for fail in (False, True): + with self.subTest(fail=fail), tempfile.TemporaryDirectory() as directory: + artifacts = Path(directory) + receipts = artifacts / 'shortcut-runtime/persistent/oci-clusters' + foreign = receipts / 'unrelated.json' + running, cleaned = {}, [] + + def invoke(command, **kwargs): + receipts.mkdir(parents=True, exist_ok=True) + if not foreign.exists(): + foreign.write_text('unrelated retained data') + task, arguments = command[2], command[3:] + output, code = '', 0 + if task == 'sh': + output = kwargs['env']['PATH'].split(':')[0] + '/crowdb-cli\n' + else: + self.assertNotIn('--all', arguments, 'fixture must never clean the shared daemon') + name = arguments[arguments.index('--cluster') + 1] + if task == 'oci-run': + code = 1 + elif task == 'oci-run-3c': + (receipts / (name + '.json')).write_text('owned') + running[name] = ['first', 'second', 'third'] + elif task == 'oci-clean': + cleaned.append((name, '--keep-data' in arguments)) + self.assertEqual((receipts / (name + '.json')).read_text(), 'owned') + running.pop(name, None) + if '--keep-data' not in arguments: + (receipts / (name + '.json')).unlink() + else: + self.fail(task) + return subprocess.CompletedProcess(command, code, output, '') + + def named(_cluster, _invoke, _ready, _ids, _credentials, one, other, _port): + if fail: + for name in (one, other): + (receipts / (name + '.json')).write_text('owned') + raise RuntimeError('injected preparation failure') + return 'one-id', 'other-id', [] + + cluster = SimpleNamespace(artifacts=artifacts, run_id='fixture-123456789012', + image='owned-image', image_id='owned-image-id', port_base=20000, + command=lambda *args: '\n'.join(running.get(args[-1].rsplit('=', 1)[1], []))) + ready = dict(phase='ready', observation=dict(policy=dict(mode='local'), + initial_voters=3, cluster_id='three-id')) + with patch.object(oci_shortcuts.subprocess, 'run', side_effect=invoke), \ + patch.object(oci_shortcuts, 'named_clusters', side_effect=named), \ + patch.object(oci_shortcuts, 'status', return_value=ready): + if fail: + with self.assertRaisesRegex(RuntimeError, 'injected preparation failure'): + oci_shortcuts.verify(cluster) + else: + oci_shortcuts.verify(cluster) + self.assertEqual(foreign.read_text(), 'unrelated retained data') + expected = [('pixi-123456789012-' + suffix, fail) + for suffix in (('a', 'b') if fail else ('c',))] + self.assertEqual(cleaned, expected) + def test_named_cluster_recreation_preserves_identity_ports_credentials_and_writes(self): saved, running, generations, racks = {}, {}, {}, {} diff --git a/e2e-test/tests/report_test.py b/e2e-test/tests/report_test.py index dfe0f3043..1ac5bc941 100644 --- a/e2e-test/tests/report_test.py +++ b/e2e-test/tests/report_test.py @@ -2,10 +2,16 @@ # Licensed under the Apache License, Version 2.0. """A passing client cannot hide an incomplete fixture lifecycle.""" +import contextlib +import importlib.util +import io import json +import subprocess +import sys import tempfile import unittest from pathlib import Path +from unittest.mock import patch from report import write_report @@ -14,6 +20,7 @@ class ReportTest(unittest.TestCase): def test_expected_nonzero_exit_is_passed_only_when_fixture_lifecycle_also_passed(self): for fixture_status, actual, expected, outcome in ( ("pass", 1, 1, "pass"), ("pass", 2, 1, "failed"), ("failed", 1, 1, "failed"), + ("not_executed", 0, 0, "not_executed"), ): with self.subTest(fixture=fixture_status, actual=actual, expected=expected): with tempfile.TemporaryDirectory() as directory: @@ -47,3 +54,36 @@ def test_failed_cleanup_is_a_failed_case_even_after_client_passed(self): self.assertEqual(results["cases"][0]["status"], "failed") self.assertEqual(results["timings"]["status"], "failed") self.assertEqual(results["slowest_clients"][0]["seconds"], 1.0) + + def test_measurement_counts_not_executed_separately_and_retains_batch_failure(self): + path = Path(__file__).resolve().parents[2] / 'tools/test-metrics/measure.py' + spec = importlib.util.spec_from_file_location('fixture_report_measure', path) + measure = importlib.util.module_from_spec(spec) + spec.loader.exec_module(measure) + + def fake_command(command, **kwargs): + if command[:3] == ['pixi', 'run', 'clean']: + return subprocess.CompletedProcess(command, 0) + output = Path(command[command.index('--artifacts') + 1]) / 'run-test' + output.mkdir(parents=True) + (output / 'results.json').write_text(json.dumps({ + 'timings': {'status': 'failed'}, + 'cases': [{'status': status} for status in ('pass', 'failed', 'not_executed')]})) + kwargs['stdout'].write('original startup rejected\n') + return subprocess.CompletedProcess(command, 1) + + with tempfile.TemporaryDirectory() as directory, \ + patch.object(measure, 'ROOT', Path(directory)), \ + patch.object(measure, 'TaskGraph') as graph, \ + patch.object(measure.subprocess, 'run', side_effect=fake_command), \ + patch.object(sys, 'argv', ['measure.py', 'test-e2e-console']), \ + contextlib.redirect_stdout(io.StringIO()) as console: + graph.return_value.tasks = {('default', 'test-e2e-console'): 'unused'} + self.assertEqual(measure.main(), 1) + record = json.loads((Path(directory) / + '.crowdb-runtime/artifacts/measure-tests/latest.json').read_text())[0] + self.assertEqual(record['passed'], 1) + self.assertEqual(record['failed'], 1) + self.assertEqual(record['not_executed'], 1) + self.assertEqual(record['exit_code'], 1) + self.assertIn('original startup rejected', console.getvalue()) diff --git a/lib/crowdb-access-iceberg/src/file/reader.rs b/lib/crowdb-access-iceberg/src/file/reader.rs index 8aa53a507..af98d0075 100644 --- a/lib/crowdb-access-iceberg/src/file/reader.rs +++ b/lib/crowdb-access-iceberg/src/file/reader.rs @@ -19,12 +19,19 @@ pub struct FileReader { end: u64, frame_bytes: usize, cached: Option<(u64, Vec)>, - leaf_directory: Option<(ChunkRoot, Vec)>, + leaf_directory: Option, digest: Option, locations: Option>, failed: bool, } +struct CachedDirectory { + root: ChunkRoot, + start: u64, + length: u64, + bytes: Vec, +} + struct ReadSource { owner: FileIdentity, length: u64, @@ -67,7 +74,12 @@ impl FileReader { }; verify_block(root, &bytes)?; ChunkDirectory::decode(&bytes, self.record.owner, root.height, self.record.length)?; - self.leaf_directory = Some((root.clone(), bytes)); + self.leaf_directory = Some(CachedDirectory { + root: root.clone(), + start: 0, + length: self.record.length, + bytes, + }); Ok(self) } /// # Errors @@ -158,7 +170,7 @@ impl FileReader { pub fn retained_directory_bytes(&self) -> usize { self.leaf_directory .as_ref() - .map_or(0, |(_, bytes)| bytes.capacity()) + .map_or(0, |directory| directory.bytes.capacity()) } /// # Errors @@ -214,11 +226,18 @@ impl FileReader { let FileContent::Chunks { root: Some(root) } = &self.record.content else { return Err(ValidationError::Record.into()); }; - let mut root = root.clone(); - let mut start = 0; - let mut length = self.record.length; + let (mut root, mut start, mut length) = self + .leaf_directory + .as_ref() + .filter(|directory| { + self.cursor >= directory.start && self.cursor - directory.start < directory.length + }) + .map_or_else( + || (root.clone(), 0, self.record.length), + |directory| (directory.root.clone(), directory.start, directory.length), + ); while root.height > 0 { - let directory = self.directory(&root, length).await?; + let directory = self.directory(&root, start, length).await?; let mut selected = None; for entry in directory.entries { if self.cursor - start < entry.length { @@ -239,11 +258,16 @@ impl FileReader { Ok((start, bytes)) } - async fn directory(&mut self, root: &ChunkRoot, length: u64) -> Result { - if let Some((selected, bytes)) = &self.leaf_directory { - if selected == root { + async fn directory( + &mut self, + root: &ChunkRoot, + start: u64, + length: u64, + ) -> Result { + if let Some(selected) = &self.leaf_directory { + if selected.root == *root && selected.start == start && selected.length == length { return Ok(ChunkDirectory::decode( - bytes, + &selected.bytes, self.record.owner, root.height, length, @@ -257,7 +281,12 @@ impl FileReader { verify_block(root, &bytes)?; let directory = ChunkDirectory::decode(&bytes, self.record.owner, root.height, length)?; if root.height == 1 { - self.leaf_directory = Some((root.clone(), bytes)); + self.leaf_directory = Some(CachedDirectory { + root: root.clone(), + start, + length, + bytes, + }); } Ok(directory) } diff --git a/lib/crowdb-access-iceberg/tests/file_stream_test.rs b/lib/crowdb-access-iceberg/tests/file_stream_test.rs index 8db08a7c3..578fc8212 100644 --- a/lib/crowdb-access-iceberg/tests/file_stream_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_stream_test.rs @@ -111,8 +111,8 @@ async fn streaming_reuses_only_one_verified_leaf_directory_without_prefetch() { let input = vec![42; length]; writer.push(&input).await.unwrap(); let tree = writer.finish().await.unwrap(); - let parent_reads = if length > 256 { length } else { 0 }; let directory_reads = length.div_ceil(256); + let parent_reads = if length > 256 { directory_reads } else { 0 }; let mut reader = FileReader::new(store.clone(), record(owner, tree), None, 1).unwrap(); for expected in &input { assert_eq!(reader.next().await.unwrap().unwrap(), [*expected]); @@ -127,6 +127,30 @@ async fn streaming_reuses_only_one_verified_leaf_directory_without_prefetch() { } } +#[tokio::test] +async fn ranged_stream_reuses_the_current_directory_and_checks_the_next_boundary() { + let store = Arc::new(TestBlocks::default()); + let owner = owner(); + let input: Vec<_> = (0..513).map(|index| u8::try_from(index % 251).unwrap()).collect(); + let mut writer = FileTreeWriter::new(store.clone(), owner, 1).unwrap(); + writer.push(&input).await.unwrap(); + let mut reader = FileReader::new( + store.clone(), + record(owner, writer.finish().await.unwrap()), + Some(ByteRange { start: 254, end: 258 }), + 1, + ) + .unwrap(); + for expected in &input[254..256] { + assert_eq!(reader.next().await.unwrap().unwrap(), [*expected]); + } + assert_eq!(store.reads.load(Ordering::SeqCst), 4); + store.corrupt_reads.store(true, Ordering::SeqCst); + assert!(reader.next().await.is_err()); + assert!(reader.next().await.is_err()); + assert_eq!(store.reads.load(Ordering::SeqCst), 5); +} + #[tokio::test] async fn retained_directory_does_not_hide_new_leaf_corruption() { let store = Arc::new(TestBlocks::default()); diff --git a/lib/crowdb-chunk-kv/src/partition.rs b/lib/crowdb-chunk-kv/src/partition.rs index 67f77e6a2..c5b9db0c0 100644 --- a/lib/crowdb-chunk-kv/src/partition.rs +++ b/lib/crowdb-chunk-kv/src/partition.rs @@ -12,6 +12,7 @@ mod frontier; mod ingress; mod journal; mod maintenance; +mod native; mod observation; mod replay; mod scan; @@ -24,6 +25,7 @@ pub use frame::{decode_frame, encode_frame, DecodedFrame, FrameDecode, MAX_FRAME pub use frontier::JournalFrontier; pub use ingress::SplitIngress; pub use journal::{PartitionJournal, StreamPartitionJournal}; +use native::native_storage_parts; pub use observation::TreeObservation; use replay::{replay_suffix, validate_replay_record, ReplayState}; pub use split::{PreparedSplit, PreparedSplitWriter, SplitHandoffStore, SplitWriterTarget}; @@ -510,33 +512,13 @@ impl Partition { "tree identity and ownership epoch must be nonzero".into(), )); } - let replay_offset = if page_store.is_chunk_backed() && tree::chunk_root_generation(&page_store)? == 0 - { - 0 - } else { - page_store.wal_replay_offset().map_err(|error| match error { - crowdb_tree_ffi::CtError::Corruption => ChunkKvError::TreeCorruption(error.to_string()), - _ => ChunkKvError::TreeUnavailable(error.to_string()), - })? - }; - let stream_name = stream.stream_name(); - let stream_generation = stream.record_generation(); - let (tree, journal) = native_storage_parts(tree_id, &range, tree_config, page_store, stream)?; - let (tree_manifest, applied_seq) = tree.checkpoint_state()?; - let root_manifest_generation = tree.root_manifest_generation()?; + let (tree, journal, checkpoint) = + native::open_latest(tree_id, range.clone(), tree_config, page_store, stream).await?; Self::recover_assignment( partition_id, range, ownership_epoch, - Checkpoint { - tree_id, - tree_manifest, - root_manifest_generation, - applied_seq, - stream_name, - stream_generation, - replay_offset, - }, + checkpoint, config, tree, journal, @@ -565,7 +547,7 @@ impl Partition { )); } let (tree, journal) = - native_storage_parts(checkpoint.tree_id, &range, tree_config, page_store, stream)?; + native::open_storage(checkpoint.tree_id, range.clone(), tree_config, page_store, stream).await?; Self::recover_assignment( partition_id, range, @@ -744,7 +726,7 @@ impl Partition { }; let range = artifact.range.clone(); let (tree, journal) = - native_storage_parts(artifact.tree_id, &range, tree_config, page_store, stream)?; + native::open_storage(artifact.tree_id, range, tree_config, page_store, stream).await?; let parent_journal: Arc = Arc::new(StreamPartitionJournal::new( parent_stream, artifact.parent_stream_name, @@ -1722,24 +1704,6 @@ impl Partition { } } -fn native_storage_parts( - tree_id: u64, - range: &PartitionRange, - mut tree_config: crowdb_tree_ffi::Config, - page_store: Arc, - stream: ChunkStream, -) -> Result<(Arc, Arc)> { - tree_config.page_store = Some(page_store); - tree_config.key_range = crowdb_tree_ffi::KeyRange::Bounded { - start: range.start.clone(), - end: range.end.clone(), - }; - let tree: Arc = Arc::new(CrowdbPartitionTree::open(tree_id, &tree_config)?); - let stream_name = stream.stream_name(); - let journal: Arc = Arc::new(StreamPartitionJournal::new(stream, stream_name)?); - Ok((tree, journal)) -} - fn validate_split_artifact(plan: &SplitPlan, artifact: &SplitArtifact, cutover_seq: u64) -> Result<()> { if artifact.transition_id != plan.transition_id || artifact.parent_id != plan.parent_id diff --git a/lib/crowdb-chunk-kv/src/partition/native.rs b/lib/crowdb-chunk-kv/src/partition/native.rs new file mode 100644 index 000000000..ec71219f2 --- /dev/null +++ b/lib/crowdb-chunk-kv/src/partition/native.rs @@ -0,0 +1,81 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Blocking native storage construction for partition recovery. + +use super::{tree, CrowdbPartitionTree, PartitionJournal, PartitionTree, StreamPartitionJournal}; +use crate::{Checkpoint, ChunkKvError, PartitionRange, Result}; +use crowdb_chunk_stream::ChunkStream; +use std::sync::Arc; + +pub(super) fn native_storage_parts( + tree_id: u64, + range: &PartitionRange, + mut tree_config: crowdb_tree_ffi::Config, + page_store: Arc, + stream: ChunkStream, +) -> Result<(Arc, Arc)> { + tree_config.page_store = Some(page_store); + tree_config.key_range = crowdb_tree_ffi::KeyRange::Bounded { + start: range.start.clone(), + end: range.end.clone(), + }; + let tree: Arc = Arc::new(CrowdbPartitionTree::open(tree_id, &tree_config)?); + let stream_name = stream.stream_name(); + let journal: Arc = Arc::new(StreamPartitionJournal::new(stream, stream_name)?); + Ok((tree, journal)) +} + +pub(super) async fn open_storage( + tree_id: u64, + range: PartitionRange, + tree_config: crowdb_tree_ffi::Config, + page_store: Arc, + stream: ChunkStream, +) -> Result<(Arc, Arc)> { + tokio::task::spawn_blocking(move || { + native_storage_parts(tree_id, &range, tree_config, page_store, stream) + }) + .await + .map_err(|error| ChunkKvError::TreeUnavailable(error.to_string()))? +} + +pub(super) async fn open_latest( + tree_id: u64, + range: PartitionRange, + tree_config: crowdb_tree_ffi::Config, + page_store: Arc, + stream: ChunkStream, +) -> Result<(Arc, Arc, Checkpoint)> { + tokio::task::spawn_blocking(move || { + let replay_offset = if page_store.is_chunk_backed() && tree::chunk_root_generation(&page_store)? == 0 + { + 0 + } else { + page_store.wal_replay_offset().map_err(|error| match error { + crowdb_tree_ffi::CtError::Corruption => ChunkKvError::TreeCorruption(error.to_string()), + _ => ChunkKvError::TreeUnavailable(error.to_string()), + })? + }; + let stream_name = stream.stream_name(); + let stream_generation = stream.record_generation(); + let (tree, journal) = native_storage_parts(tree_id, &range, tree_config, page_store, stream)?; + let (tree_manifest, applied_seq) = tree.checkpoint_state()?; + let root_manifest_generation = tree.root_manifest_generation()?; + Ok(( + tree, + journal, + Checkpoint { + tree_id, + tree_manifest, + root_manifest_generation, + applied_seq, + stream_name, + stream_generation, + replay_offset, + }, + )) + }) + .await + .map_err(|error| ChunkKvError::TreeUnavailable(error.to_string()))? +} diff --git a/lib/crowdb-chunk-kv/src/partition/tree.rs b/lib/crowdb-chunk-kv/src/partition/tree.rs index c12de5d29..6dcd17fd0 100644 --- a/lib/crowdb-chunk-kv/src/partition/tree.rs +++ b/lib/crowdb-chunk-kv/src/partition/tree.rs @@ -469,10 +469,13 @@ impl PartitionTree for CrowdbPartitionTree { wal_replay_offset: u64, ) -> Result<(u64, u64, std::sync::Arc)> { let (generation, applied_seq) = self.checkpoint(wal_replay_offset).await?; - let config = self.config.as_ref().ok_or_else(|| { + let config = self.config.clone().ok_or_else(|| { ChunkKvError::InvalidRequest("native tree was not opened from a retained configuration".into()) })?; - let snapshot = Self::open(self.tree_id, config)?; + let tree_id = self.tree_id; + let snapshot = tokio::task::spawn_blocking(move || Self::open(tree_id, &config)) + .await + .map_err(|error| ChunkKvError::TreeUnavailable(error.to_string()))??; let observed = snapshot.tree.snapshot_state().map_err(map_tree_read_error)?; if observed != (generation, applied_seq) { return Err(ChunkKvError::TreeCorruption(format!( @@ -500,13 +503,20 @@ impl PartitionTree for CrowdbPartitionTree { start: range.start.clone(), end: range.end.clone(), }; - let retained_config = config.clone(); - let (tree, stats) = self.tree.rebuild_range(&config).map_err(map_tree_read_error)?; + let source = std::sync::Arc::clone(&self.tree); + let (tree, stats, config) = tokio::task::spawn_blocking(move || { + source + .rebuild_range(&config) + .map(|(tree, stats)| (tree, stats, config)) + }) + .await + .map_err(|error| ChunkKvError::TreeUnavailable(error.to_string()))? + .map_err(map_tree_read_error)?; Ok(( std::sync::Arc::new(Self { tree_id, tree: std::sync::Arc::new(tree), - config: Some(retained_config), + config: Some(config), }), stats, )) diff --git a/lib/crowdb-chunk-kv/tests/common/root_catalog.rs b/lib/crowdb-chunk-kv/tests/common/root_catalog.rs new file mode 100644 index 000000000..927a0585e --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/common/root_catalog.rs @@ -0,0 +1,67 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use arc_swap::ArcSwap; +use crowdb_tree_ffi::{CtError, RootCatalogObject, RootCatalogStore}; +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +#[derive(Default)] +pub struct TestRootCatalog { + objects: ArcSwap>>, + next_reference: AtomicU64, + pub armed: AtomicBool, + pub entered: tokio::sync::Notify, + pub async_progress: AtomicBool, + pub observed_progress: AtomicBool, +} + +impl TestRootCatalog { + fn key(tree_id: u64, object: RootCatalogObject) -> (u64, u8, u64) { + match object { + RootCatalogObject::CurrentManifest => (tree_id, 0, 0), + RootCatalogObject::Manifest(generation) => (tree_id, 1, generation), + RootCatalogObject::ReferenceSegment(id) => (tree_id, 2, id), + } + } + + fn wait_for_async_progress(&self) { + if !self.armed.swap(false, Ordering::AcqRel) { + return; + } + self.entered.notify_one(); + let deadline = Instant::now() + Duration::from_secs(1); + while !self.async_progress.load(Ordering::Acquire) && Instant::now() < deadline { + std::thread::yield_now(); + } + self.observed_progress + .store(self.async_progress.load(Ordering::Acquire), Ordering::Release); + } +} + +impl RootCatalogStore for TestRootCatalog { + fn load(&self, tree_id: u64, object: RootCatalogObject) -> Result>, CtError> { + self.wait_for_async_progress(); + Ok(self.objects.load().get(&Self::key(tree_id, object)).cloned()) + } + + fn store(&self, tree_id: u64, object: RootCatalogObject, data: &[u8]) -> Result<(), CtError> { + self.objects.rcu(|previous| { + let mut next = (**previous).clone(); + next.insert(Self::key(tree_id, object), data.to_vec()); + next + }); + Ok(()) + } + + fn publish(&self, tree_id: u64, _: u64, _: u64, generation: u64, manifest: &[u8]) -> Result<(), CtError> { + self.wait_for_async_progress(); + self.store(tree_id, RootCatalogObject::Manifest(generation), manifest)?; + self.store(tree_id, RootCatalogObject::CurrentManifest, manifest) + } + + fn allocate_reference_segment_id(&self, _: u64) -> Result { + Ok(self.next_reference.fetch_add(1, Ordering::AcqRel) + 1) + } +} diff --git a/lib/crowdb-chunk-kv/tests/native_runtime_test.rs b/lib/crowdb-chunk-kv/tests/native_runtime_test.rs new file mode 100644 index 000000000..d21253a94 --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/native_runtime_test.rs @@ -0,0 +1,135 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/root_catalog.rs"] +mod root_catalog; + +use crowdb_chunk_kv::{CrowdbPartitionTree, MutationOperation, PartitionRange, PartitionTree}; +use crowdb_chunk_kv::{Partition, PartitionConfig, PartitionId}; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamName, StreamPurpose, StreamRecord, + StreamRecordState, +}; +use crowdb_tree_ffi::{ChunkPageStoreOptions, ChunkRootCatalog, Config, PageStore}; +use root_catalog::TestRootCatalog; +use std::sync::atomic::Ordering; +use std::sync::Arc; + +fn config(tree_id: u64, catalog: &Arc) -> Config { + Config { + page_store: Some(Arc::new( + PageStore::open_chunk( + ChunkPageStoreOptions { + tree_id, + owner_epoch: 1, + open_generation: 0, + pack_bytes: 4096, + iu_size: 1, + max_concurrent_packs: 2, + materialization_bytes_per_pass: 4096, + max_chunk_bytes: 1024 * 1024, + mirror_copies: 0, + }, + catalog.clone(), + None, + ) + .unwrap(), + )), + ..Config::default() + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn range_rebuild_keeps_the_runtime_available_during_catalog_io() { + let backend = Arc::new(TestRootCatalog::default()); + let catalog = Arc::new(ChunkRootCatalog::open_callback(backend.clone()).unwrap()); + let source = CrowdbPartitionTree::open(91, &config(91, &catalog)).unwrap(); + source + .apply( + 1, + &MutationOperation::Put { + key: b"key".to_vec(), + value: b"value".to_vec(), + }, + ) + .await + .unwrap(); + source.checkpoint(0).await.unwrap(); + let child_config = config(92, &catalog); + let progress = backend.clone(); + let heartbeat = tokio::spawn(async move { + progress.entered.notified().await; + progress.async_progress.store(true, Ordering::Release); + }); + backend.armed.store(true, Ordering::Release); + let (child, _) = tokio::spawn(async move { + source + .rebuild_range(92, &PartitionRange::default(), child_config) + .await + .unwrap() + }) + .await + .unwrap(); + heartbeat.await.unwrap(); + assert!( + backend.observed_progress.load(Ordering::Acquire), + "catalog IO blocked the only async worker" + ); + assert_eq!(child.get(b"key").await.unwrap().unwrap().value, b"value"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn native_recovery_keeps_the_runtime_available_during_catalog_io() { + let backend = Arc::new(TestRootCatalog::default()); + let catalog = Arc::new(ChunkRootCatalog::open_callback(backend.clone()).unwrap()); + let config = config(93, &catalog); + let pages = config.page_store.clone().unwrap(); + let store = Arc::new(MemoryStreamStore::new(4096)); + let stream = ChunkStream::create( + StreamRecord { + purpose: StreamPurpose::Wal, + stream_name: StreamName { high: 1, low: 93 }, + metadata_group_id: 7, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + 1, + StreamConfig::default(), + store.clone(), + store.clone(), + store, + ) + .await + .unwrap(); + let progress = backend.clone(); + let heartbeat = tokio::spawn(async move { + progress.entered.notified().await; + progress.async_progress.store(true, Ordering::Release); + }); + backend.armed.store(true, Ordering::Release); + let recovered = tokio::spawn(async move { + Partition::recover_native_latest_prepared_assignment( + PartitionId { high: 1, low: 93 }, + PartitionRange::default(), + 1, + 93, + PartitionConfig::default(), + config, + pages, + stream, + ) + .await + .unwrap() + }) + .await + .unwrap(); + heartbeat.await.unwrap(); + assert!( + backend.observed_progress.load(Ordering::Acquire), + "native recovery blocked the only async worker" + ); + assert_eq!(recovered.snapshot().applied_seq, 0); +} diff --git a/lib/crowdb-rpc/ffi/src/logging.rs b/lib/crowdb-rpc/ffi/src/logging.rs index 573ab583e..8c428df6e 100644 --- a/lib/crowdb-rpc/ffi/src/logging.rs +++ b/lib/crowdb-rpc/ffi/src/logging.rs @@ -51,6 +51,7 @@ pub fn add_log_stderr(level: &str) { /// Flush and stop the C++ spdlog logger. Call at process exit. pub fn shutdown_logging() { + crate::owned_route::drain_before_native_shutdown(); unsafe { sys::crowdb_rpc_shutdown_logging() }; } diff --git a/lib/crowdb-rpc/ffi/src/owned_route.rs b/lib/crowdb-rpc/ffi/src/owned_route.rs index 7f33c94ae..d6c9f73d6 100644 --- a/lib/crowdb-rpc/ffi/src/owned_route.rs +++ b/lib/crowdb-rpc/ffi/src/owned_route.rs @@ -8,6 +8,10 @@ use std::sync::Arc; use crate::{Connection, RpcClient, RpcServer}; +mod retirement; + +pub(crate) use retirement::shutdown as drain_before_native_shutdown; + /// Supplies the raw crowdb-rpc client handle while an owner keeps it alive. pub trait RpcClientHandle: Send + Sync { #[doc(hidden)] @@ -23,19 +27,41 @@ impl RpcClientHandle for RpcClient { /// A matching client, server, and connection whose owners travel together. /// /// This is the safe handoff boundary for higher-level FFIs that retain raw -/// crowdb-rpc handles after their constructor returns. +/// crowdb-rpc handles after their constructor returns. Clones share one owner +/// bundle. Its final release schedules native teardown outside RPC callbacks; +/// those callbacks must return without waiting for that teardown. #[derive(Clone)] pub struct OwnedClientRoute { + handles: Arc, +} + +struct RouteHandles { + owners: Option, +} + +struct RouteOwners { client: Arc, server: Arc, connection: Connection, } +impl Drop for RouteHandles { + fn drop(&mut self) { + if let Some(owners) = self.owners.take() { + retirement::retire(owners); + } + } +} + impl OwnedClientRoute { + fn owners(&self) -> &RouteOwners { + self.handles.owners.as_ref().expect("live route owners") + } + /// Whether this retained connection can still accept requests. #[must_use] pub fn is_open(&self) -> bool { - self.connection.is_open() + self.owners().connection.is_open() } #[must_use] @@ -43,22 +69,52 @@ impl OwnedClientRoute { where C: RpcClientHandle + 'static, { + // Initialize the lifecycle worker before publishing a route. Ordinary + // request-path clones and releases only change this Arc's refcount. + retirement::register_owner(); Self { - client, - server, - connection, + handles: Arc::new(RouteHandles { + owners: Some(RouteOwners { + client, + server, + connection, + }), + }), } } #[doc(hidden)] #[must_use] pub fn raw_handles(&self) -> (*mut c_void, *mut c_void, *mut c_void) { + let owners = self.owners(); ( - self.client.rpc_client_handle(), - self.server.handle().cast(), - self.connection.handle().cast(), + owners.client.rpc_client_handle(), + owners.server.handle().cast(), + owners.connection.handle().cast(), ) } + + /// Wait for queued and recursively queued owner teardown after route + /// creators and callbacks have quiesced. Never call from an RPC callback. + #[cfg(feature = "test-util")] + #[must_use] + pub fn drain_retired_for_tests(timeout: std::time::Duration) -> bool { + retirement::drain(timeout) + } + + /// Number of owner bundles queued or being destroyed on the lifecycle worker. + #[cfg(feature = "test-util")] + #[must_use] + pub fn pending_retired_for_tests() -> usize { + retirement::pending() + } + + /// Finish queued native teardown after quiescing route creators and + /// callbacks and dropping their routes. Never call from an RPC callback. + #[must_use] + pub fn drain_retired(timeout: std::time::Duration) -> bool { + retirement::drain(timeout) && retirement::quiescent() + } } impl std::fmt::Debug for OwnedClientRoute { diff --git a/lib/crowdb-rpc/ffi/src/owned_route/retirement.rs b/lib/crowdb-rpc/ffi/src/owned_route/retirement.rs new file mode 100644 index 000000000..725555434 --- /dev/null +++ b/lib/crowdb-rpc/ffi/src/owned_route/retirement.rs @@ -0,0 +1,147 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Process-lifetime owner teardown, independent of Tokio runtime shutdown. +//! +//! Only the last clone of a route wrapper queues work. In-flight RPC lease +//! clones never use the channel. Teardown can synchronously complete other +//! RPCs, so neither producers nor the worker may wait for queue capacity. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{mpsc, Arc, OnceLock}; +use std::time::Instant; + +use crowdb_common::metrics::{self, Counter, Gauge, LatencyHistogram}; + +use super::RouteOwners; + +unsafe extern "C" { + fn atexit(callback: extern "C" fn()) -> std::ffi::c_int; +} + +static WORKER: OnceLock = OnceLock::new(); + +struct Task { + owners: RouteOwners, + queued_at: Instant, +} + +struct State { + live: AtomicUsize, + pending: AtomicUsize, + pending_metric: Arc, + completed: Arc, + latency: Arc, +} + +struct Retirement { + sender: mpsc::Sender, + state: Arc, +} + +fn worker() -> &'static Retirement { + WORKER.get_or_init(|| { + let (sender, receiver) = mpsc::channel::(); + let state = Arc::new(State { + live: AtomicUsize::new(0), + pending: AtomicUsize::new(0), + pending_metric: metrics::global_gauge("rpc.route.retirement.pending"), + completed: metrics::global_counter("rpc.route.retirement.completed"), + latency: metrics::global_histogram("rpc.route.retirement.lh"), + }); + let worker_state = Arc::clone(&state); + // The static sender remains admitted through the last resolver/callback + // release. No runtime shutdown or JoinHandle drop can close it early. + std::thread::Builder::new() + .name("rpc-route-retire".to_owned()) + .spawn(move || { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + for task in receiver { + drop(task.owners); + worker_state + .latency + .observe(task.queued_at.elapsed().as_nanos().try_into().unwrap_or(u64::MAX)); + worker_state.completed.inc(); + worker_state.pending_metric.dec(); + worker_state.live.fetch_sub(1, Ordering::Release); + // Include native joins and cascading retirements in the + // drain condition, rather than using a FIFO marker. + worker_state.pending.fetch_sub(1, Ordering::Release); + } + })); + if outcome.is_err() { + fatal("route retirement worker panicked"); + } + }) + .unwrap_or_else(|_| fatal("could not start route retirement worker")); + // Native logging globals are constructed before these route owners. + // Drain queued destruction before those globals are destroyed. + // SAFETY: this C callback never unwinds and uses process-lifetime state. + // It does not close admission for cascading owner releases. + if unsafe { atexit(shutdown) } != 0 { + fatal("could not register route retirement exit drain"); + } + Retirement { sender, state } + }) +} + +pub(super) fn register_owner() { + worker().state.live.fetch_add(1, Ordering::AcqRel); +} + +pub(super) fn retire(owners: RouteOwners) { + let worker = worker(); + worker.state.pending.fetch_add(1, Ordering::AcqRel); + worker.state.pending_metric.inc(); + if let Err(undelivered) = worker.sender.send(Task { + owners, + queued_at: Instant::now(), + }) { + // A dead receiver must never drop these owners on the enqueueing RPC + // thread. Retain them until abort instead of recreating a self-join. + std::mem::forget(undelivered); + fatal("route retirement receiver unexpectedly closed"); + } +} + +fn fatal(message: &str) -> ! { + eprintln!("critical: {message}; next step: inspect RPC lifecycle worker failure"); + std::process::abort(); +} + +pub(super) fn pending() -> usize { + WORKER + .get() + .map_or(0, |worker| worker.state.pending.load(Ordering::Acquire)) +} + +pub(super) fn drain(timeout: std::time::Duration) -> bool { + let deadline = Instant::now() + timeout; + while pending() != 0 { + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + true +} + +pub(super) fn quiescent() -> bool { + WORKER + .get() + .is_none_or(|worker| worker.state.live.load(Ordering::Acquire) == 0) +} + +pub(crate) extern "C" fn shutdown() { + // Explicit logging shutdown already requires quiescing callbacks and + // dropping instances first. Normal exit has also released Rust stack and + // runtime owners. Do not wait indefinitely for arbitrary live wrappers: + // process::exit intentionally leaves stack variables alive. Fail explicitly + // rather than run native static destructors under their active RPC workers. + if !drain(std::time::Duration::from_secs(30)) { + fatal("RPC route owner teardown did not drain before native shutdown"); + } + if !quiescent() { + fatal("live RPC route owners remain at native shutdown; quiesce and drop routes first"); + } +} diff --git a/lib/crowdb-rpc/ffi/tests/common/owned_route.rs b/lib/crowdb-rpc/ffi/tests/common/owned_route.rs new file mode 100644 index 000000000..fe7202ebc --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/common/owned_route.rs @@ -0,0 +1,184 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::ffi::c_void; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{mpsc, Arc}; +use std::thread::{self, ThreadId}; +use std::time::{Duration, Instant}; + +use arc_swap::ArcSwapOption; +use crowdb_protocol::fb::{ConnectionPingRequest, ConnectionPingRequestArgs}; +use crowdb_rpc_ffi::{sys, Buffer, OwnedClientRoute, RpcClient, RpcClientHandle, RpcServer}; +use flatbuffers::FlatBufferBuilder; + +#[derive(Clone, Copy)] +pub enum TestReply { + Success, + Close, + Timeout, +} + +struct TestClient { + rpc: Option, + destroyed: mpsc::Sender, +} + +impl RpcClientHandle for TestClient { + fn rpc_client_handle(&self) -> *mut c_void { + self.rpc.as_ref().unwrap().handle().cast() + } +} + +impl Drop for TestClient { + fn drop(&mut self) { + drop(self.rpc.take()); + let _ = self.destroyed.send(thread::current().id()); + } +} + +struct TestCompletion { + lease: OwnedClientRoute, + evicted: Arc, + completed: mpsc::Sender<(i32, ThreadId, bool)>, +} + +unsafe extern "C" fn complete( + _request: u64, + control: sys::crowdb_rpc_buffer_t, + data: sys::crowdb_rpc_buffer_t, + status: i32, + context: *mut c_void, +) { + // SAFETY: send transfers this unique box to exactly one completion. Both + // response handles also belong to this callback, including error outcomes. + let completion = unsafe { Box::from_raw(context.cast::()) }; + for buffer in [control, data] { + if !buffer.is_null() { + unsafe { sys::crowdb_rpc_buffer_release(buffer) }; + } + } + drop(completion.lease); + let _ = completion.completed.send(( + status, + thread::current().id(), + completion.evicted.load(Ordering::Acquire), + )); +} + +pub fn check_evicted_route(reply: TestReply) { + let rpc = RpcClient::new(); + rpc.set_completion_pool_size(8); + rpc.start_reaper(1_000_000_000, 10_000_000); + let remote = Arc::new(RpcServer::new(None)); + remote.listen("127.0.0.1", 0).unwrap(); + let release = Arc::new(AtomicBool::new(false)); + let (entered_tx, entered_rx) = mpsc::channel(); + remote.register_owned_handler(31011, { + let release = Arc::clone(&release); + move |request| { + let _ = entered_tx.send(()); + let deadline = Instant::now() + Duration::from_secs(3); + while !release.load(Ordering::Acquire) && Instant::now() < deadline { + thread::sleep(Duration::from_millis(1)); + } + match reply { + TestReply::Success => { + let _ = request.reply(control(), None, 31012); + } + TestReply::Close => request.close_connection(), + TestReply::Timeout => {} + } + } + }); + remote.start(); + let server = Arc::new(RpcServer::new(None)); + server.start(); + let connection = server.connect("127.0.0.1", remote.port()).unwrap(); + rpc.attach(&connection); + let (destroyed_tx, destroyed_rx) = mpsc::channel(); + let client = Arc::new(TestClient { + rpc: Some(rpc), + destroyed: destroyed_tx, + }); + let weak_client = Arc::downgrade(&client); + let weak_server = Arc::downgrade(&server); + let cache = ArcSwapOption::from(Some(Arc::new(OwnedClientRoute::new( + Arc::clone(&client), + Arc::clone(&server), + connection.clone(), + )))); + let (completed_tx, completed_rx) = mpsc::channel(); + let evicted = Arc::new(AtomicBool::new(false)); + let lease = cache.load_full().unwrap().as_ref().clone(); + let handles = lease.raw_handles(); + let completion = Box::into_raw(Box::new(TestCompletion { + lease, + evicted: Arc::clone(&evicted), + completed: completed_tx, + })); + // Model the native transport: its lease is the sole owner before submission. + // Setup cannot consume the request's timeout before cache eviction. + drop(cache.swap(None)); + drop(connection); + drop(client); + drop(server); + assert!(weak_client.upgrade().is_some()); + assert!(weak_server.upgrade().is_some()); + evicted.store(true, Ordering::Release); + // SAFETY: the unique completion lease retains all matching raw handles; + // success transfers the box to one callback, rejection leaves it to us. + let submitted = unsafe { + sys::crowdb_rpc_client_send( + handles.0.cast(), + handles.1.cast(), + handles.2.cast(), + 1, + control().into_raw(), + std::ptr::null_mut(), + 31011, + Some(complete), + completion.cast(), + ) + }; + if submitted != sys::CROWDB_RPC_OK { + // SAFETY: a rejected submission guarantees callback ownership was not + // transferred, so the caller must reclaim the original box. + drop(unsafe { Box::from_raw(completion) }); + } + assert_eq!(submitted, sys::CROWDB_RPC_OK); + entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + release.store(true, Ordering::Release); + let (status, callback_thread, after_eviction) = + completed_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + assert!(after_eviction, "completion ran before last-owner eviction"); + let expected = match reply { + TestReply::Success => sys::CROWDB_RPC_OK, + TestReply::Close => sys::CROWDB_RPC_ERR_CONN_CLOSED, + TestReply::Timeout => sys::CROWDB_RPC_ERR_TIMEOUT, + }; + assert_eq!(status, expected); + let destructor_thread = destroyed_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + assert_ne!(destructor_thread, callback_thread); + assert!(weak_client.upgrade().is_none()); + let deadline = Instant::now() + Duration::from_secs(3); + while weak_server.upgrade().is_some() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(1)); + } + assert!(weak_server.upgrade().is_none()); + assert!(OwnedClientRoute::drain_retired_for_tests(Duration::from_secs(3))); + remote.stop(); +} + +fn control() -> Buffer { + let mut builder = FlatBufferBuilder::new(); + let ping = ConnectionPingRequest::create( + &mut builder, + &ConnectionPingRequestArgs { + id: 1, + rpc_create_nano: 0, + }, + ); + builder.finish(ping, None); + Buffer::from_bytes(builder.finished_data()) +} diff --git a/lib/crowdb-rpc/ffi/tests/common/route_retirement.rs b/lib/crowdb-rpc/ffi/tests/common/route_retirement.rs new file mode 100644 index 000000000..962ad4301 --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/common/route_retirement.rs @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::ffi::c_void; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{mpsc, Arc}; +use std::time::{Duration, Instant}; + +use crowdb_rpc_ffi::{OwnedClientRoute, RpcClient, RpcClientHandle, RpcServer}; + +pub enum TestCleanup { + None, + ExitDelay, + Cascade(Option), + Block { + entered: mpsc::Sender<()>, + release: Arc, + }, +} + +pub struct TestReleaseGuard(pub Arc); + +impl Drop for TestReleaseGuard { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } +} + +struct TestClient { + rpc: Option, + cleanup: TestCleanup, + released: mpsc::Sender<()>, +} + +impl RpcClientHandle for TestClient { + fn rpc_client_handle(&self) -> *mut c_void { + self.rpc.as_ref().unwrap().handle().cast() + } +} + +impl Drop for TestClient { + fn drop(&mut self) { + match &mut self.cleanup { + TestCleanup::None => {} + TestCleanup::ExitDelay => { + std::thread::sleep(Duration::from_millis(100)); + eprintln!("ROUTE_RETIREMENT_EXIT_DRAINED"); + } + TestCleanup::Cascade(child) => drop(child.take()), + TestCleanup::Block { entered, release } => { + let _ = entered.send(()); + let deadline = Instant::now() + Duration::from_secs(3); + while !release.load(Ordering::Acquire) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(1)); + } + } + } + drop(self.rpc.take()); + let _ = self.released.send(()); + } +} + +pub fn route(cleanup: TestCleanup, released: mpsc::Sender<()>) -> OwnedClientRoute { + let server = Arc::new(RpcServer::new(None)); + server.listen("127.0.0.1", 0).unwrap(); + server.start(); + let connection = server.connect("127.0.0.1", server.port()).unwrap(); + let rpc = RpcClient::new(); + rpc.attach(&connection); + OwnedClientRoute::new( + Arc::new(TestClient { + rpc: Some(rpc), + cleanup, + released, + }), + server, + connection, + ) +} diff --git a/lib/crowdb-rpc/ffi/tests/owned_route_lifecycle_test.rs b/lib/crowdb-rpc/ffi/tests/owned_route_lifecycle_test.rs new file mode 100644 index 000000000..55af62e0c --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/owned_route_lifecycle_test.rs @@ -0,0 +1,22 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/owned_route.rs"] +mod owned_route; + +use owned_route::{check_evicted_route, TestReply}; + +#[test] +fn evicted_route_releases_last_owners_after_response_callback() { + check_evicted_route(TestReply::Success); +} + +#[test] +fn evicted_route_releases_last_owners_after_close_callback() { + check_evicted_route(TestReply::Close); +} + +#[test] +fn evicted_route_releases_last_owners_after_timeout_callback() { + check_evicted_route(TestReply::Timeout); +} diff --git a/lib/crowdb-rpc/ffi/tests/route_retirement_test.rs b/lib/crowdb-rpc/ffi/tests/route_retirement_test.rs new file mode 100644 index 000000000..dff49d2a7 --- /dev/null +++ b/lib/crowdb-rpc/ffi/tests/route_retirement_test.rs @@ -0,0 +1,111 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/route_retirement.rs"] +mod route_retirement; + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{mpsc, Arc}; +use std::time::Duration; + +use crowdb_rpc_ffi::OwnedClientRoute; +use route_retirement::{route, TestCleanup, TestReleaseGuard}; + +#[test] +fn normal_process_exit_drains_queued_owners_before_native_statics() { + const CHILD: &str = "CROWDB_ROUTE_EXIT_CHILD"; + if let Ok(mode) = std::env::var(CHILD) { + let (released, _receiver) = mpsc::channel(); + let route = route(TestCleanup::ExitDelay, released); + if mode == "live" { + std::mem::forget(route); + std::process::exit(0); + } + drop(route); + if mode == "logging" { + crowdb_rpc_ffi::shutdown_logging(); + } + return; + } + for mode in ["normal", "logging", "live"] { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "normal_process_exit_drains_queued_owners_before_native_statics", + "--nocapture", + ]) + .env(CHILD, mode) + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + if mode == "live" { + assert!(!output.status.success(), "{stdout}\n{stderr}"); + assert!( + stderr.contains("live RPC route owners remain at native shutdown"), + "{stdout}\n{stderr}" + ); + continue; + } + assert!(output.status.success(), "{stdout}\n{stderr}"); + assert!( + stderr.contains("ROUTE_RETIREMENT_EXIT_DRAINED"), + "{stdout}\n{stderr}" + ); + assert!(!stderr.contains("LOG ERROR"), "{stdout}\n{stderr}"); + } +} + +#[test] +fn request_clones_keep_one_owner_bundle_until_the_last_release() { + let (released_tx, released_rx) = mpsc::channel(); + let route = route(TestCleanup::None, released_tx); + let clones = vec![route.clone(); 256]; + let handles = route.raw_handles(); + assert!(clones.iter().all(|route| route.raw_handles() == handles)); + drop(clones); + assert!(released_rx.try_recv().is_err()); + assert!(route.is_open()); + drop(route); + released_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + assert!(OwnedClientRoute::drain_retired_for_tests(Duration::from_secs(3))); + assert!(released_rx.try_recv().is_err()); +} + +#[test] +fn cascading_retirement_drains_and_a_stalled_destructor_keeps_backlog_visible() { + let (released_tx, released_rx) = mpsc::channel(); + let (entered_tx, entered_rx) = mpsc::channel(); + let release = Arc::new(AtomicBool::new(false)); + let guard = TestReleaseGuard(Arc::clone(&release)); + let child = route( + TestCleanup::Block { + entered: entered_tx, + release: Arc::clone(&release), + }, + released_tx.clone(), + ); + let parent = route(TestCleanup::Cascade(Some(child)), released_tx.clone()); + drop(parent); + entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + assert!(OwnedClientRoute::pending_retired_for_tests() >= 1); + assert!(!OwnedClientRoute::drain_retired_for_tests(Duration::from_millis( + 10 + ))); + let tail = route(TestCleanup::None, released_tx); + let (queued_tx, queued_rx) = mpsc::channel(); + let producer = std::thread::spawn(move || { + drop(tail); + queued_tx.send(()).unwrap(); + }); + queued_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + assert!(OwnedClientRoute::pending_retired_for_tests() >= 2); + release.store(true, Ordering::Release); + drop(guard); + producer.join().unwrap(); + assert!(OwnedClientRoute::drain_retired_for_tests(Duration::from_secs(3))); + for _ in 0..3 { + released_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + } + assert!(released_rx.try_recv().is_err()); +} diff --git a/lib/crowdb-rpc/src/c_api.cpp b/lib/crowdb-rpc/src/c_api.cpp index f62655bf2..4823cee9a 100644 --- a/lib/crowdb-rpc/src/c_api.cpp +++ b/lib/crowdb-rpc/src/c_api.cpp @@ -443,6 +443,12 @@ crowdb_rpc_status crowdb_rpc_client_send(crowdb_rpc_client_t client, crowdb_rpc_ return CROWDB_RPC_ERR_INVALID_ARG; } + // Completion can synchronously destroy all caller-side wrappers. Keep + // their native owners through submission and never read wrappers after it. + const auto transport = server->server->shared_transport(); + const auto caller = client->client; + const auto connection = conn->conn; + crowdb::rpc::Buffer *ctrl_buf = control->buf; crowdb::rpc::Buffer *data_buf = (data != nullptr) ? data->buf : nullptr; @@ -454,8 +460,8 @@ crowdb_rpc_status crowdb_rpc_client_send(crowdb_rpc_client_t client, crowdb_rpc_ data_buf->ref_clone(); } - bool ok = client->client->send(server->server->transport(), conn->conn.get(), request_id, ctrl_buf, data_buf, - msg_type, on_complete, user_data); + bool ok = caller->send(transport.get(), connection.get(), request_id, ctrl_buf, data_buf, msg_type, on_complete, + user_data); // Release the caller's wrapper handles (decrements the ref bumped // above and frees the crowdb_rpc_buffer_s struct). @@ -464,7 +470,7 @@ crowdb_rpc_status crowdb_rpc_client_send(crowdb_rpc_client_t client, crowdb_rpc_ crowdb_rpc_buffer_release(data); } - return submit_status(ok, conn->conn.get()); + return submit_status(ok, connection.get()); } catch (...) { return CROWDB_RPC_ERR_CONN_ERROR; @@ -487,6 +493,10 @@ crowdb_rpc_status crowdb_rpc_client_send_chain(crowdb_rpc_client_t client, crowd return CROWDB_RPC_ERR_INVALID_ARG; } + const auto transport = server->server->shared_transport(); + const auto caller = client->client; + const auto connection = conn->conn; + std::array buffers{}; uint64_t total = 0; for (uint8_t index = 0; index < data_view_count; ++index) { @@ -504,14 +514,14 @@ crowdb_rpc_status crowdb_rpc_client_send_chain(crowdb_rpc_client_t client, crowd for (uint8_t index = 0; index < data_view_count; ++index) { buffers[index]->ref_clone(); } - bool ok = client->client->send_chain(server->server->transport(), conn->conn.get(), request_id, control->buf, - buffers.data(), data_view_count, msg_type, on_complete, user_data); + bool ok = caller->send_chain(transport.get(), connection.get(), request_id, control->buf, buffers.data(), + data_view_count, msg_type, on_complete, user_data); crowdb_rpc_buffer_release(control); for (uint8_t index = 0; index < data_view_count; ++index) { crowdb_rpc_buffer_release(data_views[index]); } - return submit_status(ok, conn->conn.get()); + return submit_status(ok, connection.get()); } catch (...) { return CROWDB_RPC_ERR_CONN_ERROR; @@ -528,6 +538,10 @@ crowdb_rpc_status crowdb_rpc_client_send_slab(crowdb_rpc_client_t client, crowdb return CROWDB_RPC_ERR_INVALID_ARG; } + const auto transport = server->server->shared_transport(); + const auto caller = client->client; + const auto connection = conn->conn; + crowdb::rpc::Buffer *ctrl_buf = control->buf; crowdb::rpc::Buffer *data_buf = (data != nullptr) ? data->buf : nullptr; if (ctrl_buf != nullptr) { @@ -537,13 +551,13 @@ crowdb_rpc_status crowdb_rpc_client_send_slab(crowdb_rpc_client_t client, crowdb data_buf->ref_clone(); } - bool ok = client->client->send_slab_only(server->server->transport(), conn->conn.get(), request_id, ctrl_buf, - data_buf, msg_type, on_complete, user_data); + bool ok = caller->send_slab_only(transport.get(), connection.get(), request_id, ctrl_buf, data_buf, msg_type, + on_complete, user_data); crowdb_rpc_buffer_release(control); if (data != nullptr) { crowdb_rpc_buffer_release(data); } - return submit_status(ok, conn->conn.get()); + return submit_status(ok, connection.get()); } catch (...) { return CROWDB_RPC_ERR_CONN_ERROR; diff --git a/lib/crowdb-rpc/tests/c_api_submit_lifetime_test.cpp b/lib/crowdb-rpc/tests/c_api_submit_lifetime_test.cpp new file mode 100644 index 000000000..7c8dcfc17 --- /dev/null +++ b/lib/crowdb-rpc/tests/c_api_submit_lifetime_test.cpp @@ -0,0 +1,124 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/c_api_internal.h" +#include "crowdb-rpc/transport/socket_transport.h" + +#include +#include +#include + +#include +#include + +namespace +{ +using namespace crowdb::rpc; + +enum class TestSubmit : uint8_t { Call, Chain, Slab }; + +struct TestCompletion +{ + crowdb_rpc_client_t client = nullptr; + crowdb_rpc_server_t server = nullptr; + crowdb_rpc_conn_t connection = nullptr; + std::shared_ptr transport; + uint32_t calls = 0; + bool returned = false; + + ~TestCompletion() + { + crowdb_rpc_client_destroy(client); + crowdb_rpc_server_destroy(server); + crowdb_rpc_conn_destroy(connection); + } +}; + +void complete(uint64_t /*request*/, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, crowdb_rpc_status status, + void *context) +{ + auto &state = *static_cast(context); + ++state.calls; + EXPECT_FALSE(state.returned); + EXPECT_EQ(status, CROWDB_RPC_ERR_CONN_CLOSED); + EXPECT_EQ(control, nullptr); + EXPECT_EQ(data, nullptr); + crowdb_rpc_client_destroy(state.client); + state.client = nullptr; + crowdb_rpc_server_destroy(state.server); + state.server = nullptr; + crowdb_rpc_conn_destroy(state.connection); + state.connection = nullptr; + // The fixture retains one native owner. The active ABI call must retain + // another, even after a synchronous callback destroys every C wrapper. + EXPECT_GE(state.transport.use_count(), 2); +} + +void check_submit(TestSubmit method, bool pooled) +{ + SystemBufferPool pool; + TestCompletion state; + state.server = new crowdb_rpc_server_s{new RpcServer(&pool)}; + state.transport = state.server->server->shared_transport(); + std::array sockets{}; + ASSERT_EQ(::socketpair(AF_UNIX, SOCK_STREAM, 0, sockets.data()), 0); + auto native_connection = state.transport->create_connection(sockets[0], "synchronous-close"); + state.connection = new crowdb_rpc_conn_s{native_connection}; + state.client = crowdb_rpc_client_create(); + ASSERT_NE(state.client, nullptr); + const auto native_client = state.client->client; + if (pooled) { + crowdb_rpc_client_set_completion_pool_size(state.client, 8); + } + crowdb_rpc_client_attach(state.client, state.connection); + const int original_write_fd = native_connection->write_fd; + // This descriptor is outside the process descriptor limit. The real + // direct-write path gets EBADF and synchronously completes on close. + native_connection->write_fd = std::numeric_limits::max(); + constexpr std::array payload{1, 2, 3, 4, 5, 6, 7}; + auto *const control = crowdb_rpc_buffer_create(payload.data(), payload.size()); + crowdb_rpc_status status = CROWDB_RPC_ERR_INVALID_ARG; + if (method == TestSubmit::Chain) { + const std::array data{crowdb_rpc_buffer_create(payload.data(), 3), + crowdb_rpc_buffer_create(payload.data() + 3, 3)}; + status = crowdb_rpc_client_send_chain(state.client, state.server, state.connection, 1, control, data.data(), + data.size(), 42, complete, &state); + } + else if (method == TestSubmit::Slab) { + status = crowdb_rpc_client_send_slab(state.client, state.server, state.connection, 1, control, nullptr, 42, + complete, &state); + } + else { + status = crowdb_rpc_client_send(state.client, state.server, state.connection, 1, control, nullptr, 42, complete, + &state); + } + state.returned = true; + native_connection->write_fd = original_write_fd; + EXPECT_EQ(status, CROWDB_RPC_OK); + EXPECT_EQ(state.calls, 1U); + EXPECT_EQ(native_client->pending_count(), 0U); + EXPECT_FALSE(native_connection->is_open()); + state.transport->stop(); + ::close(sockets[1]); +} +} // namespace + +TEST(CApiSubmitLifetimeTest, MapCompletionMayDestroyWrappersBeforeSendReturns) +{ + check_submit(TestSubmit::Call, false); +} + +TEST(CApiSubmitLifetimeTest, SlabCompletionMayDestroyWrappersBeforeSendReturns) +{ + check_submit(TestSubmit::Call, true); +} + +TEST(CApiSubmitLifetimeTest, ChainCompletionMayDestroyWrappersBeforeSendReturns) +{ + check_submit(TestSubmit::Chain, true); +} + +TEST(CApiSubmitLifetimeTest, SlabOnlyCompletionMayDestroyWrappersBeforeSendReturns) +{ + check_submit(TestSubmit::Slab, true); +} diff --git a/lib/crowdb-tree/ffi/src/tree.rs b/lib/crowdb-tree/ffi/src/tree.rs index 54e26573d..8f36a39d5 100644 --- a/lib/crowdb-tree/ffi/src/tree.rs +++ b/lib/crowdb-tree/ffi/src/tree.rs @@ -559,6 +559,11 @@ pub fn ct_flush_logging() { /// buffered log messages are written to disk. Safe to call when logging /// was never initialized (no-op in that case). pub fn ct_shutdown_logging() { + #[cfg(feature = "chunk-rpc")] + assert!( + crowdb_rpc_ffi::OwnedClientRoute::drain_retired(std::time::Duration::from_secs(30)), + "RPC route owner teardown did not drain before native logging shutdown" + ); unsafe { sys::ct_shutdown_logging() }; } diff --git a/tools/test-metrics/measure.py b/tools/test-metrics/measure.py index d3776c5aa..fb5a546d4 100644 --- a/tools/test-metrics/measure.py +++ b/tools/test-metrics/measure.py @@ -99,13 +99,14 @@ def main(): print(text, flush=True) raise SystemExit(f"{suite} succeeded without canonical container results") passed = sum(case["status"] == "pass" for case in cases) - failed = sum(case["status"] != "pass" for case in cases) + failed = sum(case["status"] not in ("pass", "not_executed") for case in cases) ignored, ignored_cases = 0, [] record = dict(suite=suite, environment=environment, passed=passed, ignored=ignored, ignored_cases=ignored_cases, failed=failed, seconds=elapsed, exit_code=result.returncode, log=str(log), run_started_at=run_date.isoformat()) if is_container: + record["not_executed"] = sum(case["status"] == "not_executed" for case in cases) record["scenario_seconds"] = sum(report["timings"].get("scenarios_wall", 0) for report in reports) record["container_artifacts"] = str(container_root) results.append(record) From 70ae86b1c62e095be2754ed178890736e49be5a4 Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 1 Nov 2026 18:56:16 +0800 Subject: [PATCH 13/24] Make CI fault and readiness checks respect lifecycle boundaries --- doc/working/plan-ci-stability-review.md | 35 ++++++++++++- doc/working/plan-physical-node-ci-handoff.md | 50 ++++++++++++++----- .../e2e/flows/01-shell-ui-behaviors.spec.ts | 19 +++++-- e2e-test/console_cli_target.py | 19 +++++-- e2e-test/tests/chunkdb/allocation.rs | 30 +++-------- e2e-test/tests/common.rs | 1 + .../common/rpc_response_proxy.rs | 2 +- e2e-test/tests/console.rs | 3 +- e2e-test/tests/console_cli_inventory_test.py | 33 ++++++++++++ 9 files changed, 146 insertions(+), 46 deletions(-) rename e2e-test/tests/{console => }/common/rpc_response_proxy.rs (98%) diff --git a/doc/working/plan-ci-stability-review.md b/doc/working/plan-ci-stability-review.md index 950990d78..a544aec3c 100644 --- a/doc/working/plan-ci-stability-review.md +++ b/doc/working/plan-ci-stability-review.md @@ -22,7 +22,8 @@ coverage, and identify the remaining work needed to improve CI stability. 2026-11-01 用户恢复执行。此前暂停于 GC 派发前；恢复后加强版 GC 三例、 辅助服务/Journal 三轮组合、Go SDK 三轮和 Dataset 分阶段验证均已通过。 -本地补充验证控制器正常退出，下一步是提交、推送和核对新 SHA 的完整 CI。 +本地补充验证控制器正常退出；首次推送后的三个新失败已修复并完成本地复验， +下一步是推送这些修复并核对新 SHA 的完整 CI。 物理容器在恢复前已停止，原因未知；七个容器、镜像和卷身份均保持不变。 ## 当前执行队列 @@ -58,9 +59,41 @@ coverage, and identify the remaining work needed to improve CI stability. guard 通过。新增断言均有独立实际运行证据，不能与旧 bundle 的全量混算。 - [~] **提交、推送并核对远端 CI**：CROWDB `task-fix-build` 一个一致提交； 记录精确 SHA、run/attempt 和全部展开 job，修复真实失败后推进。 + 已推送 `76fc6508189dd31018e953674a8f1c693011498d`，新 run `41295426521` + attempt 1 已结束：27 项中 24 成功、3 失败。CI merge checkout 与 head + 的完整 tree 一致。三个新失败的本地修复与受影响组复验均已通过，待推送 + 新 head 并重新核对全部 27 项。完整日志与三个校验过的 artifact 保留在 + `ci/published-76fc6508/`，不并入原 16 项人工 review。 - [ ] **恢复物理验证**：远端 CI 核对后继续 P13 和 P14–P19，保留 native 崩溃的新镜像验证及历史数据审计；当前物理资源和失败证据继续保留。 +## Run 41295426521：首次发布后的三个新失败 + +- **Chunk allocation 丢回复注入**：旧代理丢弃首段 TCP 数据，其中可能只有 + `0xfffe` capability handshake，allocation 尚未执行，busy bytes 仅为 1 MiB。 + 复用共享 frame-aware 代理，要求恰好转发一次握手、丢弃一次 application + response，保留 Unreachable、精确 2 MiB 和不重放 allocation 的断言。 + 原精确/默认 scope/串行基线均通过；该 scope 只有一个 case，不能据此声称 + 复现了多 case 竞争。修复精确场景通过；完整失败 job 的 47 个剩余场景 + 本地通过，保留三个未受影响的远端 PASS。 +- **Shell lazy inventory**：原 trace 在 N-3 选中后、DOM 断言返回前发生正常 + topology refresh；四次 inventory 均只请求 N-3，均返回 200。现按 topology + 观察周期精确断言 `[3]`，保留初始零发现、全部 DOM 断言和三秒期限。 + 原精确基线 4.2 秒，修复后 3.9 秒；原 shell/capacity 并发组合分别 + 4.8/10.8 秒通过。生产 UI 不变。 +- **CLI final readiness**：自动 split 发布 generation 2 后，旧 grant 短暂 + StaleCatalog，最终即时 check 返回 degraded；真实进程仍存活。最终 check + 现在仅使用启动时原 180 秒绝对期限的剩余量，调用一次官方 `--wait`，保留 + Ready、cluster identity 和整个等待前后的 lifecycle equality。过期不派发、 + 失败不重试、错 cluster 拒绝均有回归；不新增 mutation replay 或等待预算。 + 原精确和五例并发基线通过；修复后六例通过，包含被独立 fixture 首错中断的 + terminal writer recovery。该 peer 不是另一个已证实的产品失败。本地修复轮 + 没有复现原 StaleCatalog 窗口，不能以快速 PASS 声称复现了该时序。 +- **门禁与保留**：fixture 304 tests、fmt、full clippy、TypeScript、diff check + 全部通过。生产 source tree 仍为 `376fdc26a0e96d7810313ea670bf043c4f0dfcae`； + 正常 clients builder 已刷新对应输入。七个物理容器及卷身份未变；尚未进行 + 物理故障、升级或退休。完整执行证据见 `local-repair-closure.json`。 + ## 通过场景的稳定性改善 用户要求同时处理 PASS 但不稳定的场景。本节记录额外改善与待验证边界， diff --git a/doc/working/plan-physical-node-ci-handoff.md b/doc/working/plan-physical-node-ci-handoff.md index 351ee1b4d..cc38e48bb 100644 --- a/doc/working/plan-physical-node-ci-handoff.md +++ b/doc/working/plan-physical-node-ci-handoff.md @@ -11,9 +11,9 @@ Upstream: [Physical Node Test Plan](../dev/physical-node-test-plan.md), Goal: finish CI stability verification and publish CROWDB, then resolve P13 and complete the two-host physical recovery matrix without losing data or evidence. -## 1. Current checkpoint: local verification complete, publication next +## 1. Current checkpoint: published CI diagnosed, local repairs verified -Updated 2026-11-01, 17:56 Asia/Shanghai. The user explicitly resumed this plan. +Updated 2026-11-01, 18:56 Asia/Shanghai. The user explicitly resumed this plan. Continue the agreed CI refinement, commit/push, exact-head CI, then physical recovery sequence. No physical fault or retirement has been dispatched on resume. @@ -44,7 +44,23 @@ recovery sequence. No physical fault or retirement has been dispatched on resume - No test/workload/fault controller is left running. Physical work remains paused; the seven retained physical containers and their volumes remain owned and preserved. Do not restart old session IDs. -- No new commit or push has been made. No new CI run has been triggered. +- Commit 76fc6508189dd31018e953674a8f1c693011498d was pushed successfully. + [CI run 41295426521](https://github.com/buzzcrow/crowdb/actions/runs/41295426521), + attempt 1, finished with 24 successful and three failed jobs out of 27. + All job logs and the three failed-job artifacts are preserved under + ci/published-76fc6508/. Artifact sizes, SHA-256 and ZIP CRCs were verified. +- All three test-boundary repairs are locally verified: a frame-aware lost-reply + proxy, exact selected-node requests per topology cycle, and final CLI readiness + within the remaining original startup deadline. No production source changed. + The chunk job's 47 remaining cases passed, retaining three unaffected remote + passes. All six CLI cases, including the interrupted terminal-recovery peer, + and the original shell/capacity browser pair passed. Fixture 304, full clippy, + fmt and TypeScript passed; the closing scope guard preserved all seven nodes. + Next publish these coherent repairs and verify the new head's complete CI. +- CI checks out merge commit 66579652b16f798d8fd0211fe39002b0f7eae594. + Its complete source tree equals the pushed head's tree + 8039607dbf9901dc8fdcbd5e78b207ed10627140. Local and CI image runtime digests + differ; source equality does not claim identical binaries. - Two physical hosts have not completed final acceptance: P13 is still failed, P14–P19 are unrun, and the latest native lifecycle fix needs physical validation. @@ -55,15 +71,16 @@ the user finishes joint review; a PASS must not check a human review box. ## 2. Repository and publication boundary - CROWDB root: /cjdata/cpp/crowdb2. -- Branch: task-fix-build; HEAD: - 7a93c4e5fdca9bc39455331bfae9f3082e167a43. -- Four existing local commits are ahead of the remote branch: +- Branch: task-fix-build; published HEAD: + 76fc6508189dd31018e953674a8f1c693011498d. +- The four previously local commits were included in the fast-forward push: cdf023d6 (recovery/liveness), 81d07f03 (nearby mirrors), d396a220 (healthy leader quorum lease), and 7a93c4e5 (cleanup/handoff). -- The substantial current production/test/documentation changes are still - uncommitted. Preserve them and any new user edits; do not reset or restore. -- Remote task-fix-build was last verified through GitHub during this session at - 8885d2adc351d9c99f0219e18ecf0c4782bc3079. +- The production changes are committed and pushed. Three subsequent test fixes + and working-plan updates remain local until the next publication. Preserve + any new user edits; do not reset or restore. +- Remote task-fix-build was fetched at 8885d2a immediately before the push; + GitHub then confirmed the new exact head 76fc6508. [PR 55](https://github.com/buzzcrow/crowdb/pull/55) is open. Fetch and verify remote state again immediately before the eventual push. - After explicit resume, the agreed order remains: finish local CI stability @@ -101,6 +118,9 @@ All abbreviated paths below are relative to: passed after the final test-only dependency-recovery predicate change. - ci/stability-final/scope-guard-before.json and scope-guard-latest.json: exact retained A3/B4 container, image, running-state and volume identities. +- ci/published-76fc6508/: all 27 job logs, failed artifacts, repair-diagnosis.json, + repair-queue.json, local-repair-closure.json and exact/original-pair results. + These three newly published failures are separate from the historical 16 jobs. - The earlier long handoff is preserved at handoff-history/plan-physical-node-ci-handoff-before-rewrite-20261101T062936Z.md. It contains superseded historical instructions, not current authorization. @@ -186,7 +206,7 @@ Final local server image: The runs are 41003611925 (8/27 failed), 41128190276 (5/27), and 41159636766 (3/27 at head 8885d2a). All 16 joint review boxes remain unchecked. -Main uncommitted source ownership: +Main source ownership in the published change: - RPC/C ABI: lib/crowdb-rpc/ffi/src/owned_route* and logging.rs, lib/crowdb-rpc/src/c_api.cpp, lib/crowdb-tree/ffi/src/tree.rs, and tests. @@ -259,12 +279,16 @@ The user has resumed; proceed in the order below. failure occurrences and three newer cases. Client binary hashes match the frozen build manifest; browser/Go/Dataset source digests agree. Stability TODO updated; all human review boxes remain unchecked. -- [~] **Commit and push CROWDB only**: run final relevant gates/diff check, +- [x] **Commit and push CROWDB only**: final relevant gates and staged diff + check passed; the pre-commit hook independently passed fmt/clippy. Created + 76fc6508 and fast-forward pushed after fetching and verifying the old remote. + The publication receipt is ci/stability-final/publication.json. Required rules: inspect staged contents, and make one coherent commit with a single-line subject, no body/trailers/doc references/requirement numbers. Fetch and verify the remote before the authorized fast-forward push; do not force-push. No Sirius-specific pre-push rules apply to this repository. -- [ ] **Inspect new exact-head CI**: PR synchronization should trigger CI. +- [~] **Inspect new exact-head CI**: run 41295426521 attempt 1 ended 24 PASS / 3 FAIL. + The three fixes and local affected regressions passed; commit/push them next. Match the actual new head SHA/run/attempt, inspect every expanded job (the three historical runs had 27), preserve every first failure and artifact, fix real errors and rerun the appropriate new-head job/run. Do not rerun an old diff --git a/e2e-test/clients/console-ui/e2e/flows/01-shell-ui-behaviors.spec.ts b/e2e-test/clients/console-ui/e2e/flows/01-shell-ui-behaviors.spec.ts index f420d8fa5..8da45850f 100644 --- a/e2e-test/clients/console-ui/e2e/flows/01-shell-ui-behaviors.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/01-shell-ui-behaviors.spec.ts @@ -24,9 +24,19 @@ test('shell defaults and tree interactions distinguish live and stopped release const store = await step('shell Store mutation response', () => request.post('/api/stores', { data: { store_id: 207, nodes: [1, 2] } })); expect(store.status(), await store.text()).toBe(201); const diskInventoryRequests: number[] = []; + const diskInventoryCycles = new Map(); + let topologyRefresh = 0; page.on('request', request => { - const match = new URL(request.url()).pathname.match(/^\/api\/nodes\/(\d+)\/disk-groups$/); - if (match) diskInventoryRequests.push(Number(match[1])); + const path = new URL(request.url()).pathname; + if (path === '/api/racks') topologyRefresh++; + const match = path.match(/^\/api\/nodes\/(\d+)\/disk-groups$/); + if (match) { + const nodeId = Number(match[1]); + diskInventoryRequests.push(nodeId); + const cycle = diskInventoryCycles.get(topologyRefresh) ?? []; + cycle.push(nodeId); + diskInventoryCycles.set(topologyRefresh, cycle); + } }); await step('shell DOM setup', () => page.goto('/?domain=KV')); const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); @@ -108,7 +118,10 @@ test('shell defaults and tree interactions distinguish live and stopped release const node = sidebar.getByTestId('tree-node-N-3').locator(':scope > [role="treeitem"]'); await node.getByRole('button', { name: 'N-3', exact: true }).click(); await expect(node).toHaveAttribute('aria-selected', 'true'); - await expect.poll(() => diskInventoryRequests, { intervals: [100] }).toEqual([3]); + await expect.poll(() => diskInventoryRequests.length, { intervals: [100] }).toBeGreaterThan(0); + // A topology poll can refresh the selected branch before the DOM assertion + // returns. Each observation cycle still requests exactly this one node once. + for (const cycle of diskInventoryCycles.values()) expect(cycle).toEqual([3]); const storage = sidebar.getByTestId('tree-node-STORAGE-3').locator(':scope > [role="treeitem"]'); await expect(storage).toHaveAttribute('aria-expanded', 'false'); await expect(page.getByRole('region', { name: 'Topology', exact: true })).toBeVisible(); diff --git a/e2e-test/console_cli_target.py b/e2e-test/console_cli_target.py index 851854369..5b0a4002f 100644 --- a/e2e-test/console_cli_target.py +++ b/e2e-test/console_cli_target.py @@ -3,6 +3,7 @@ """Attach a host CLI to ordinary Docker nodes without a launch receipt.""" import json +import time from pathlib import Path from allocation import create_network, node_data, start_node @@ -48,6 +49,7 @@ def start(self): for name in self.servers: if not json.loads(self.command("inspect", name))[0]["State"]["Running"]: raise AssertionError("plain Docker node exited: " + self.command("logs", name)) + self.ready_deadline = time.monotonic() + 180 report = json.loads(self.invoke("cluster", "check", "--wait", "180", "--json", endpoint=self.endpoint(0))) assert report["phase"] == "ready", report @@ -116,6 +118,17 @@ def inventory(self): "processes": sorted((node, service, *identity) for (node, service), identity in processes.items()), "registrations": sorted(registrations, key=lambda item: item["id"])} + def check_ready(self, endpoint=None): + # Automatic catalog publication can temporarily fence an older serving + # grant. Use the original startup deadline for the final lifecycle gate; + # the operation does not receive another readiness budget or a replay. + remaining = int(self.ready_deadline - time.monotonic()) + assert remaining > 0, "CLI flow exhausted its original readiness deadline" + report = json.loads(self.invoke("cluster", "check", "--wait", str(remaining), "--json", + endpoint=endpoint)) + assert report["phase"] == "ready" and report["observation"]["cluster_id"] == self.cluster_id, report + return report + def replay(self, checkpoint): if checkpoint != "attachment": before = self.inventory() @@ -124,9 +137,8 @@ def replay(self, checkpoint): "logical": console_cli_operations.logical, "data": console_cli_operations.data}[checkpoint] operation(self) + self.check_ready() assert self.inventory() == before, "logical CLI operation changed runtime lifecycle" - report = json.loads(self.invoke("cluster", "check", "--json")) - assert report["phase"] == "ready" and report["observation"]["cluster_id"] == self.cluster_id (self.artifacts / "operation-proof.json").write_text(json.dumps({ "cluster_id": self.cluster_id, "operation": checkpoint, "inventory": before}, indent=2)) print("PASS selected-cluster CLI", checkpoint, flush=True) @@ -154,8 +166,7 @@ def attachment(self): assert self.invoke("s3", "object", "get", "cli-attach", "witness", endpoint=self.endpoint(index)) == value self.invoke("cluster", "stop", endpoint=self.endpoint(0), expected_exit=2) - report = json.loads(self.invoke("cluster", "check", "--json", endpoint=self.endpoint(2))) - assert report["phase"] == "ready" and report["observation"]["cluster_id"] == self.cluster_id + self.check_ready(endpoint=self.endpoint(2)) assert self.inventory() == before, "attachment changed container or service lifecycle" assert not (self.root / "persistent/oci-clusters").exists(), "attachment created a runtime receipt" (self.artifacts / "attachment-proof.json").write_text(json.dumps({ diff --git a/e2e-test/tests/chunkdb/allocation.rs b/e2e-test/tests/chunkdb/allocation.rs index 7595c3e7b..6e2dab7e9 100644 --- a/e2e-test/tests/chunkdb/allocation.rs +++ b/e2e-test/tests/chunkdb/allocation.rs @@ -3,6 +3,9 @@ use crowdb_chunkdb::allocator::DiskdbClientPool; use crowdb_kv_client::ServiceRegistryClient; +use std::sync::atomic::Ordering; + +use crate::common::rpc_response_proxy::TestResponseProxy; #[tokio::test] #[ignore = "requires the owned Diskdb container fixture"] @@ -10,7 +13,6 @@ async fn allocation_connect_failure_reroutes_but_lost_reply_does_not() { use crate::common::hardware::{DG_ID, UNIT_SIZE_BYTES}; use crowdb_diskdb_client::{DiskdbClientError, DiskdbRpcTransport}; use crowdb_protocol::common::ChunkId; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; let pool = DiskdbClientPool::new(ServiceRegistryClient::new(crate::common::hardware::kv_client())); pool.refresh_endpoints().await.expect("discover live owner"); @@ -34,26 +36,9 @@ async fn allocation_connect_failure_reroutes_but_lost_reply_does_not() { // The proxy forwards the allocation, then closes without its response. // Group-0 still advertises the direct endpoint, so an unsafe replay would // allocate another set there. - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let proxy = listener.local_addr().unwrap().to_string(); - let upstream_endpoint = endpoint.clone(); - let forward = tokio::spawn(async move { - let (downstream, _) = listener.accept().await.unwrap(); - let upstream = tokio::net::TcpStream::connect(upstream_endpoint).await.unwrap(); - let (mut input, mut output) = downstream.into_split(); - let (mut responses, mut requests) = upstream.into_split(); - let send = tokio::io::copy(&mut input, &mut requests); - let discard = async { - let mut buffer = [0; 8192]; - assert!(responses.read(&mut buffer).await.unwrap() > 0); - output.shutdown().await.unwrap(); - }; - tokio::select! { - _ = send => panic!("request stream ended before response"), - () = discard => {} - } - }); - pool.replace_endpoints_for_tests(vec![(DG_ID, proxy)]); + let proxy = TestResponseProxy::start(endpoint.clone()).await; + proxy.armed.store(true, Ordering::SeqCst); + pool.replace_endpoints_for_tests(vec![(DG_ID, proxy.endpoint.clone())]); let result = pool .allocate_blocks_reusing_disks(DG_ID, 1, 1, &ChunkId { high: 0, low: 42 }) .await; @@ -61,7 +46,8 @@ async fn allocation_connect_failure_reroutes_but_lost_reply_does_not() { matches!(result, Err(DiskdbClientError::Unreachable(_))), "{result:?}" ); - forward.await.unwrap(); + assert_eq!(proxy.forwarded_handshakes.load(Ordering::SeqCst), 1); + assert_eq!(proxy.dropped.load(Ordering::SeqCst), 1); let transport = DiskdbRpcTransport::new(); let info = transport diff --git a/e2e-test/tests/common.rs b/e2e-test/tests/common.rs index ff6461a39..e66cfc8bf 100644 --- a/e2e-test/tests/common.rs +++ b/e2e-test/tests/common.rs @@ -6,6 +6,7 @@ pub(crate) mod hardware; mod logging; pub(crate) mod management; pub(crate) mod native_s3; +pub(crate) mod rpc_response_proxy; mod test_client; use bytes::Bytes; diff --git a/e2e-test/tests/console/common/rpc_response_proxy.rs b/e2e-test/tests/common/rpc_response_proxy.rs similarity index 98% rename from e2e-test/tests/console/common/rpc_response_proxy.rs rename to e2e-test/tests/common/rpc_response_proxy.rs index 86eb0b7c1..d8743a606 100644 --- a/e2e-test/tests/console/common/rpc_response_proxy.rs +++ b/e2e-test/tests/common/rpc_response_proxy.rs @@ -125,7 +125,7 @@ async fn forward( return; } // Capability advertisements precede application replies and do not - // establish that the conditional write has completed. + // establish that the application request has completed. let handshake = message_type == 0xfffe; if !handshake && armed.swap(false, Ordering::SeqCst) { dropped.fetch_add(1, Ordering::SeqCst); diff --git a/e2e-test/tests/console.rs b/e2e-test/tests/console.rs index 8640cf481..f85c29bc6 100644 --- a/e2e-test/tests/console.rs +++ b/e2e-test/tests/console.rs @@ -7,8 +7,7 @@ mod group_publication; mod hardware; mod logical_publication; -#[path = "console/common/rpc_response_proxy.rs"] -mod rpc_response_proxy; +use crate::common::rpc_response_proxy; #[path = "console/common/bootstrap_authority.rs"] mod bootstrap_authority; diff --git a/e2e-test/tests/console_cli_inventory_test.py b/e2e-test/tests/console_cli_inventory_test.py index f2ac76b72..d3e69af4b 100644 --- a/e2e-test/tests/console_cli_inventory_test.py +++ b/e2e-test/tests/console_cli_inventory_test.py @@ -87,5 +87,38 @@ def test_old_positive_status_does_not_hide_process_exit_or_missing_web(self): self.cluster.inventory() +class AttachedReadinessTest(unittest.TestCase): + def setUp(self): + self.cluster = object.__new__(CliAttachedCluster) + self.cluster.cluster_id = "cluster" + self.cluster.ready_deadline = 280 + self.cluster.invoke = Mock(return_value=json.dumps({ + "phase": "ready", "observation": {"cluster_id": "cluster"}})) + + def test_final_check_uses_only_the_remaining_startup_budget(self): + with patch("console_cli_target.time.monotonic", return_value=245.5): + self.cluster.check_ready(endpoint="http://node3:9090") + self.cluster.invoke.assert_called_once_with( + "cluster", "check", "--wait", "34", "--json", endpoint="http://node3:9090") + + def test_expired_budget_does_not_dispatch_another_check(self): + with patch("console_cli_target.time.monotonic", return_value=280), \ + self.assertRaisesRegex(AssertionError, "original readiness deadline"): + self.cluster.check_ready() + self.cluster.invoke.assert_not_called() + + def test_check_failure_is_not_retried_and_wrong_cluster_is_rejected(self): + self.cluster.invoke.side_effect = AssertionError("degraded") + with patch("console_cli_target.time.monotonic", return_value=250): + with self.assertRaisesRegex(AssertionError, "degraded"): + self.cluster.check_ready() + self.cluster.invoke.assert_called_once() + self.cluster.invoke.side_effect = None + self.cluster.invoke.return_value = json.dumps({ + "phase": "ready", "observation": {"cluster_id": "foreign"}}) + with self.assertRaises(AssertionError): + self.cluster.check_ready() + + if __name__ == "__main__": unittest.main() From 5af2737fd5797a1afd1395eac63427e8fdb892d2 Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 1 Nov 2026 20:03:18 +0800 Subject: [PATCH 14/24] Preserve split materialization after lease recovery --- .../domain_monitor/chunk_kv/catalog.rs | 52 +++- .../tests/common/catalog_publication.rs | 4 +- .../tests/common/split_materialization.rs | 270 ++++++++++++++++++ .../tests/domain_monitor_test.rs | 2 + doc/working/plan-ci-stability-review.md | 33 ++- doc/working/plan-physical-node-ci-handoff.md | 84 ++++-- 6 files changed, 417 insertions(+), 28 deletions(-) create mode 100644 app/crowdb-kv-server/tests/common/split_materialization.rs diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs index 065965e33..3d51546b8 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs @@ -10,7 +10,9 @@ use crowdb_protocol::chunk_kv::{ ChunkKvRangeCatalogPageRef, ChunkKvRangeCatalogPartitionState, SplitPhase, SplitTransition, TransferPhase, TransferTransition, }; -use crowdb_protocol::key::{ChunkKvRangeCatalogHeadKey, ChunkKvRangeCatalogPageKey, TextKey}; +use crowdb_protocol::key::{ + ChunkKvRangeCatalogHeadKey, ChunkKvRangeCatalogPageKey, ChunkKvSplitKey, TextKey, +}; use crate::group0_control_plane::Group0ControlPlane; @@ -160,10 +162,54 @@ pub async fn publish_materialized_partition( let transition_id = current .transition_id .ok_or_else(|| "materialized split child has no transition identity".to_string())?; + validate_materialized_split(control, &catalog.pages, transition_id).await?; let (head, pages) = release_materialized_split(catalog.head, catalog.pages, transition_id, partition_id)?; publish(control, head, pages, catalog.head_revision).await } +async fn validate_materialized_split( + control: &Group0ControlPlane, + pages: &[ChunkKvRangeCatalogPage], + transition_id: crowdb_protocol::chunk_kv::Id128, +) -> Result<(), String> { + let path = ChunkKvSplitKey { transition_id }.to_path(); + let read = control + .get(path.as_bytes()) + .await + .map_err(|error| error.to_string())?; + let bytes = read.value.ok_or("materialized split transition is absent")?; + let transition: SplitTransition = serde_json::from_slice(&bytes).map_err(|error| error.to_string())?; + transition.validate().map_err(|error| error.to_string())?; + if transition.transition_id != transition_id + || !matches!( + transition.phase, + SplitPhase::ChildPrepared | SplitPhase::CatalogCommitted + ) + { + return Err("materialized split transition is not published".into()); + } + let desired = split_entries(&transition); + for entry in pages.iter().flat_map(|page| &page.entries) { + if entry.transition_id == Some(transition_id) + && !desired + .iter() + .any(|wanted| published_assignment_matches(entry, wanted)) + { + return Err("materialized split assignment differs from its published transition".into()); + } + } + // Once the split commit is durable, lease-excluded recovery may supersede + // either writer independently. Only the remaining exact split assignments + // own this overlay and marker; a successor assignment must stay untouched. + // Before that status write, both exact successors must still prove publication. + if transition.phase != SplitPhase::CatalogCommitted + && !publication_complete(pages, &desired, transition_id)? + { + return Err("materialized split must retain both uncommitted published assignments".into()); + } + Ok(()) +} + pub async fn publish_materialized_transfer( control: &Group0ControlPlane, partition_id: crowdb_protocol::chunk_kv::Id128, @@ -274,8 +320,8 @@ fn release_materialized_split( page.generation = generation; } } - if matched != 2 || !partition_matched { - return Err("materialized split must match one partition in an exact writer pair".into()); + if !(1..=2).contains(&matched) || !partition_matched { + return Err("materialized split must match a remaining published partition".into()); } let all_materialized = pages .iter() diff --git a/app/crowdb-kv-server/tests/common/catalog_publication.rs b/app/crowdb-kv-server/tests/common/catalog_publication.rs index 89f108060..5599d7a60 100644 --- a/app/crowdb-kv-server/tests/common/catalog_publication.rs +++ b/app/crowdb-kv-server/tests/common/catalog_publication.rs @@ -169,7 +169,7 @@ impl TestPublication { } } -async fn replace_json(control: &Group0ControlPlane, key: String, value: &T) { +pub(super) async fn replace_json(control: &Group0ControlPlane, key: String, value: &T) { let current = control.get(key.as_bytes()).await.unwrap(); control .compare_and_put( @@ -227,7 +227,7 @@ async fn publish_materialized_catalog( head } -async fn tick(control: &Group0ControlPlane) -> Result<(), String> { +pub(super) async fn tick(control: &Group0ControlPlane) -> Result<(), String> { let mut policy = descriptor(); policy.domain = "chunk-kv".into(); policy.service_registry_name = "chunk-kv".into(); diff --git a/app/crowdb-kv-server/tests/common/split_materialization.rs b/app/crowdb-kv-server/tests/common/split_materialization.rs new file mode 100644 index 000000000..e02e1a7f2 --- /dev/null +++ b/app/crowdb-kv-server/tests/common/split_materialization.rs @@ -0,0 +1,270 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::catalog_publication::{replace_json, tick}; +use super::*; + +struct TestMaterialization { + control: Group0ControlPlane, + split: SplitTransition, + head: ChunkKvRangeCatalogHead, + page: ChunkKvRangeCatalogPage, +} + +#[tokio::test] +async fn child_materializes_before_the_split_commit_status_is_persisted() { + let (control, prepared) = assert_chunk_kv_split_plan(2, u64::MAX).await; + replace_json( + &control, + ChunkKvSplitKey { + transition_id: prepared.transition_id, + } + .to_path(), + &prepared, + ) + .await; + test_process::TestProcessLease::seed(&control, 1).await; + let (before, original) = current_catalog(&control).await; + observe_partition(&control, &original.entries[1], true).await; + tick(&control).await.unwrap(); + let (head, page) = current_catalog(&control).await; + assert_eq!(head.generation, before.generation + 1); + for (mut expected, actual) in original.entries.into_iter().zip(page.entries) { + expected.artifact.tail_overlay = None; + expected.transition_id = None; + assert_eq!(actual, expected); + } + let committed: SplitTransition = get_json( + &control, + &ChunkKvSplitKey { + transition_id: prepared.transition_id, + } + .to_path(), + ) + .await; + assert_eq!(committed.phase, SplitPhase::CatalogCommitted); +} + +#[tokio::test] +async fn transferred_split_parent_does_not_block_child_materialization_or_grants() { + let fixture = TestMaterialization::transferred_parent().await; + let parent = fixture.page.entries[0].clone(); + fixture.observe_child().await; + tick(&fixture.control).await.unwrap(); + let (head, page) = current_catalog(&fixture.control).await; + assert_eq!(head.generation, fixture.head.generation + 1); + assert_eq!(page.entries[0], parent); + let mut child = fixture.page.entries[1].clone(); + child.artifact.tail_overlay = None; + child.transition_id = None; + assert_eq!(page.entries[1], child); + for entry in &page.entries { + let grant: ServingGrant = get_json( + &fixture.control, + &ServingGrantKey { + instance_id: entry.owner.instance_id, + } + .to_path(), + ) + .await; + assert_eq!(grant.catalog_generation, head.generation); + assert_eq!(grant.assignments.len(), 1); + assert_eq!(grant.assignments[0].partition_id, entry.partition_id); + assert_eq!(grant.assignments[0].owner_epoch, entry.owner_epoch); + } + tick(&fixture.control).await.unwrap(); + assert_eq!(current_catalog(&fixture.control).await.0, head); +} + +#[tokio::test] +async fn split_materialization_rejects_changed_assignment_or_reused_marker() { + for mismatch in 0..3 { + let mut fixture = TestMaterialization::transferred_parent().await; + match mismatch { + 0 => fixture.page.entries[1].owner_epoch += 1, + 1 => fixture.page.entries[1].artifact.tree_id += 1, + 2 => fixture.page.entries[0].transition_id = Some(fixture.split.transition_id), + _ => unreachable!(), + } + fixture.publish_page().await; + fixture.observe_child().await; + let error = tick(&fixture.control) + .await + .expect_err("changed split identity must be rejected"); + assert!(error.contains("materialized split"), "{error}"); + assert_eq!(current_catalog(&fixture.control).await.0, fixture.head); + } +} + +#[tokio::test] +async fn uncommitted_split_still_requires_both_published_assignments() { + let fixture = TestMaterialization::transferred_parent().await; + replace_json( + &fixture.control, + ChunkKvSplitKey { + transition_id: fixture.split.transition_id, + } + .to_path(), + &fixture.split, + ) + .await; + assert_eq!(fixture.split.phase, SplitPhase::ChildPrepared); + fixture.observe_child().await; + assert!(tick(&fixture.control).await.is_err()); + assert_eq!(current_catalog(&fixture.control).await.0, fixture.head); +} + +impl TestMaterialization { + async fn transferred_parent() -> Self { + let (control, split) = assert_chunk_kv_split_plan(2, u64::MAX).await; + let (_, page) = current_catalog(&control).await; + let parent = &page.entries[0]; + let target = OwnerDescriptor { + instance_id: 2, + rpc_endpoint: "127.0.0.1:17002".into(), + }; + let proof = TargetReadinessProof { + target_instance_id: target.instance_id, + target_epoch: parent.owner_epoch + 1, + artifact: parent.artifact.clone(), + durable_tail: 11, + }; + let transfer = TransferTransition { + transition_id: Id128 { high: 97, low: 98 }, + partition_id: parent.partition_id, + range: parent.range.clone(), + source: parent.owner.clone(), + source_epoch: parent.owner_epoch, + target: target.clone(), + target_epoch: proof.target_epoch, + artifact: parent.artifact.clone(), + target_artifact: parent.artifact.clone(), + readiness_limits: TransferReadinessLimits { + max_tail_records: 100, + max_tail_bytes: 1_000_000, + max_estimated_catchup_ms: 1_000, + prepare_deadline_ms: 1_000, + forwarding_grace_ms: 1_000, + }, + planned_at_ms: 0, + old_grant_expires_at_ms: 1, + phase: TransferPhase::TargetReady, + release_proof: Some(AuthorityReleaseProof::LeaseExpired { + activation_not_before_ms: 2, + }), + readiness_proof: Some(proof.clone()), + catchup_proof: Some(proof), + failure: None, + }; + transfer.validate().unwrap(); + put_json( + &control, + ChunkKvTransferKey { + transition_id: transfer.transition_id, + } + .to_path(), + &transfer, + ) + .await; + for instance_id in [1, 2] { + test_process::TestProcessLease::seed(&control, instance_id).await; + } + let mut moved = parent.clone(); + moved.owner = target; + moved.owner_epoch = transfer.target_epoch; + observe_partition(&control, &moved, false).await; + observe_partition(&control, &page.entries[1], false).await; + tick(&control).await.unwrap(); + let (head, page) = current_catalog(&control).await; + assert_eq!(page.entries[0].transition_id, Some(transfer.transition_id)); + assert_eq!(page.entries[1].transition_id, Some(split.transition_id)); + assert_eq!(page.entries[0].owner_epoch, transfer.target_epoch); + Self { + control, + split, + head, + page, + } + } + + async fn observe_child(&self) { + observe_partition(&self.control, &self.page.entries[1], true).await; + } + + async fn publish_page(&mut self) { + self.page.generation += 1; + self.page.seal().unwrap(); + self.head.previous_generation = Some(self.head.generation); + self.head.generation = self.page.generation; + self.head.pages[0].page_generation = self.page.generation; + self.head.pages[0].page_checksum = self.page.checksum; + self.head.seal().unwrap(); + self.head + .validate_pages(std::slice::from_ref(&self.page)) + .unwrap(); + put_json( + &self.control, + ChunkKvRangeCatalogPageKey { + generation: self.page.generation, + page_index: 0, + } + .to_path(), + &self.page, + ) + .await; + replace_json(&self.control, ChunkKvRangeCatalogHeadKey.to_path(), &self.head).await; + } +} + +async fn observe_partition( + control: &Group0ControlPlane, + entry: &ChunkKvRangeCatalogEntry, + independent: bool, +) { + let instance = InstanceValue { + instance_id: entry.owner.instance_id, + rpc_endpoint: entry.owner.rpc_endpoint.clone(), + last_heartbeat_ms: u64::MAX, + extra: Some(ServiceExtra { + chunk_kv: Some(ChunkKvExtra { + process_epoch: 1, + hosted: vec![HostedPartition { + partition_id: entry.partition_id, + owner_epoch: entry.owner_epoch, + recovering: false, + }], + partition_loads: vec![ChunkKvPartitionLoad { + partition_id: entry.partition_id, + independently_recoverable: independent, + ..ChunkKvPartitionLoad::default() + }], + ..ChunkKvExtra::default() + }), + ..ServiceExtra::default() + }), + }; + replace_json( + control, + InstanceKey { + service: "chunk-kv".into(), + instance_id: instance.instance_id, + } + .to_path(), + &instance, + ) + .await; +} + +async fn current_catalog(control: &Group0ControlPlane) -> (ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage) { + let head: ChunkKvRangeCatalogHead = get_json(control, &ChunkKvRangeCatalogHeadKey.to_path()).await; + let page = get_json( + control, + &ChunkKvRangeCatalogPageKey { + generation: head.pages[0].page_generation, + page_index: 0, + } + .to_path(), + ) + .await; + (head, page) +} diff --git a/app/crowdb-kv-server/tests/domain_monitor_test.rs b/app/crowdb-kv-server/tests/domain_monitor_test.rs index cea9ed61a..8553aae15 100644 --- a/app/crowdb-kv-server/tests/domain_monitor_test.rs +++ b/app/crowdb-kv-server/tests/domain_monitor_test.rs @@ -38,6 +38,8 @@ mod test_process; mod catalog_publication; #[path = "common/endpoint_publication.rs"] mod endpoint_publication; +#[path = "common/split_materialization.rs"] +mod split_materialization; struct CountingDriver { ticks: Arc, diff --git a/doc/working/plan-ci-stability-review.md b/doc/working/plan-ci-stability-review.md index a544aec3c..fa79df5c6 100644 --- a/doc/working/plan-ci-stability-review.md +++ b/doc/working/plan-ci-stability-review.md @@ -23,7 +23,9 @@ coverage, and identify the remaining work needed to improve CI stability. 2026-11-01 用户恢复执行。此前暂停于 GC 派发前；恢复后加强版 GC 三例、 辅助服务/Journal 三轮组合、Go SDK 三轮和 Dataset 分阶段验证均已通过。 本地补充验证控制器正常退出；首次推送后的三个新失败已修复并完成本地复验， -下一步是推送这些修复并核对新 SHA 的完整 CI。 +修复已推送为 `70ae86b1`，完整 CI 为 26/27 PASS；新发现的 terminal writer +恢复故障已定位为 split 物化与父分区租约转移的冲突，修复和本地复验已完成， +待提交推送并核对下一轮精确 SHA CI。 物理容器在恢复前已停止，原因未知；七个容器、镜像和卷身份均保持不变。 ## 当前执行队列 @@ -61,8 +63,11 @@ coverage, and identify the remaining work needed to improve CI stability. 记录精确 SHA、run/attempt 和全部展开 job，修复真实失败后推进。 已推送 `76fc6508189dd31018e953674a8f1c693011498d`，新 run `41295426521` attempt 1 已结束：27 项中 24 成功、3 失败。CI merge checkout 与 head - 的完整 tree 一致。三个新失败的本地修复与受影响组复验均已通过，待推送 - 新 head 并重新核对全部 27 项。完整日志与三个校验过的 artifact 保留在 + 的完整 tree 一致。三个新失败的本地修复与受影响组复验均已通过，已推送 + `70ae86b1c62e095be2754ed178890736e49be5a4`；新 run `41299662435` + attempt 1 已结束，26/27 PASS，只有 console-api 的 terminal writer recovery + 失败；上一轮三个修复场景已远端通过。全部日志和校验过的失败 artifact + 保留于 `ci/published-70ae86b1/`。此前完整日志与三个 artifact 保留在 `ci/published-76fc6508/`，不并入原 16 项人工 review。 - [ ] **恢复物理验证**：远端 CI 核对后继续 P13 和 P14–P19，保留 native 崩溃的新镜像验证及历史数据审计；当前物理资源和失败证据继续保留。 @@ -94,6 +99,28 @@ coverage, and identify the remaining work needed to improve CI stability. 正常 clients builder 已刷新对应输入。七个物理容器及卷身份未变；尚未进行 物理故障、升级或退休。完整执行证据见 `local-repair-closure.json`。 +## Run 41299662435：split 物化与租约恢复 + +- 完整 CI 为 26/27 PASS；console-api 的 92 例中 91 例通过，只有 + `terminal_chunk_writer_recovery` 在原 180 秒恢复期限内未恢复。 + 故障解除后的首次持续分歧是 Group 0 monitor 的 exact writer pair 错误， + 80 次错误阻止后续 serving grant 更新，最终两个 ChunkKV 范围 LeaseExpired。 +- 已提交 split 的父分区因租约失效转移后，其 marker 正常更新为 transfer ID； + 子分区独立可恢复时，旧物化逻辑仍要求两个 split marker。确定性回归在 + 旧代码重现同一错误，也发现仅计数会接受复用 marker 的错误身份。 +- 修复从持久化 split proof 核对每个剩余 marker 的完整 assignment；只有 + CatalogCommitted 才允许另一分区已被后继转移取代，并保持其新状态不变。 + ChildPrepared 仍需两个精确 successor，旧 epoch、错误 artifact 和复用 + marker 均拒绝。原超时、错误处理、故障时长及写入次数保持不变。 +- 精确容器基线通过，但不据此声称复现远端时序。旧代码确定性回归两例失败， + 修复后的完整 KV Server 86 tests 通过，包含四个新回归。全新生产镜像、 + ChunkKV Server 112 tests（另有一个 opt-in RDMA ignore）、fmt 和 full clippy + 均通过。新镜像精确 terminal 151.1 秒通过，恢复 check 21.2 秒；默认并发 + CLI 九例、ChunkKV 两例、balance 一例和 console-api 剩余 83 例全部通过， + 共 95 个不同 canonical case。七个保留物理容器及卷身份不变。 + 证据：`ci/split-materialization-fixed/`；下一次精确 SHA CI 尚未执行。 + 此前 396-case PASS 属于旧生产镜像，不能将两轮输入混为最终镜像全量验证。 + ## 通过场景的稳定性改善 用户要求同时处理 PASS 但不稳定的场景。本节记录额外改善与待验证边界， diff --git a/doc/working/plan-physical-node-ci-handoff.md b/doc/working/plan-physical-node-ci-handoff.md index cc38e48bb..5e1389300 100644 --- a/doc/working/plan-physical-node-ci-handoff.md +++ b/doc/working/plan-physical-node-ci-handoff.md @@ -11,9 +11,9 @@ Upstream: [Physical Node Test Plan](../dev/physical-node-test-plan.md), Goal: finish CI stability verification and publish CROWDB, then resolve P13 and complete the two-host physical recovery matrix without losing data or evidence. -## 1. Current checkpoint: published CI diagnosed, local repairs verified +## 1. Current checkpoint: split recovery repair locally verified -Updated 2026-11-01, 18:56 Asia/Shanghai. The user explicitly resumed this plan. +Updated 2026-11-01, 20:00 Asia/Shanghai. The user explicitly resumed this plan. Continue the agreed CI refinement, commit/push, exact-head CI, then physical recovery sequence. No physical fault or retirement has been dispatched on resume. @@ -27,7 +27,7 @@ recovery sequence. No physical fault or retirement has been dispatched on resume uses `scope-guard-resumed-before.json`, preserving exact identities and the observed stopped state throughout CI. Physical restart remains deferred. -- The final-image container acceptance has completed: all 396 canonical cases +- The previous-image container acceptance completed: all 396 canonical cases have a finalized latest PASS; required concurrent isolation also passed. - The strengthened CLI five cases, real GC three cases, three fresh paired auxiliary/Journal browser rounds, three fresh Go SDK rounds and instrumented @@ -56,10 +56,27 @@ recovery sequence. No physical fault or retirement has been dispatched on resume passes. All six CLI cases, including the interrupted terminal-recovery peer, and the original shell/capacity browser pair passed. Fixture 304, full clippy, fmt and TypeScript passed; the closing scope guard preserved all seven nodes. - Next publish these coherent repairs and verify the new head's complete CI. -- CI checks out merge commit 66579652b16f798d8fd0211fe39002b0f7eae594. + The coherent test repair commit 70ae86b1c62e095be2754ed178890736e49be5a4 + was pushed successfully. Its new run 41299662435, attempt 1, finished with + 26 successful jobs and one failure: console-api's terminal chunk writer + recovery. All 27 job logs and its verified failure artifact are preserved + in ci/published-70ae86b1/. The three previous failed cases passed remotely. +- After a lease-excluded transfer superseded a committed split's retained + parent, child materialization still required two original split markers. + The monitor repeatedly failed before serving-grant renewal; the original + 180-second recovery deadline expired. An exact local run passed, but the + durable-transition regression reproduced the same error deterministically. + The production fix validates remaining assignments against the durable split + proof and preserves the transferred parent. All 86 KV Server and 112 ChunkKV + Server tests passed, with one existing opt-in RDMA ignore. Full clippy and fmt + passed. A new source-verified image passed the exact terminal recovery case + (151.1 seconds; recovery check 21.2 seconds), all nine CLI cases at default + concurrency, two ChunkKV cases, balance and 83 remaining console-api cases. + These are 95 distinct current-image cases. Publication and new exact-head CI + remain pending. Previous 396-case coverage belongs to the previous image. +- This CI checked out merge commit e9f241349146046d52c1198cf3f1b0e0dbdbccff. Its complete source tree equals the pushed head's tree - 8039607dbf9901dc8fdcbd5e78b207ed10627140. Local and CI image runtime digests + 9a7a6a85ba27c2722610d4185912edf67b3c86bf. Local and CI image runtime digests differ; source equality does not claim identical binaries. - Two physical hosts have not completed final acceptance: P13 is still failed, P14–P19 are unrun, and the latest native lifecycle fix needs physical validation. @@ -72,15 +89,16 @@ the user finishes joint review; a PASS must not check a human review box. - CROWDB root: /cjdata/cpp/crowdb2. - Branch: task-fix-build; published HEAD: - 76fc6508189dd31018e953674a8f1c693011498d. + 70ae86b1c62e095be2754ed178890736e49be5a4. - The four previously local commits were included in the fast-forward push: cdf023d6 (recovery/liveness), 81d07f03 (nearby mirrors), d396a220 (healthy leader quorum lease), and 7a93c4e5 (cleanup/handoff). -- The production changes are committed and pushed. Three subsequent test fixes - and working-plan updates remain local until the next publication. Preserve +- The earlier production changes and three subsequent test fixes are committed + and pushed. The new split materialization fix, regression and plan updates + remain local. Preserve any new user edits; do not reset or restore. -- Remote task-fix-build was fetched at 8885d2a immediately before the push; - GitHub then confirmed the new exact head 76fc6508. +- Remote task-fix-build was fetched at 76fc6508 immediately before the last push; + GitHub then confirmed the new exact head 70ae86b1. [PR 55](https://github.com/buzzcrow/crowdb/pull/55) is open. Fetch and verify remote state again immediately before the eventual push. - After explicit resume, the agreed order remains: finish local CI stability @@ -100,7 +118,7 @@ All abbreviated paths below are relative to: /cjdata/cpp/crowdb2/.crowdb-runtime/artifacts/physical-nodes/physical-20261031T194149Z - CURRENT-STATE.json: current pause and next actions. -- ci/stability-final/: final local gates, queues, complete logs, current-image +- ci/stability-final/: previous-image local gates, queues, complete logs, coverage, independent reviews and pending refinement evidence. - ci/stability-final/final-current-image-evidence.json: finalized 396-case coverage and current-image mappings for all original @@ -121,11 +139,32 @@ All abbreviated paths below are relative to: - ci/published-76fc6508/: all 27 job logs, failed artifacts, repair-diagnosis.json, repair-queue.json, local-repair-closure.json and exact/original-pair results. These three newly published failures are separate from the historical 16 jobs. +- ci/published-70ae86b1/: all 27 job logs, the checked terminal-recovery artifact, + exact baseline, deterministic red/green and both affected component suites. +- ci/split-materialization-fixed/: new image build, exact and grouped container + queues, 95-case source-specific local closure, quality gates and diff review. - The earlier long handoff is preserved at handoff-history/plan-physical-node-ci-handoff-before-rewrite-20261101T062936Z.md. It contains superseded historical instructions, not current authorization. -Final local server image: +Current local server image: + +- Tag: crowdb-node:split-fixed. +- Production source tree: 861fc9d45ae32f72a5199585a8bb1b231048b448. +- Docker image ID: + sha256:cec0d2c1bf885429024298a884aecdfd137a40c0b38ec9f4a077c36cc47394ae. +- OCI config digest: + sha256:c77782fdd13916f42e72e283b06662f8cf22a2158233faf74fd4a63069b42eda. +- Runtime SHA-256: + 9fab75b6dae267c98946c7c9da7866deaa406531255497f9fd3b8f6c3faadfdc. +- OCI archive: image/split-materialization-fixed/crowdb.oci.tar; SHA-256: + 11a6ff4a615f55b638f412cd3b93a5f248c68792a33bbbb77b59385d5ef6d3bc. +- Revision label 70ae86b1 predates this uncommitted fix; the source-tree label + records the actual source. Both identities are preserved. Physical hosts have + not been upgraded. Setup and retirement require this exact image, current + local verification and the eventual new all-27-job CI PASS. + +Previous local server image and retained 396-case evidence: - Tag: crowdb-node:ci-stable. - Production source tree: 376fdc26a0e96d7810313ea670bf043c4f0dfcae. @@ -147,12 +186,12 @@ Final local server image: The CLI used 65819b6aa8cdacf284941faf932681219d01cd0a; resumed GC used d862cc4d3238afd6e9ccb39e4996ab415d10c5bc. Later changes affect only the auxiliary browser helper and documentation, so those prior passes remain valid. -- ci/stability-final/refined-source-verification.json confirms current server - sources match the final image. Later changes are client/test/documentation - changes; the full all-domain run used an earlier copied client bundle. +- ci/stability-final/refined-source-verification.json confirmed the previous + server sources at that checkpoint. The full all-domain run used an earlier + copied client bundle. Its PASS cannot validate the newer GC/Journal assertions; their separate - strengthened runs now pass. resume-server-source-verification.json confirms - current production sources still match this image. + strengthened runs passed. The new production split fix requires the separate + split-fixed image and its current source-verification receipt above. - This handoff update changes the repository source identity again. Before another E2E run, refresh compiled clients through the normal builder. Do not bypass manifest validation or blindly rerun all-queue, which is complete. @@ -195,7 +234,7 @@ Final local server image: liveness while preserving 18 unique full endpoint registrations. Four regressions and all five strengthened actual CLI cases now pass. This does not add waits or caller retries. -- [x] **Final image container acceptance**: 396/396 latest canonical PASS and +- [x] **Previous image container acceptance**: 396/396 latest canonical PASS and required four-case concurrent isolation PASS. The progress helper shows eight isolation passes because two all-domain attempts each ran four; do not count them as eight distinct isolation cases. All seven physical resources retained @@ -220,6 +259,8 @@ Main source ownership in the published change: oci_shortcuts.py, console_cli_target.py, GC convergence/pressure/sdk_pressure, console/native_balance.rs, console-ui lifecycle/Journal flows and helpers, their regressions, and tools/test-metrics/measure.py. +- Split materialization: app/crowdb-kv-server/src/background/domain_monitor/ + chunk_kv/catalog.rs and tests/common/split_materialization.rs. - Documentation: this handoff, plan-ci-stability-review.md and doc/dev/physical-node-test-plan.md. Inspect git status before staging. @@ -288,7 +329,10 @@ The user has resumed; proceed in the order below. the remote before the authorized fast-forward push; do not force-push. No Sirius-specific pre-push rules apply to this repository. - [~] **Inspect new exact-head CI**: run 41295426521 attempt 1 ended 24 PASS / 3 FAIL. - The three fixes and local affected regressions passed; commit/push them next. + The three fixes and local affected regressions passed and were pushed as + 70ae86b1. Run 41299662435 attempt 1 ended 26 PASS / 1 FAIL. Its production + split materialization repair now has 198 affected component PASS and 95 + exact-image container PASS; publish the coherent fix and inspect its new run. Match the actual new head SHA/run/attempt, inspect every expanded job (the three historical runs had 27), preserve every first failure and artifact, fix real errors and rerun the appropriate new-head job/run. Do not rerun an old From 18015caf0cc99978eec99517a3f52b468128c0e2 Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 1 Nov 2026 21:23:41 +0800 Subject: [PATCH 15/24] Preserve dependency waits during service deployment --- .../ui/src/services/useNodeServicePlans.ts | 10 +- .../useNodeServicePlansDependencies.test.tsx | 54 +++++++++ doc/working/plan-ci-stability-review.md | 35 +++++- doc/working/plan-physical-node-ci-handoff.md | 103 +++++++++++++----- .../e2e/fixtures/defaultDeployments.ts | 5 +- .../flows/11-cluster-diskdb-lifecycle.spec.ts | 12 ++ e2e-test/tests/console/native_balance.rs | 4 +- e2e-test/tests/console/native_browser.rs | 7 +- 8 files changed, 190 insertions(+), 40 deletions(-) create mode 100644 app/crowdb-web/ui/src/services/useNodeServicePlansDependencies.test.tsx diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlans.ts b/app/crowdb-web/ui/src/services/useNodeServicePlans.ts index 91f88985d..4f6b6c8c8 100644 --- a/app/crowdb-web/ui/src/services/useNodeServicePlans.ts +++ b/app/crowdb-web/ui/src/services/useNodeServicePlans.ts @@ -175,7 +175,7 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< const step: ServiceStep = registered.pid ? { state: 'deployed' } : registered.start_pending ? { state: 'waiting', detail: startingService } : { state: 'failed', detail: 'Registered service is stopped; restart it from the Node menu before resuming.' }; - if (plan[kind].state !== step.state) await write(kind, step); + if (plan[kind].state !== step.state || plan[kind].detail !== step.detail) await write(kind, step); continue; } retryRegistered.current.delete(`${id}/${kind}`); @@ -225,10 +225,10 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< deploymentStarted = true; } catch (error) { if (disposed || stopped.current) return; - // Hardware can change between the readiness probe and mutation. - // A rejected prerequisite has not launched a service; keep it queued. - const dependency = String(error).match(/^Error: HTTP 409: (Waiting: .+)$/); - if (dependency && (kind === 'chunkdb' || kind === 'chunk-kv')) { + // A prerequisite can change after its probe. The monitor may + // already own the intent; the next pass observes that registration. + const dependency = String(error).match(/^Error: HTTP 409: (Waiting: [\s\S]+)$/); + if (dependency && (kind === 'chunkdb' || kind === 'chunk-kv' || kind === 'access-server')) { waitingForStorage = true; await write(kind, { state: 'waiting', detail: dependency[1] }); continue; diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlansDependencies.test.tsx b/app/crowdb-web/ui/src/services/useNodeServicePlansDependencies.test.tsx new file mode 100644 index 000000000..4f8bb2ff9 --- /dev/null +++ b/app/crowdb-web/ui/src/services/useNodeServicePlansDependencies.test.tsx @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +import { act, renderHook } from '@testing-library/react'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { listServers, restartServer, restartDiskdb } from '../api'; +import { serviceRequest } from './client'; +import { useNodeServicePlans } from './useNodeServicePlans'; +import type { EnrichedStoreView } from '../types'; + +vi.mock('../api', () => ({ listServers: vi.fn(), deployServer: vi.fn(), deployDiskdb: vi.fn(), restartServer: vi.fn(), restartDiskdb: vi.fn() })); +vi.mock('./client', () => ({ serviceNames: {}, serviceRequest: vi.fn() })); + +beforeEach(() => { vi.useFakeTimers(); }); +afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); + +it.each([ + ['chunk-kv', 'Waiting: config error: monitor SSH command failed: Error: managed service exited before readiness: node=1 service=chunk-kv-1 exit_code=1\n'], + ['access-server', 'Waiting: service dependency has no committed intent'], +] as const)('observes the accepted %s dependency wait without another deployment or restart', async (kind, detail) => { + const stores = [{ store_id: '0', groups: [{ group_id: '0' }, { group_id: '1' }] }] as unknown as EnrichedStoreView[]; + let accepted = false; + let running = false; + vi.mocked(listServers).mockImplementation(async () => [ + { node_id: 1, service_type: 'chunkdb', pid: 11 }, + { node_id: 1, service_type: 'diskio', pid: 12 }, + { node_id: 2, service_type: 'diskio', pid: 22 }, + { node_id: 2, service_type: 'chunk-kv', pid: 23 }, + ...(accepted ? [{ id: `${kind}-1`, node_id: 1, service_type: kind, + pid: running ? 42 : undefined, start_pending: !running }] : []), + ] as Awaited>); + vi.mocked(serviceRequest).mockImplementation(async (path, method) => { + if (path === '/service-plans') return {}; + if (path === '/deployment-defaults') return { [kind]: { instance_id: '1', http_port: 7400, rpc_port: 7401 } }; + if (path === '/nodes/1/services/deploy' && method === 'POST') { + accepted = true; + throw new Error(`HTTP 409: ${detail}`); + } + return { revision: 1, ready: true }; + }); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => { await result.current.start(1, [kind]); }); + expect(result.current.plans[1][kind]).toEqual({ state: 'waiting', detail }); + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1][kind]).toEqual({ state: 'waiting', detail: 'Waiting: target monitor is starting the registered service' }); + running = true; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1][kind]).toEqual({ state: 'deployed' }); + expect(vi.mocked(serviceRequest).mock.calls.filter(([path, method]) => + path.endsWith('/services/deploy') && method === 'POST')).toHaveLength(1); + expect(restartServer).not.toHaveBeenCalled(); + expect(restartDiskdb).not.toHaveBeenCalled(); + expect(vi.mocked(serviceRequest).mock.calls.some(([path]) => path.endsWith('/restart'))).toBe(false); +}); diff --git a/doc/working/plan-ci-stability-review.md b/doc/working/plan-ci-stability-review.md index fa79df5c6..0fa5429bf 100644 --- a/doc/working/plan-ci-stability-review.md +++ b/doc/working/plan-ci-stability-review.md @@ -25,7 +25,9 @@ coverage, and identify the remaining work needed to improve CI stability. 本地补充验证控制器正常退出；首次推送后的三个新失败已修复并完成本地复验， 修复已推送为 `70ae86b1`，完整 CI 为 26/27 PASS；新发现的 terminal writer 恢复故障已定位为 split 物化与父分区租约转移的冲突，修复和本地复验已完成， -待提交推送并核对下一轮精确 SHA CI。 +已推送为 `5af2737f`，run `41304660599` attempt 1 已结束，25/27 PASS。 +terminal writer recovery 已远端通过；auxiliary 与六节点部署修复及本地复验已完成， +新的提交和精确 SHA CI 尚待完成。 物理容器在恢复前已停止，原因未知；七个容器、镜像和卷身份均保持不变。 ## 当前执行队列 @@ -118,9 +120,38 @@ coverage, and identify the remaining work needed to improve CI stability. 均通过。新镜像精确 terminal 151.1 秒通过，恢复 check 21.2 秒；默认并发 CLI 九例、ChunkKV 两例、balance 一例和 console-api 剩余 83 例全部通过， 共 95 个不同 canonical case。七个保留物理容器及卷身份不变。 - 证据：`ci/split-materialization-fixed/`；下一次精确 SHA CI 尚未执行。 + 证据：`ci/split-materialization-fixed/`；已推送 `5af2737f`，其精确 SHA + CI run `41304660599` attempt 1 已结束；原 terminal recovery 已通过。 此前 396-case PASS 属于旧生产镜像，不能将两轮输入混为最终镜像全量验证。 +## Run 41304660599：依赖等待与生命周期前置条件 + +- 全部 27 个 job 日志和两个失败 artifact 已保留并校验；25 PASS / 2 FAIL。 + `console-native-operations` 为 18 PASS / 1 FAIL，`console-native-data` 为 + 13 PASS / 1 FAIL。六节点 trace 为 27.4 MB，超过原 16 MiB 导出上限， + 远端未保留；该部分只能依据完整失败日志与本地重现诊断。 +- Auxiliary 外部 Stop、OS 进程退出和页面 Failed 均通过，随后 Restart + 在 ChunkKV 初始放置的 TransferFencing 窗口返回 409 Waiting。原精确 + 场景 12.4 秒 PASS；原并发组合 12.1 秒复现；串行对照 13.7 秒 PASS。 + 修复前置条件：沿用已有正常 placement settle，再执行故障，不调整生产 + balance 策略、response/PID/OS/sibling/readiness 断言或 timeout。 +- 六节点 Access Waiting 缺少分类，本地另复现带换行的 ChunkKV Waiting + 不匹配原正则。monitor 已恢复实际进程，计划仍为 failed。新增两例确定性 + 回归先红；修正分类后又捕获同 waiting 状态保留旧 detail，导致已登记启动 + 误用十秒缺盘退避。修复按已有 CLU-03/SVC-03 契约观察目标 monitor， + 回归要求最终 deployed，且 deploy 一次、restart 零次。 +- 原有序并发组在更早的 KV lifecycle 失败；精确串行同样失败。停止 Group 0 + leader 后过早提交 restart，权威校验三秒到期。增加同 cluster/operation + 的权威恢复观察，使用已有选举预算，再提交一次 mutation，保留响应预算。 +- 新镜像的 48 个独立 case 全部通过：native operations 19、browser 27、 + 六节点和固定端口/失败重试两例。先通过三个精确场景，再在原有序生命周期 + 组复验。六节点两次 Playwright 为 51.2/49.2 秒；固定端口为 5.4 秒。 + 数据组虽配置 concurrency 2，资源预算实际使两例顺序执行，不能声称并发。 +- UI 366、plan-hook 28（含两例新回归）、fixture 304、完整 TypeScript、 + fmt/full clippy 均通过。七个物理容器及卷身份未变。证据保存在 + `ci/lifecycle-dependencies-fixed/`；新的精确 SHA CI 尚待完成。 + 原 16 项人工 review 保持未勾选。 + ## 通过场景的稳定性改善 用户要求同时处理 PASS 但不稳定的场景。本节记录额外改善与待验证边界， diff --git a/doc/working/plan-physical-node-ci-handoff.md b/doc/working/plan-physical-node-ci-handoff.md index 5e1389300..c2427533f 100644 --- a/doc/working/plan-physical-node-ci-handoff.md +++ b/doc/working/plan-physical-node-ci-handoff.md @@ -11,9 +11,9 @@ Upstream: [Physical Node Test Plan](../dev/physical-node-test-plan.md), Goal: finish CI stability verification and publish CROWDB, then resolve P13 and complete the two-host physical recovery matrix without losing data or evidence. -## 1. Current checkpoint: split recovery repair locally verified +## 1. Current checkpoint: lifecycle repair locally verified, publication pending -Updated 2026-11-01, 20:00 Asia/Shanghai. The user explicitly resumed this plan. +Updated 2026-11-01, 21:21 Asia/Shanghai. The user explicitly resumed this plan. Continue the agreed CI refinement, commit/push, exact-head CI, then physical recovery sequence. No physical fault or retirement has been dispatched on resume. @@ -41,9 +41,11 @@ recovery sequence. No physical fault or retirement has been dispatched on resume took 5574.963 seconds; the earlier actual CLI failure remains in its attempts. - The CI pause marker was removed after the normal client refresh and resumed scope guard passed. The separate physical investigation marker remains. -- No test/workload/fault controller is left running. Physical work remains - paused; the seven retained physical containers and their volumes remain owned - and preserved. Do not restart old session IDs. +- The previous full-suite controller completed. Current lifecycle repair tests + are tracked under ci/lifecycle-dependencies-fixed/; original failures and + baseline comparisons remain under ci/published-5af2737f/. Physical work remains deferred; the + seven retained physical containers and their volumes remain owned and preserved. + Do not restart old session IDs. - Commit 76fc6508189dd31018e953674a8f1c693011498d was pushed successfully. [CI run 41295426521](https://github.com/buzzcrow/crowdb/actions/runs/41295426521), attempt 1, finished with 24 successful and three failed jobs out of 27. @@ -72,9 +74,40 @@ recovery sequence. No physical fault or retirement has been dispatched on resume passed. A new source-verified image passed the exact terminal recovery case (151.1 seconds; recovery check 21.2 seconds), all nine CLI cases at default concurrency, two ChunkKV cases, balance and 83 remaining console-api cases. - These are 95 distinct current-image cases. Publication and new exact-head CI - remain pending. Previous 396-case coverage belongs to the previous image. -- This CI checked out merge commit e9f241349146046d52c1198cf3f1b0e0dbdbccff. + These are 95 distinct current-image cases. The coherent fix was committed and + fast-forward pushed as 5af2737fd5797a1afd1395eac63427e8fdb892d2; commit hooks + passed fmt/clippy. Run 41304660599 attempt 1 ended 25 PASS / 2 FAIL. Its merge checkout + 3c87db4bcf71f2d532762bbadb3a4416732514f1 has the same complete tree as HEAD: + 4b93303bfaf8cc2e0b797b14815d8e884771e473. Previous 396-case coverage belongs + to the previous image. All 27 current jobs must pass before physical dispatch. +- The previous terminal writer recovery now passes remotely. The two new failures + are auxiliary external Access restart during an initial ChunkKV transfer and + six-node Access deployment Waiting classified as terminal failure. All 27 logs + and both verified artifacts are retained. The six-node trace exceeded the + existing 16-MiB export limit and was not uploaded; do not claim it was inspected. +- Local exact auxiliary passed (12.4 seconds), its original concurrent pair + reproduced the same restart 409 (12.1 seconds), and the serial auxiliary + passed (13.7 seconds). Finish normal initial placement before lifecycle faults; + preserve every original response, process, sibling and readiness assertion. +- The original ordered local partition stopped earlier at KV restart after + stopping Group 0's leader. The isolated serial case also failed. A new leader + was elected after submission; the authority guard returned 502. Observe the + same cluster's surviving authority within the existing election budget before + submitting one restart. The mutation response budget remains unchanged. +- Two deterministic plan-hook regressions reproduced Access Waiting and multiline + ChunkKV Waiting becoming failed. A further regression exposed stale waiting + detail selecting the ten-second missing-disk backoff for a registered startup. + The repair restores the existing dependency-wait contract and observes the + accepted operation without a duplicate deploy/restart. The new source-verified + image passed all 48 distinct affected cases: native operations 19, browser 27 + and native deployment plans two. Exact KV, auxiliary pair and six-node cases + passed first; lifecycle cases passed again at original default concurrency. + The two data-plan cases used concurrency two but resource-aware batching ran + them sequentially (12,800 + 6,656 MiB exceed the 16,384-MiB budget). + UI 366, plan-hook 28 including two new regressions, fixture 304, TypeScript, + fmt and full clippy passed. New publication and all 27 exact-head CI jobs + remain required before any physical dispatch. +- Run 41299662435 checked out merge commit e9f241349146046d52c1198cf3f1b0e0dbdbccff. Its complete source tree equals the pushed head's tree 9a7a6a85ba27c2722610d4185912edf67b3c86bf. Local and CI image runtime digests differ; source equality does not claim identical binaries. @@ -88,17 +121,17 @@ the user finishes joint review; a PASS must not check a human review box. ## 2. Repository and publication boundary - CROWDB root: /cjdata/cpp/crowdb2. -- Branch: task-fix-build; published HEAD: - 70ae86b1c62e095be2754ed178890736e49be5a4. +- Branch: task-fix-build; published HEAD before this lifecycle repair: + 5af2737fd5797a1afd1395eac63427e8fdb892d2. - The four previously local commits were included in the fast-forward push: cdf023d6 (recovery/liveness), 81d07f03 (nearby mirrors), d396a220 (healthy leader quorum lease), and 7a93c4e5 (cleanup/handoff). -- The earlier production changes and three subsequent test fixes are committed - and pushed. The new split materialization fix, regression and plan updates - remain local. Preserve +- The production changes, subsequent test fixes and new split materialization + repair are committed and pushed. The current lifecycle/dependency repair and + working-plan updates are prepared for the next coherent commit. Preserve any new user edits; do not reset or restore. -- Remote task-fix-build was fetched at 76fc6508 immediately before the last push; - GitHub then confirmed the new exact head 70ae86b1. +- Remote task-fix-build was fetched at 70ae86b1 immediately before the last push; + GitHub then confirmed the new exact head 5af2737f. [PR 55](https://github.com/buzzcrow/crowdb/pull/55) is open. Fetch and verify remote state again immediately before the eventual push. - After explicit resume, the agreed order remains: finish local CI stability @@ -143,26 +176,35 @@ All abbreviated paths below are relative to: exact baseline, deterministic red/green and both affected component suites. - ci/split-materialization-fixed/: new image build, exact and grouped container queues, 95-case source-specific local closure, quality gates and diff review. +- ci/published-5af2737f/: third publication, all 27 job logs, both checked failure + artifacts, concurrent/serial baselines and deterministic dependency regressions. +- ci/lifecycle-dependencies-fixed/: current prepare/exact/operations/browser/data + queues, 48-case source-specific closure, quality gates and diff review. - The earlier long handoff is preserved at handoff-history/plan-physical-node-ci-handoff-before-rewrite-20261101T062936Z.md. It contains superseded historical instructions, not current authorization. Current local server image: -- Tag: crowdb-node:split-fixed. -- Production source tree: 861fc9d45ae32f72a5199585a8bb1b231048b448. +- Tag: crowdb-node:lifecycle-fixed. +- Source tree: 1516797b67ad493c9bbf9c25048e574e7d994f31. - Docker image ID: - sha256:cec0d2c1bf885429024298a884aecdfd137a40c0b38ec9f4a077c36cc47394ae. + sha256:359897a74ffa13d05e6c3bcf6708c7291a2b761e7c7193b706ba436060d75119. - OCI config digest: - sha256:c77782fdd13916f42e72e283b06662f8cf22a2158233faf74fd4a63069b42eda. + sha256:a35afa03888629de0d172ca9a186f5c67d558c64125b9f075747f11b049b08e0. - Runtime SHA-256: - 9fab75b6dae267c98946c7c9da7866deaa406531255497f9fd3b8f6c3faadfdc. -- OCI archive: image/split-materialization-fixed/crowdb.oci.tar; SHA-256: - 11a6ff4a615f55b638f412cd3b93a5f248c68792a33bbbb77b59385d5ef6d3bc. -- Revision label 70ae86b1 predates this uncommitted fix; the source-tree label - records the actual source. Both identities are preserved. Physical hosts have - not been upgraded. Setup and retirement require this exact image, current - local verification and the eventual new all-27-job CI PASS. + c03d3b78cf126c42923cfd91b3e6d03160ae082e77039708afafda465d9a5e26. +- OCI archive: image/lifecycle-dependencies-fixed/crowdb.oci.tar; SHA-256: + 41107c698fb1ff1ddbd51aee05401e797cb4587b3103dcd1145bad6ae83cc020. +- Revision label 5af2737f predates this uncommitted fix; the source-tree label + records the actual source. Physical hosts have not been upgraded. Setup and + retirement require this image, current local verification and the eventual + new all-27-job CI PASS. +- The preceding split-fixed image and its 95-case validation remain in + image/split-materialization-fixed/ and ci/split-materialization-fixed/. + Its Docker ID begins cec0d2c1 and source tree is + 861fc9d45ae32f72a5199585a8bb1b231048b448; those passes are not relabeled as + current-image coverage. Previous local server image and retained 396-case evidence: @@ -332,7 +374,10 @@ The user has resumed; proceed in the order below. The three fixes and local affected regressions passed and were pushed as 70ae86b1. Run 41299662435 attempt 1 ended 26 PASS / 1 FAIL. Its production split materialization repair now has 198 affected component PASS and 95 - exact-image container PASS; publish the coherent fix and inspect its new run. + exact-image container PASS. The fix is pushed as 5af2737f; inspect all jobs in + run 41304660599 attempt 1 ended 25 PASS / 2 FAIL. Repair the two new lifecycle + failures and the local Group 0 restart boundary, then publish and inspect the + next exact-head run. Current diagnosis: ci/published-5af2737f/repair-diagnosis.json. Match the actual new head SHA/run/attempt, inspect every expanded job (the three historical runs had 27), preserve every first failure and artifact, fix real errors and rerun the appropriate new-head job/run. Do not rerun an old @@ -426,7 +471,9 @@ Physical work is deferred until the CI/publication stage above is complete. - [ ] **P17–P19 close only after all prerequisites pass**: freeze complete evidence/results/summary, inspect identities across the entire fault and audit intervals, privately preserve remaining data, then exact owned-only retirement. - Review harness/close_run.py before dispatch; no global cleanup. + Use only the reviewed harness/close_physical_final.py and + cleanup_host_final.py stages. Never replay obsolete close_run.py or + cleanup_host.py; no global cleanup. The separate investigate-unexpected-P12-restart marker prevents recovery_final.py from dispatching P14. It is not the CI pause marker. Do not remove it merely to diff --git a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts index 47189ce07..a568ea0df 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts @@ -23,7 +23,7 @@ export function observeDefaultDeployments(page: Page) { const identity = key(response); if (identity) replies.set(identity, [...(replies.get(identity) ?? []), response]); }); - const expectedWait = (response: Response, kind: string) => response.status() === 409 && ['chunkdb', 'chunk-kv'].includes(kind); + const expectedWait = (response: Response, kind: string) => response.status() === 409 && ['chunkdb', 'chunk-kv', 'access-server'].includes(kind); async function completed(nodes: number[], kind: string) { const timeout = process.platform === 'darwin' ? 30_000 : 10_000; let found: { node: number; response: Response } | undefined; @@ -65,7 +65,8 @@ export function observeDefaultDeployments(page: Page) { } async function verifyResponse(response: Response, kind: string) { if (expectedWait(response, kind)) { - expect((await response.json()).error).toMatch(/^Waiting: /); + const node = Number(new URL(response.url()).pathname.split('/')[3]); + expect(await response.json()).toMatchObject({ error: expect.stringMatching(/^Waiting: /), service: kind, node_id: node, retryable: true }); } else { expect(response.status(), await response.text()).toBe(201); } diff --git a/e2e-test/clients/console-ui/e2e/flows/11-cluster-diskdb-lifecycle.spec.ts b/e2e-test/clients/console-ui/e2e/flows/11-cluster-diskdb-lifecycle.spec.ts index 51357c536..af03a5d1b 100644 --- a/e2e-test/clients/console-ui/e2e/flows/11-cluster-diskdb-lifecycle.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/11-cluster-diskdb-lifecycle.spec.ts @@ -5,6 +5,7 @@ import { test, expect } from '../fixtures/realBackend'; import { diskMenu } from '../fixtures/capacity'; import { nativeProcessAlive } from '../fixtures/nativeProcess'; import { liveService, service, serviceRow, stoppedService, unchangedService, type LiveService } from '../fixtures/serviceLifecycle'; +import { step } from '../fixtures/stepTimer'; // Baseline: 15.9s (2026-10-12). test('full deploy flow: deploy diskdb via UI, restart, stop, delete via context menu', async ({ page, request }) => { @@ -100,11 +101,22 @@ async function diskDbLifecycle(page: Page, request: APIRequestContext, kv: LiveS } async function kvLifecycle(page: Page, request: APIRequestContext, original: LiveService, ddb: LiveService) { + const before = await request.get('/api/node/status'); + expect(before.ok(), await before.text()).toBe(true); + const authority = await before.json(); + expect(authority).toMatchObject({ phase: 'active', available: true, cluster_id: expect.any(String), operation_id: expect.any(String) }); await act(page, original, 'Stop CrowDB Storage', '/api/nodes/1/server/stop'); await stoppedService(request, original); await unchangedService(request, ddb); await expect(serviceRow(page, original).getByTitle('Healthy', { exact: true })).toHaveCount(0, { timeout: 10_000 }); await expect(serviceRow(page, ddb).getByTitle('Healthy', { exact: true })).toBeVisible({ timeout: 10_000 }); + // The stopped process may have been Group 0's leader. Confirm its successor + // before submitting the one restart mutation, using the election budget. + await step('KV lifecycle: surviving Group 0 authority', () => expect.poll(async () => { + const response = await request.get('/api/node/status'); + expect(response.ok(), await response.text()).toBe(true); + return response.json(); + }, { timeout: 10_000, intervals: [100] }).toEqual(authority)); await act(page, original, 'Restart CrowDB Storage', '/api/nodes/1/server/restart'); const resumed = await liveService(request, 'paxos-kv', original.pid); await unchangedService(request, ddb); diff --git a/e2e-test/tests/console/native_balance.rs b/e2e-test/tests/console/native_balance.rs index cb8fef8e5..6008510ac 100644 --- a/e2e-test/tests/console/native_balance.rs +++ b/e2e-test/tests/console/native_balance.rs @@ -70,14 +70,14 @@ pub(super) async fn settle_existing(context: &TestNativeContext) { }) && placement_ready(&context.kv, catalog.generation()).await { eprintln!( - "[PHASE] journal fixture placement settled in {:?}", + "[PHASE] native fixture placement settled in {:?}", started.elapsed() ); return; } assert!( started.elapsed() < Duration::from_secs(40), - "journal fixture's normal placement did not settle" + "native fixture's normal placement did not settle" ); } } diff --git a/e2e-test/tests/console/native_browser.rs b/e2e-test/tests/console/native_browser.rs index 33d56af98..7b6c19298 100644 --- a/e2e-test/tests/console/native_browser.rs +++ b/e2e-test/tests/console/native_browser.rs @@ -135,7 +135,12 @@ async fn browse_diskdb() { #[tokio::test] #[ignore = "requires normal storage and owned container PID observations"] async fn auxiliary_process_menus() { - browse_storage().await; + let context = TestNativeContext::connect().await; + super::native_storage::start(&context).await; + // Finish initial placement before independent Stop/Start operations. + super::native_balance::settle_existing(&context).await; + let result = crate::common::control::native_browser().await; + assert_eq!(result["passed"], true); } #[tokio::test] From 1814405e4dd1fe0fcd64f9f18ac7c71e98d64fd2 Mon Sep 17 00:00:00 2001 From: Gian Date: Sun, 1 Nov 2026 23:11:53 +0800 Subject: [PATCH 16/24] Respect Access readiness and confirmed S3 deletions --- app/crowdb-web/ui/src/s3/S3View.test.tsx | 72 +++++++++++++++ app/crowdb-web/ui/src/views/S3View.tsx | 1 + .../console/design-crowdb-console-ui.md | 5 + doc/working/plan-ci-stability-review.md | 45 ++++++++- doc/working/plan-physical-node-ci-handoff.md | 91 +++++++++++++++---- .../e2e/fixtures/nativeServiceLifecycle.ts | 21 +++++ .../flows/11-cluster-server-lifecycle.spec.ts | 14 ++- e2e-test/tests/console/native_browser.rs | 7 +- 8 files changed, 217 insertions(+), 39 deletions(-) create mode 100644 app/crowdb-web/ui/src/s3/S3View.test.tsx diff --git a/app/crowdb-web/ui/src/s3/S3View.test.tsx b/app/crowdb-web/ui/src/s3/S3View.test.tsx new file mode 100644 index 000000000..38357052a --- /dev/null +++ b/app/crowdb-web/ui/src/s3/S3View.test.tsx @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +import { act, fireEvent, render, screen, waitFor, within } from '@testing-library/react'; +import { afterEach, expect, it, vi } from 'vitest'; +import { S3View } from '../views/S3View'; +import { SelectionProvider } from '../contexts/SelectionContext'; +import { DomainProvider } from '../contexts/DomainContext'; +import { Domain } from '../types'; + +vi.mock('../contexts/ActivityContext', () => ({ useActivity: () => ({ log: vi.fn() }) })); +vi.mock('./useObjectLocations', () => ({ useObjectLocations: () => ({ selected: null, page: null, snapshot: () => null }) })); +afterEach(() => { vi.unstubAllGlobals(); }); + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(complete => { resolve = complete; }); + return { promise, resolve }; +} + +const listing = (keys: string[]) => new Response(`false${keys.map(key => + `${key}12etagtoday`).join('')}`); + +async function deletion() { + const removed = deferred(); + const refreshed = deferred(); + let lists = 0; + const fetcher = vi.fn(async (input: RequestInfo | URL, options?: RequestInit) => { + const path = new URL(String(input), 'http://localhost').pathname; + if (path === '/api/access/connections') return new Response(JSON.stringify({ s3: 'http://s3' })); + if (path === '/api/access/s3/') return new Response('photos'); + if (path === '/api/access/s3/photos') return ++lists === 1 ? listing(['delete.txt', 'keep.txt', 'delete.txt.backup']) : refreshed.promise; + if (path === '/api/access/s3/photos/delete.txt') { + if (options?.method === 'DELETE') return removed.promise; + if (options?.method === 'HEAD') return new Response(null, { headers: { 'content-length': '12' } }); + } + throw new Error(`Unexpected request: ${options?.method} ${input}`); + }); + vi.stubGlobal('fetch', fetcher); + render(); + const buckets = await screen.findByRole('table', { name: 'S3 bucket list' }); + fireEvent.click(await within(buckets).findByRole('button', { name: 'photos' })); + fireEvent.click(await screen.findByRole('button', { name: 'delete.txt' })); + await waitFor(() => expect(screen.getByRole('region', { name: 'Object metadata' })).toHaveTextContent('content length12')); + fireEvent.contextMenu(screen.getByRole('heading', { level: 1, name: 'delete.txt' })); + fireEvent.click(screen.getByRole('menuitem', { name: 'Delete object…' })); + fireEvent.click(within(screen.getByRole('dialog', { name: 'Delete object' })).getByRole('button', { name: 'Delete object' })); + return { removed, refreshed, fetcher, lists: () => lists }; +} + +it('removes only the confirmed object while its list refresh is still pending', async () => { + const state = await deletion(); + await act(async () => state.removed.resolve(new Response(null, { status: 204 }))); + await waitFor(() => expect(state.lists()).toBe(2)); + const table = screen.getByRole('table', { name: 'S3 objects' }); + expect(within(table).queryByRole('button', { name: 'delete.txt' })).toBeNull(); + expect(within(table).getByRole('button', { name: 'keep.txt' })).toBeVisible(); + expect(within(table).getByRole('button', { name: 'delete.txt.backup' })).toBeVisible(); + await act(async () => state.refreshed.resolve(new Response('refresh unavailable', { status: 503 }))); + expect(screen.getByRole('alert')).toHaveTextContent('refresh unavailable'); + expect(within(table).queryByRole('button', { name: 'delete.txt' })).toBeNull(); + expect(state.fetcher.mock.calls.filter(([, options]) => options?.method === 'DELETE')).toHaveLength(1); +}); + +it.each([403, 503])('retains the selected row after a rejected or unknown delete (%s)', async status => { + const state = await deletion(); + await act(async () => state.removed.resolve(new Response('delete unavailable', { status }))); + expect(screen.getByRole('alert')).toHaveTextContent('delete unavailable'); + expect(screen.getByRole('heading', { level: 1, name: 'delete.txt' })).toBeVisible(); + expect(state.lists()).toBe(1); + expect(state.fetcher.mock.calls.filter(([, options]) => options?.method === 'DELETE')).toHaveLength(1); +}); diff --git a/app/crowdb-web/ui/src/views/S3View.tsx b/app/crowdb-web/ui/src/views/S3View.tsx index dacc02031..db0c6f2b7 100644 --- a/app/crowdb-web/ui/src/views/S3View.tsx +++ b/app/crowdb-web/ui/src/views/S3View.tsx @@ -169,6 +169,7 @@ export function S3View({ active, readonly, onChunk }: { active: boolean; readonl if (action === 'abort') { await listUploads(undefined, target.bucket); return; } if (action === 'create' || action === 'delete-bucket') { if (target.bucket === bucket) root(); await listBuckets(undefined, bucketListedPrefix); } else if (target.bucket === bucket) { + if (action === 'delete-object') setRows(current => current.filter(row => row.key !== target.key)); if (selected?.key === target.key) { setPreview(''); setDetail(null); if (action === 'delete-object') setSelected(null); } await loadObjects(bucket, undefined, listedPrefix); if (action === 'replace' && selected && selected.key === target.key) { diff --git a/doc/design/console/design-crowdb-console-ui.md b/doc/design/console/design-crowdb-console-ui.md index 7c81f4570..023698ce9 100644 --- a/doc/design/console/design-crowdb-console-ui.md +++ b/doc/design/console/design-crowdb-console-ui.md @@ -1295,6 +1295,10 @@ and footer structure; it does not imply reading data pages or SQL querying. use the same menu contract without adding every Object to the left tree. Prefix is a key filter, not a physical folder; Bucket deletion does not silently cascade. Demo controls are not primary browsing actions. Exact bucket/key is visible. + A successful Object deletion immediately removes that exact Bucket/key from + the current object list before its server refresh completes; other rows remain. + Failed or unknown deletion retains the row. A later refresh failure preserves + the confirmed removal and reports the refresh error. Multipart state distinguishes uploaded parts, completion, abort, failure and unknown outcome. Downloads stream; previews read at most the advertised cap. - **S3-05:** Object details include Storage locations below HEAD, 20 extents per @@ -1345,6 +1349,7 @@ S3: Bucket object list (S3-01/02/04): | | | | | | Objects stay in this bounded list. | | | | Right-click Object -> its own resource menu. | | +| | Deleted Object disappears on success, then list refreshes. | | +----------------------------+--------------------------------------------------------------+------------------------------+ ``` diff --git a/doc/working/plan-ci-stability-review.md b/doc/working/plan-ci-stability-review.md index 0fa5429bf..69cf5898f 100644 --- a/doc/working/plan-ci-stability-review.md +++ b/doc/working/plan-ci-stability-review.md @@ -26,8 +26,10 @@ coverage, and identify the remaining work needed to improve CI stability. 修复已推送为 `70ae86b1`，完整 CI 为 26/27 PASS；新发现的 terminal writer 恢复故障已定位为 split 物化与父分区租约转移的冲突，修复和本地复验已完成， 已推送为 `5af2737f`，run `41304660599` attempt 1 已结束，25/27 PASS。 -terminal writer recovery 已远端通过；auxiliary 与六节点部署修复及本地复验已完成， -新的提交和精确 SHA CI 尚待完成。 +terminal writer recovery 已远端通过；auxiliary 与六节点部署修复已推送为 +`18015caf`，run `41311084753` attempt 1 全部 27 个 job 通过。 +后续只读复核发现早期 placement settle 不能排除自动 split，正在补强 +Access Restart 前的实际依赖就绪观察；该测试修正仍需独立发布和精确 SHA CI。 物理容器在恢复前已停止，原因未知；七个容器、镜像和卷身份均保持不变。 ## 当前执行队列 @@ -133,8 +135,10 @@ terminal writer recovery 已远端通过；auxiliary 与六节点部署修复及 - Auxiliary 外部 Stop、OS 进程退出和页面 Failed 均通过，随后 Restart 在 ChunkKV 初始放置的 TransferFencing 窗口返回 409 Waiting。原精确 场景 12.4 秒 PASS；原并发组合 12.1 秒复现；串行对照 13.7 秒 PASS。 - 修复前置条件：沿用已有正常 placement settle，再执行故障，不调整生产 - balance 策略、response/PID/OS/sibling/readiness 断言或 timeout。 + 首次修复沿用 placement settle，后续复核发现其 transfer observation + 不覆盖同一或下一 tick 的自动 split。现改为每次 Access Restart 前直接 + 观察同节点 ChunkKV /ready 与 instance identity，保留三秒观察期限， + 不调整生产 balance 策略、response/PID/OS/sibling/readiness 断言。 - 六节点 Access Waiting 缺少分类，本地另复现带换行的 ChunkKV Waiting 不匹配原正则。monitor 已恢复实际进程，计划仍为 failed。新增两例确定性 回归先红；修正分类后又捕获同 waiting 状态保留旧 detail，导致已登记启动 @@ -149,9 +153,40 @@ terminal writer recovery 已远端通过；auxiliary 与六节点部署修复及 数据组虽配置 concurrency 2，资源预算实际使两例顺序执行，不能声称并发。 - UI 366、plan-hook 28（含两例新回归）、fixture 304、完整 TypeScript、 fmt/full clippy 均通过。七个物理容器及卷身份未变。证据保存在 - `ci/lifecycle-dependencies-fixed/`；新的精确 SHA CI 尚待完成。 + `ci/lifecycle-dependencies-fixed/`；已推送 `18015caf`，对应 run + `41311084753` attempt 1 全部 27 个 job 通过，完整日志保留。 原 16 项人工 review 保持未勾选。 +## Run 41311084753 后：Access Restart 的实际依赖边界 + +- 27/27 PASS，上一轮 auxiliary 和六节点部署失败组均通过；CI merge + checkout 与 HEAD 完整 tree 相同。源码一致不表示本地与 CI 二进制相同。 +- 复核 balance monitor 发现 transfer observation 可报告 within tolerance， + 同一 tick 仍可规划 split。早期 settle 通过不证明后续 Restart 依赖就绪。 + 新辅助函数在两处 Access Restart 前 GET 本机 ChunkKV /ready，核对 + instance_id 与真实 ready，再保留原单次 mutation 和响应断言。 +- 首次新单例因使用不存在的 url 字段报 Invalid URL；已改为 API 实际 + mgmt_url，保留失败证据。修正后精确单例 35.6 秒通过，原并发组合中的 + auxiliary Playwright 14.5 秒通过。完整操作组为 17 PASS / 1 FAIL / 1 未执行。 + 生产镜像输入未变，继续使用 lifecycle-fixed 镜像并核对 source projection。 +- 新失败为 S3 multipart 删除后旧行超过原三秒期限。完整 trace 显示 + DELETE 204 用时 1.279 秒，后续 list 200/KeyCount 0 用时 2.036 秒； + 页面成功删除后仍等待 list 才清除缓存行。精确单例和并发组合对照通过， + 串行对照再次复现同一断言。建议收到成功后立即移除精确 Bucket/key 行， + 再执行原刷新；失败/未知结果保留行，原三秒 E2E 断言不变。 + 用户已同意该行为，先更新了 S3-04 和 ASCII 布局。可控延迟列表回归在 + 旧 UI 复现原行未移除，单行生产修正后三项通过，覆盖精确 key、其他行、 + 刷新失败以及删除拒绝/未知结果。初始测试 provider/字段标签错误单独保留， + 不作为产品 red 证据。新镜像已通过精确 multipart（Playwright 4.6 秒）、 + 原 auxiliary 并发组合（auxiliary 12.7 秒）和全部 19-case 操作组 + （417.230 秒；multipart 再次 4.1 秒通过）。UI 369、TypeScript、 + fmt 和 full clippy 通过；新测试不支持的 exact 选项所致类型错误与 + 修正后的门禁分别保留。新的提交、推送和 27-job CI 尚待完成。 + 证据：`ci/access-readiness-fixed/s3-delete/`、`ci/s3-ui-fixed/`。 +- 物理 observer 的离线回归补充了恢复后固定 35 个进程身份的 fence， + 数据审计期间不再沿用故障目标的宽松重启许可。两项 fence 回归及七项 + closure 接受性检查通过；未派发物理启动、故障、归档或退休。 + ## 通过场景的稳定性改善 用户要求同时处理 PASS 但不稳定的场景。本节记录额外改善与待验证边界， diff --git a/doc/working/plan-physical-node-ci-handoff.md b/doc/working/plan-physical-node-ci-handoff.md index c2427533f..8a13d570b 100644 --- a/doc/working/plan-physical-node-ci-handoff.md +++ b/doc/working/plan-physical-node-ci-handoff.md @@ -11,9 +11,9 @@ Upstream: [Physical Node Test Plan](../dev/physical-node-test-plan.md), Goal: finish CI stability verification and publish CROWDB, then resolve P13 and complete the two-host physical recovery matrix without losing data or evidence. -## 1. Current checkpoint: lifecycle repair locally verified, publication pending +## 1. Current checkpoint: S3 UI repair locally verified, publication pending -Updated 2026-11-01, 21:21 Asia/Shanghai. The user explicitly resumed this plan. +Updated 2026-11-01, 23:10 Asia/Shanghai. The user explicitly resumed this plan. Continue the agreed CI refinement, commit/push, exact-head CI, then physical recovery sequence. No physical fault or retirement has been dispatched on resume. @@ -87,8 +87,11 @@ recovery sequence. No physical fault or retirement has been dispatched on resume existing 16-MiB export limit and was not uploaded; do not claim it was inspected. - Local exact auxiliary passed (12.4 seconds), its original concurrent pair reproduced the same restart 409 (12.1 seconds), and the serial auxiliary - passed (13.7 seconds). Finish normal initial placement before lifecycle faults; - preserve every original response, process, sibling and readiness assertion. + passed (13.7 seconds). The first repair used placement settle; subsequent + source review found that its transfer observation cannot exclude a split + planned in the same or next tick. Use the actual local ChunkKV readiness + immediately before each Access restart. Preserve every original response, + process, sibling and readiness assertion. - The original ordered local partition stopped earlier at KV restart after stopping Group 0's leader. The isolated serial case also failed. A new leader was elected after submission; the authority guard returned 502. Observe the @@ -105,8 +108,44 @@ recovery sequence. No physical fault or retirement has been dispatched on resume The two data-plan cases used concurrency two but resource-aware batching ran them sequentially (12,800 + 6,656 MiB exceed the 16,384-MiB budget). UI 366, plan-hook 28 including two new regressions, fixture 304, TypeScript, - fmt and full clippy passed. New publication and all 27 exact-head CI jobs - remain required before any physical dispatch. + fmt and full clippy passed. Commit 18015caf0cc99978eec99517a3f52b468128c0e2 + was fast-forward pushed; run 41311084753 attempt 1 completed all 27 jobs + successfully. All logs, exact-head/merge source equality and the actual CI + image receipt are retained in ci/published-18015caf/. +- A test-only refinement replaces the insufficient early placement predicate + with a direct, identity-checked ChunkKV /ready probe before the single Access + restart. It retains the three-second observation budget and all mutation + assertions. Its first local attempt failed on an incorrect inventory URL + field; the corrected mgmt_url exact case and original concurrent pair passed. + The full operation group stopped at a separate S3 delete failure: 17 PASS, + one FAIL and one unrun. Preserve every attempt in ci/access-readiness-fixed/. + The current refinement and S3 UI repair need publication and all 27 exact-head + CI jobs before physical dispatch. +- S3 multipart delete's original three-second DOM assertion failed both in the + full group and a fresh serial baseline. The isolated and concurrent-pair + baselines passed. In the full-group trace, DELETE returned 204 in 1.279 seconds; + its subsequent list returned 200 with zero keys in 2.036 seconds. S3View clears + selection on deletion success but retains cached rows until listing finishes. + Correction: remove only the successfully deleted exact object's + current-bucket row before the existing refresh; retain failed/unknown rows. + The user approved that behavior. S3-04 and its ASCII layout were updated + first, followed by a deterministic deferred-list regression. After correcting + its provider/field-label setup, the old UI fails the exact stale-row assertion; + the one-line removal fix passes all three cases, including rejected and + unknown deletes and a failed post-delete list refresh. The original three-second + E2E assertion remains unchanged. Diagnosis, trace timeline and baselines: + ci/access-readiness-fixed/s3-delete/. New image and verification: + ci/s3-ui-fixed/ and image/s3-ui-fixed/. The new image passed the exact multipart + case (4.6-second Playwright), original auxiliary pair (12.7-second auxiliary) + and all 19 native operation cases at default concurrency (417.230 seconds). + Multipart passed again in the group at 4.1 seconds. UI 369, full TypeScript, + fmt and full clippy passed. Initial unsupported Testing Library options failed + type checking after all UI cases passed; that attempt and the corrected gate + are preserved separately. Publication and new exact-head CI remain pending. +- Offline physical observer regressions now require an immutable 35-process + identity fence after recovery and throughout the subsequent full data audit. + A second fence cannot accept a later restart. Both fence regressions and all + seven closure acceptance checks passed; physical resources remain untouched. - Run 41299662435 checked out merge commit e9f241349146046d52c1198cf3f1b0e0dbdbccff. Its complete source tree equals the pushed head's tree 9a7a6a85ba27c2722610d4185912edf67b3c86bf. Local and CI image runtime digests @@ -121,17 +160,17 @@ the user finishes joint review; a PASS must not check a human review box. ## 2. Repository and publication boundary - CROWDB root: /cjdata/cpp/crowdb2. -- Branch: task-fix-build; published HEAD before this lifecycle repair: - 5af2737fd5797a1afd1395eac63427e8fdb892d2. +- Branch: task-fix-build; latest published HEAD: + 18015caf0cc99978eec99517a3f52b468128c0e2. - The four previously local commits were included in the fast-forward push: cdf023d6 (recovery/liveness), 81d07f03 (nearby mirrors), d396a220 (healthy leader quorum lease), and 7a93c4e5 (cleanup/handoff). - The production changes, subsequent test fixes and new split materialization - repair are committed and pushed. The current lifecycle/dependency repair and - working-plan updates are prepared for the next coherent commit. Preserve + repair and lifecycle/dependency repair are committed and pushed. The Access + test precondition and working-plan updates form the next commit. Preserve any new user edits; do not reset or restore. -- Remote task-fix-build was fetched at 70ae86b1 immediately before the last push; - GitHub then confirmed the new exact head 5af2737f. +- Remote task-fix-build was fetched at 5af2737f immediately before the last push; + GitHub then confirmed the new exact head 18015caf. [PR 55](https://github.com/buzzcrow/crowdb/pull/55) is open. Fetch and verify remote state again immediately before the eventual push. - After explicit resume, the agreed order remains: finish local CI stability @@ -180,26 +219,38 @@ All abbreviated paths below are relative to: artifacts, concurrent/serial baselines and deterministic dependency regressions. - ci/lifecycle-dependencies-fixed/: current prepare/exact/operations/browser/data queues, 48-case source-specific closure, quality gates and diff review. +- ci/published-18015caf/: all 27 successful job logs, source equivalence, + CI image receipt and offline physical audit/retirement checks. +- ci/access-readiness-fixed/: the subsequent test-only readiness refinement, + first failed endpoint-field attempt, corrected verification and S3 diagnosis. +- ci/s3-ui-fixed/: UI red/green evidence, corrected gates, exact cases and full + 19-case operation group on the new source-verified image. - The earlier long handoff is preserved at handoff-history/plan-physical-node-ci-handoff-before-rewrite-20261101T062936Z.md. It contains superseded historical instructions, not current authorization. Current local server image: -- Tag: crowdb-node:lifecycle-fixed. -- Source tree: 1516797b67ad493c9bbf9c25048e574e7d994f31. +- Tag: crowdb-node:s3-ui-fixed. +- Source tree: 4b061ce8ab3d74493e975659c79505e5b620b054. - Docker image ID: - sha256:359897a74ffa13d05e6c3bcf6708c7291a2b761e7c7193b706ba436060d75119. + sha256:05bbc6cb53d6aafbc18004f8a2271ddc4708c88ac69d32a6a41fd30e96468c2c. - OCI config digest: - sha256:a35afa03888629de0d172ca9a186f5c67d558c64125b9f075747f11b049b08e0. + sha256:381d352c595863b44b7deaa55c75cf1779ec038b40822d58b39466aeca0bf7b0. - Runtime SHA-256: - c03d3b78cf126c42923cfd91b3e6d03160ae082e77039708afafda465d9a5e26. -- OCI archive: image/lifecycle-dependencies-fixed/crowdb.oci.tar; SHA-256: - 41107c698fb1ff1ddbd51aee05401e797cb4587b3103dcd1145bad6ae83cc020. -- Revision label 5af2737f predates this uncommitted fix; the source-tree label + 84cdaba42a73f087a5b278834cf768bc210b08af2774839166a74ea7e4dd778f. +- OCI archive: image/s3-ui-fixed/crowdb.oci.tar; SHA-256: + 4ab605d4b0f060cc040e5d217d94850bfa88e9f54437b980ce846867f7dac979. +- Revision label 18015caf predates the new fix; the source-tree label records the actual source. Physical hosts have not been upgraded. Setup and retirement require this image, current local verification and the eventual new all-27-job CI PASS. +- The preceding lifecycle-fixed image and its 48-case validation remain in + image/lifecycle-dependencies-fixed/ and ci/lifecycle-dependencies-fixed/. + Its Docker ID begins 359897a7, source tree is + 1516797b67ad493c9bbf9c25048e574e7d994f31 and runtime SHA-256 is + c03d3b78cf126c42923cfd91b3e6d03160ae082e77039708afafda465d9a5e26. + Those 48 cases are not relabeled as new S3-image coverage. - The preceding split-fixed image and its 95-case validation remain in image/split-materialization-fixed/ and ci/split-materialization-fixed/. Its Docker ID begins cec0d2c1 and source tree is diff --git a/e2e-test/clients/console-ui/e2e/fixtures/nativeServiceLifecycle.ts b/e2e-test/clients/console-ui/e2e/fixtures/nativeServiceLifecycle.ts index ade16770e..4bd895773 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/nativeServiceLifecycle.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/nativeServiceLifecycle.ts @@ -8,6 +8,7 @@ import { step } from './stepTimer'; interface NativeServer { id: string; + mgmt_url?: string; node_id: number; pid?: number; pid_node_id?: number; @@ -22,6 +23,25 @@ export async function nativeServerInventory(request: APIRequestContext): Promise return response.json(); } +/** Probe the local dependency immediately before the single Access restart. */ +export async function accessDependencyReady(request: APIRequestContext, node: number) { + const dependency = (await nativeServerInventory(request)).find(row => row.node_id === node && row.service_type === 'chunk-kv'); + expect(dependency).toBeDefined(); + expect(dependency.id).toMatch(/^chunk-kv-\d+$/); + expect(dependency.mgmt_url).toMatch(/^https?:\/\//); + const instance = Number(dependency.id.slice('chunk-kv-'.length)); + const deadline = performance.now() + 3000; + await step('native lifecycle: Access dependency readiness', () => expect.poll(async () => { + const remaining = Math.ceil(deadline - performance.now()); + if (remaining <= 0) return false; + const response = await request.get(new URL('/ready', dependency.mgmt_url).href, { timeout: remaining }); + const health = await response.json(); + expect([200, 503], await response.text()).toContain(response.status()); + expect(health.instance_id).toBe(instance); + return response.status() === 200 && health.ready === true; + }, { timeout: 3000, intervals: [100] }).toBe(true)); +} + async function siblingIdentities(target: string) { const nodes = await Promise.all([0, 1, 2].map(node => nativeServices(node))); return nodes.flatMap((services, node) => Object.entries(services) @@ -57,6 +77,7 @@ async function stopAuxiliary(page: Page, request: APIRequestContext, original: N async function startAuxiliary(page: Page, request: APIRequestContext, original: NativeServer, kind: string, label: string, prefix: string, oldOperation: string) { const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); + if (kind === 'access-server') await accessDependencyReady(request, original.node_id); let submitted = false; try { await sidebar.getByRole('button', { name: 'N-1', exact: true }).click({ button: 'right' }); diff --git a/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts b/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts index f13a31845..b1b9773bc 100644 --- a/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts @@ -6,7 +6,7 @@ import { observeDeployingSteps } from '../fixtures/servicePlanObservation'; import { step } from '../fixtures/stepTimer'; import { observeDefaultDeployments } from '../fixtures/defaultDeployments'; import { nativeProcessAlive } from '../fixtures/nativeProcess'; -import { cycleAuxiliaryService, nativeServerInventory } from '../fixtures/nativeServiceLifecycle'; +import { accessDependencyReady, cycleAuxiliaryService, nativeServerInventory } from '../fixtures/nativeServiceLifecycle'; // Baseline: 9.7s (2026-11-01), original combined 3s assertions on the independent-observation image. // Previous: 20.7s (2026-10-29), including external stop/start observation without Refresh. @@ -22,13 +22,11 @@ test('native diagnostics: auxiliary menus stop and restart the actual typed proc const access = (await list()).find((row: { node_id: number; service_type: string }) => row.node_id === 1 && row.service_type === 'access-server'); const stopped = await request.post(`/api/services/${access.id}/stop`, { data: {} }); expect(stopped.ok(), await stopped.text()).toBe(true); - try { - await expect.poll(() => alive(access.pid), { intervals: [100] }).toBe(false); - await expect(page.locator('header').getByTitle('Cluster health: Failed', { exact: true })).toBeVisible(); - } finally { - const started = await request.post(`/api/services/${access.id}/restart`, { data: {} }); - expect(started.ok(), await started.text()).toBe(true); - } + await expect.poll(() => alive(access.pid), { intervals: [100] }).toBe(false); + await expect(page.locator('header').getByTitle('Cluster health: Failed', { exact: true })).toBeVisible(); + await accessDependencyReady(request, access.node_id); + const started = await request.post(`/api/services/${access.id}/restart`, { data: {} }); + expect(started.ok(), await started.text()).toBe(true); await expect(page.locator('header').getByTitle('Cluster health: Healthy', { exact: true })).toBeVisible(); }); expect(await list()).toHaveLength(18); diff --git a/e2e-test/tests/console/native_browser.rs b/e2e-test/tests/console/native_browser.rs index 7b6c19298..33d56af98 100644 --- a/e2e-test/tests/console/native_browser.rs +++ b/e2e-test/tests/console/native_browser.rs @@ -135,12 +135,7 @@ async fn browse_diskdb() { #[tokio::test] #[ignore = "requires normal storage and owned container PID observations"] async fn auxiliary_process_menus() { - let context = TestNativeContext::connect().await; - super::native_storage::start(&context).await; - // Finish initial placement before independent Stop/Start operations. - super::native_balance::settle_existing(&context).await; - let result = crate::common::control::native_browser().await; - assert_eq!(result["passed"], true); + browse_storage().await; } #[tokio::test] From c063a169f5e400a44c48a5a66aa42fc48609b396 Mon Sep 17 00:00:00 2001 From: Gian Date: Mon, 2 Nov 2026 00:58:58 +0800 Subject: [PATCH 17/24] Split asynchronous reads across block extents --- doc/working/plan-ci-stability-review.md | 19 ++- doc/working/plan-physical-node-ci-handoff.md | 124 +++++++++++++--- lib/crowdb-tree/CMakeLists.txt | 4 +- .../backend/local/block_async_page_store.cpp | 73 +++++++++- .../unit/block_async_page_store_test.cpp | 135 ++++++++++++++++++ 5 files changed, 322 insertions(+), 33 deletions(-) create mode 100644 lib/crowdb-tree/tests/unit/block_async_page_store_test.cpp diff --git a/doc/working/plan-ci-stability-review.md b/doc/working/plan-ci-stability-review.md index 69cf5898f..67b052421 100644 --- a/doc/working/plan-ci-stability-review.md +++ b/doc/working/plan-ci-stability-review.md @@ -63,7 +63,7 @@ Access Restart 前的实际依赖就绪观察；该测试修正仍需独立发 260.002 秒、512 条完整回读 0.273 秒；12 partitions、三个 owners。 最终 fmt/full clippy、受影响 C++ format/tree-lint、TypeScript 和 scope guard 通过。新增断言均有独立实际运行证据，不能与旧 bundle 的全量混算。 -- [~] **提交、推送并核对远端 CI**：CROWDB `task-fix-build` 一个一致提交； +- [x] **提交并推送修复**：CROWDB `task-fix-build` 一个一致提交； 记录精确 SHA、run/attempt 和全部展开 job，修复真实失败后推进。 已推送 `76fc6508189dd31018e953674a8f1c693011498d`，新 run `41295426521` attempt 1 已结束：27 项中 24 成功、3 失败。CI merge checkout 与 head @@ -73,8 +73,21 @@ Access Restart 前的实际依赖就绪观察；该测试修正仍需独立发 失败；上一轮三个修复场景已远端通过。全部日志和校验过的失败 artifact 保留于 `ci/published-70ae86b1/`。此前完整日志与三个 artifact 保留在 `ci/published-76fc6508/`，不并入原 16 项人工 review。 -- [ ] **恢复物理验证**：远端 CI 核对后继续 P13 和 P14–P19，保留 native - 崩溃的新镜像验证及历史数据审计；当前物理资源和失败证据继续保留。 + 最新修复已推送为 `1814405e`；本地新镜像 19-case 操作组、UI 369 和 + fmt/clippy/TypeScript 全部通过，提交钩子通过。 +- **CI 检测已按用户要求停止**：不再查询 `1814405e` 的 run `41320020131`， + 不推断最新结果，保留已收集的证据；不作为物理工作的前置。 +- [~] **恢复物理验证**：用户取消新增卷归档。B 重启后已连通并核对主机身份， + 七个原始停止容器及卷身份均通过检查。镜像传输及 SHA-256 校验通过； + A 使用 manifest ID、B 使用 config ID，已通过内容哈希及完整层列表证明一致。 + 五个当前节点已用原卷升级，完整历史审计通过，包括五端点 S3/Iceberg SF1、 + 187 份 Dataset 报告及全部见证/删除记录。625 次观察通过，35 个身份不变。 + 新 P09 在 A2 Group 0 复现真实的跨块异步短读，已停止五个节点保留原卷， + P09–P19 尚未完成。最小回归重现完全相同的 958464/1212416 字节错误； + 跨块读取修复、19 项 focused 检查及完整 C++/FFI、core、storage、fmt、 + tree-lint、clippy 门禁均通过；发布和新镜像物理验证继续进行。 + 这不是新增 CI job 失败，不并入原 16 项共同 review。 + A 已完成的三个备份和全部原始失败证据继续保留，不再新增备份。 ## Run 41295426521：首次发布后的三个新失败 diff --git a/doc/working/plan-physical-node-ci-handoff.md b/doc/working/plan-physical-node-ci-handoff.md index 8a13d570b..7135782e8 100644 --- a/doc/working/plan-physical-node-ci-handoff.md +++ b/doc/working/plan-physical-node-ci-handoff.md @@ -11,11 +11,72 @@ Upstream: [Physical Node Test Plan](../dev/physical-node-test-plan.md), Goal: finish CI stability verification and publish CROWDB, then resolve P13 and complete the two-host physical recovery matrix without losing data or evidence. -## 1. Current checkpoint: S3 UI repair locally verified, publication pending - -Updated 2026-11-01, 23:10 Asia/Shanghai. The user explicitly resumed this plan. -Continue the agreed CI refinement, commit/push, exact-head CI, then physical -recovery sequence. No physical fault or retirement has been dispatched on resume. +## 1. Current checkpoint: cross-extent read repair locally verified + +Updated 2026-11-02, 00:55 Asia/Shanghai. The earlier repair is published as +1814405e4dd1fe0fcd64f9f18ac7c71e98d64fd2. The user's latest instructions are +"不要检测CI" and "归啥档，测试能过就好了". CI inspection is stopped; do not +query its latest state or require it before physical work. No additional volume +archives are required or authorized by this plan. Preserve existing evidence +and A's three completed archives (2.835 GiB); these are not a cleanup prerequisite. + +B rebooted and is reachable as cj7960 at 192.168.31.184. All seven original +stopped container/volume identities passed the fresh guard. Image transfer and +archive SHA-256 verification passed. The first ID check failed because A exposes +the manifest digest as its Docker ID while B exposes the config digest. The +continuation verifies the manifest-to-config hash, identical platform, labels +and all rootfs layers. See image/s3-ui-fixed/resume-image-verified.json. Upgrade +the five current containers with their original volumes and recorded per-host +image IDs; do not create additional backups. The five-node same-volume upgrade +passed. Groups 0/1/3 recovered in at most 2.995 seconds, catalog in 5.727 seconds +and all services in 25.547 seconds, each within its original 180-second bound. +Complete retained history passed: three 512-record sets, the 4096-record set, +five Group-3 endpoints, baseline allocation, five-endpoint S3/Iceberg SF1, +187 Dataset reports, witnesses and acknowledged deletions. The audit observer +passed 625 complete samples (maximum gap 3.823 seconds), with all 35 identities +fixed. Fresh P09 found a production cross-extent read defect; P09–P19 remain +unfinished. All five current nodes are stopped for repair, with all seven +original volumes retained and no new archives or cleanup. + +- The first startup observer rejected monitor timestamps while managed services + were still launching. P02 requires fresh observations at recovered readiness; + it does not require unavailable recovery health to be Ready. The corrected + observer records transitional timestamps only for explicitly faulted nodes, + keeps all unaffected process identities fixed, and requires all five fresh + snapshots when recovery seals and for the complete audit. Four transition + checks, two immutable-identity-fence tests and seven acceptance-gate checks + passed. The original failure and stop marker remain preserved under + observations/final-image-upgrade/; start-audited reuses successful recovery + gates without restarting nodes or replaying mutations. +- The first fresh P09 recovered all groups, catalog and services within their + original bounds, but the sampler discarded complete node observations when + a deliberately stopping process disappeared from `/proc`. The fixed collector + records absent/unreadable process IDs alongside live siblings and cgroup data; + loss of any unaffected or audit-fenced process still fails. Four focused + checks passed, including an actual-container collection. Original evidence + is retained under observations/final-P09/. The new fault/witness identity is + final-P09-sampling-fixed; the ordered queue is final-queue-sampling-fixed/. +- That second P09 attempt exposed A2 Group 0's latched tree I/O fault after + cold startup. At global address 133259264, asynchronous reading requested + 1212416 bytes but returned 958464, exactly reaching the 128-MiB global + boundary between 64-MiB block files. `BlockAsyncPageStore::submit_read` + mapped only the first fd, while synchronous reads and asynchronous writes + already split extents. A deterministic reopened-store regression reproduces + the identical short read. The fix chains reads across the mapped extents, + completing once after all parts or the first real error; no retry or lock was + added. Cross-extent, missing/truncated-tail and existing async tests pass + (19 focused tests). Complete C++/FFI, core and storage verification, fmt, + clean changed-file tree-lint and full clippy all passed. Publish this repair, + build a fresh verified image and resume retained-volume physical verification. + Evidence: first-failures/P09-retained-engine-unhealthy/ and + repair/cross-extent-read-fixed/. Do not dispatch P10–P19 on the old image. +- The new deployment controller is harness/physical_storage_repair.py with + explicit scope/image/upgrade/start stages and immutable per-stage receipts. + It checks the published source, quality source hashes and image identity; + no CI result or new volume archive is a prerequisite. Its frozen runtime + inputs belong under image/cross-extent-read-fixed/. The new ordered queue + is final-queue-cross-extent-read-fixed/, using cross-extent-P09–P13 labels. + Keep both earlier failed P09 controllers and their original logs. - The first resume scope check failed because all five current physical nodes are stopped. All seven container/image/volume identities remain unchanged. @@ -161,22 +222,21 @@ the user finishes joint review; a PASS must not check a human review box. - CROWDB root: /cjdata/cpp/crowdb2. - Branch: task-fix-build; latest published HEAD: - 18015caf0cc99978eec99517a3f52b468128c0e2. + 1814405e4dd1fe0fcd64f9f18ac7c71e98d64fd2. - The four previously local commits were included in the fast-forward push: cdf023d6 (recovery/liveness), 81d07f03 (nearby mirrors), d396a220 (healthy leader quorum lease), and 7a93c4e5 (cleanup/handoff). - The production changes, subsequent test fixes and new split materialization - repair and lifecycle/dependency repair are committed and pushed. The Access - test precondition and working-plan updates form the next commit. Preserve + repair, lifecycle/dependency repair, Access test precondition and S3 UI repair + are committed and pushed. Preserve any new user edits; do not reset or restore. -- Remote task-fix-build was fetched at 5af2737f immediately before the last push; - GitHub then confirmed the new exact head 18015caf. +- Remote task-fix-build was fetched at 18015caf immediately before the last push; + GitHub then confirmed the new exact head 1814405e. [PR 55](https://github.com/buzzcrow/crowdb/pull/55) is open. Fetch and verify remote state again immediately before the eventual push. -- After explicit resume, the agreed order remains: finish local CI stability - gates; commit/push CROWDB; inspect the new exact-head-SHA CI and every expanded - job; then resume P13 and the physical matrix. Physical completion is not a - prerequisite for that CROWDB push. +- Earlier stability gates and their authorized push are complete. The latest user + instruction stops CI inspection and resumes the physical matrix. CI evidence + remains separate from physical PASS; preserve actual source epochs and failures. - Loader and website publication are outside scope. Last recorded loader HEAD was local 447a1e619550bbdc0150139976e0c442675f92b0, with origin/main at 4cafdc9227871f55942a2b215ccb5a3078ade70d; website had local c9306c51. @@ -225,6 +285,9 @@ All abbreviated paths below are relative to: first failed endpoint-field attempt, corrected verification and S3 diagnosis. - ci/s3-ui-fixed/: UI red/green evidence, corrected gates, exact cases and full 19-case operation group on the new source-verified image. +- ci/published-1814405e/: publication, source equality, background CI logs, + ci-wait-deferred.json and the B-host connection failure. Independent A backup + progress is recorded in independent-A-archives.json. - The earlier long handoff is preserved at handoff-history/plan-physical-node-ci-handoff-before-rewrite-20261101T062936Z.md. It contains superseded historical instructions, not current authorization. @@ -421,7 +484,11 @@ The user has resumed; proceed in the order below. subject, no body/trailers/doc references/requirement numbers. Fetch and verify the remote before the authorized fast-forward push; do not force-push. No Sirius-specific pre-push rules apply to this repository. -- [~] **Inspect new exact-head CI**: run 41295426521 attempt 1 ended 24 PASS / 3 FAIL. +- **CI inspection stopped by user**: do not inspect run 41320020131 or infer + its latest result. Retain only already collected evidence. This supersedes + the historical inspection instructions in this section. + Run 41311084753 passed all 27 on 18015caf. + Earlier run 41295426521 attempt 1 ended 24 PASS / 3 FAIL. The three fixes and local affected regressions passed and were pushed as 70ae86b1. Run 41299662435 attempt 1 ended 26 PASS / 1 FAIL. Its production split materialization repair now has 198 affected component PASS and 95 @@ -444,7 +511,21 @@ while production sources still match. Every command and original error is durabl ## 6. Physical checkpoint and remaining work -Physical work is deferred until the CI/publication stage above is complete. +The user explicitly stopped CI inspection and additional volume archives. +Earlier stability gates and publication are complete. The additional storage +repair passed local gates and is ready for publication. Physical work continues +using the original volumes, with exact ownership checks and complete evidence. + +- [x] **Image upgrade**: B connectivity and identical image content are verified. + All five current containers were recreated with their original volumes, + mounts, environment and limits. Seven volume identities remain unchanged. + A and B expose different Docker IDs for the same manifest/config content; + image/s3-ui-fixed/resume-image-verified.json records both. +- [~] **Cross-extent storage repair**: deterministic red/green regression and + all local quality gates passed. Publish the fix, build/verify the new image, + use the original volumes, complete its read-only history audit, then execute + fresh P09–P16 and the scoped P17–P19 closure. Earlier-image passes retain + their original source identities. - Hosts: A 192.168.31.129 (A1/A2/A3) and B 192.168.31.184 (B1/B2). Existing SSH uses BatchMode/IdentitiesOnly. Verify host identity/clocks on resume. @@ -479,13 +560,10 @@ Physical work is deferred until the CI/publication stage above is complete. history returns after B recovery. Keep this harness boundary distinct from production repair and retain the complete post-recovery historical audit. -- [ ] **Archive current physical data before image upgrade**: existing private - archives are complete/tested/hashed but predate later P12/P13 data. Make fresh - private archives of every still-existing volume before any new upgrade or - retirement. Archive helper containers must use --log-driver none so binary - tar data cannot fill Docker JSON logs. Keep incomplete old streams invalid. - Transfer/verify the final production image and preserve exact mounts/limits. - Do not blindly replay resume_repaired_image.py or old witness mutation IDs. +- **Additional archives canceled by user**: preserve existing A archives and + all logs. Upgrade and retire using exact resource ownership/identity checks, + without creating more volume archives. Preserve the original failure receipts; + do not blindly replay resume_repaired_image.py or old witness mutation IDs. - [ ] **Close unexpected P12 A3 identity change**: A3 ChunkKV changed PID451/gen2 to PID66135/gen3, stderr Resource deadlock avoided, no usable core. Local Rust-FFI gdb proves callback-final-owner destruction can join its own diff --git a/lib/crowdb-tree/CMakeLists.txt b/lib/crowdb-tree/CMakeLists.txt index d8b2ebe2f..26c54fce6 100644 --- a/lib/crowdb-tree/CMakeLists.txt +++ b/lib/crowdb-tree/CMakeLists.txt @@ -100,7 +100,9 @@ include(GoogleTest) file(GLOB_RECURSE CROWDB_TREE_TEST_SRC CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/tests/*.cpp) if(NOT (LIBURING_INCLUDE_DIR AND LIBURING_LIB)) - list(REMOVE_ITEM CROWDB_TREE_TEST_SRC ${CMAKE_CURRENT_SOURCE_DIR}/tests/unit/reactor_test.cpp) + list(REMOVE_ITEM CROWDB_TREE_TEST_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/tests/unit/reactor_test.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/tests/unit/block_async_page_store_test.cpp) endif() list(APPEND CROWDB_TREE_TEST_SRC ${CMAKE_CURRENT_SOURCE_DIR}/../crowdb-common/cpp/tests/test_main.cpp) add_executable(crowdb_tree_tests ${CROWDB_TREE_TEST_SRC}) diff --git a/lib/crowdb-tree/src/backend/local/block_async_page_store.cpp b/lib/crowdb-tree/src/backend/local/block_async_page_store.cpp index 167e63faf..502bd7de5 100644 --- a/lib/crowdb-tree/src/backend/local/block_async_page_store.cpp +++ b/lib/crowdb-tree/src/backend/local/block_async_page_store.cpp @@ -10,9 +10,12 @@ #include #include +#include #include #include #include +#include +#include #include #include @@ -23,22 +26,51 @@ namespace { // Maps a DiskIOUring callback's raw CQE `res` (>=0 bytes transferred, <0 // -errno) to a Status, treating a short read/write (res >= 0 but < the -// requested `len`) as an io_error too -- a single-shot op per -// submit_read/write call, unlike the synchronous stores' internal -// retry-until-full-length loop; a later phase can add that retry if real -// usage needs it. +// requested `len`) as an io_error too. Each extent is submitted once; +// crossing a block boundary does not retry a short read in the first file. Status result_to_status(int res, size_t len, const char *op) { if (res < 0) { return Status::io_error(std::string(op) + ": " + std::strerror(-res)); } - if (static_cast(res) < len) { + if (std::cmp_less(res, len)) { return Status::io_error(std::string("short ") + op + " (" + std::to_string(res) + " of " + std::to_string(len) + " bytes)"); } return Status::Ok(); } +struct ExtentRead +{ + int fd; + void *buffer; + size_t length; + uint64_t offset; +}; + +struct ExtentReads : std::enable_shared_from_this +{ + ::crowdb::common::DiskIOUring *uring = nullptr; + std::vector parts; + AsyncCompletion completion; + + void submit(size_t index) + { + const auto &part = parts[index]; + uring->submit_read(part.fd, part.buffer, part.length, static_cast(part.offset), + [state = shared_from_this(), index, length = part.length](int result) { + Status status = result_to_status(result, length, "read"); + if (!status.ok() || index + 1 == state->parts.size()) { + if (state->completion) { + state->completion.complete(std::move(status)); + } + return; + } + state->submit(index + 1); + }); + } +}; + // RAII aligned buffer for O_DIRECT writes via io_uring. std::vector memory // is not guaranteed IU-aligned; this wraps posix_memalign + memcpy. class AlignedIoBuf @@ -92,6 +124,35 @@ BlockAsyncPageStore::BlockAsyncPageStore(BlockPageStore *store, ::crowdb::common uint64_t BlockAsyncPageStore::submit_read(PageAddr addr, void *buf, size_t len, AsyncCompletion on_complete) { + if (len == 0 || len > std::numeric_limits::max() - addr) { + if (on_complete) { + on_complete.complete(len == 0 ? Status::Ok() : Status::invalid_argument("read range overflow")); + } + return 0; + } + const uint64_t block_size = store_->block_size(); + if (block_size > 0 && len > block_size - (addr % block_size)) { + auto state = std::make_shared(); + state->uring = uring_; + state->completion = on_complete; + size_t done = 0; + while (done < len) { + uint64_t local = 0; + int fd = store_->fd_for_offset(addr + done, &local); + if (fd < 0) { + if (on_complete) { + on_complete.complete(Status::io_error("BlockAsyncPageStore: no fd for offset")); + } + return 0; + } + size_t chunk = std::min(static_cast(len - done), block_size - local); + state->parts.push_back( + {.fd = fd, .buffer = static_cast(buf) + done, .length = chunk, .offset = local}); + done += chunk; + } + state->submit(0); + return 0; + } uint64_t local = 0; int fd = store_->fd_for_offset(addr, &local); if (fd < 0) { @@ -188,7 +249,7 @@ uint64_t BlockAsyncPageStore::submit_write(PageAddr addr, const void *buf, size_ state->pending++; uring_->submit_write(fd, write_buf, chunk, static_cast(local), [state, chunk](int res) { - if (res < 0 || static_cast(res) < chunk) { + if (res < 0 || std::cmp_less(res, chunk)) { Status s = result_to_status(res, chunk, "write"); if (state->first_error.ok()) { state->first_error = std::move(s); diff --git a/lib/crowdb-tree/tests/unit/block_async_page_store_test.cpp b/lib/crowdb-tree/tests/unit/block_async_page_store_test.cpp new file mode 100644 index 000000000..0a386cd0d --- /dev/null +++ b/lib/crowdb-tree/tests/unit/block_async_page_store_test.cpp @@ -0,0 +1,135 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "async_completion_adapter.h" +#include "crowdb-common/diskio_uring.h" +#include "crowdb-tree/backend/async_page_store.h" +#include "crowdb-tree/backend/block_page_store.h" +#include "test_tmp.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace crowdb::tree; + +namespace +{ +crowdb::common::DiskIOUring make_uring() +{ + crowdb::common::Topology topology; + crowdb::common::PipelineConfig pipeline; + pipeline.entries = 256; + pipeline.mode = crowdb::common::PollingMode::Hybrid; + topology.pipelines.push_back(pipeline); + return crowdb::common::DiskIOUring(std::move(topology)); +} + +bool wait_for(const std::atomic &completed) +{ + for (unsigned i = 0; i < 200; ++i) { + if (completed.load(std::memory_order_acquire) != 0) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return completed.load(std::memory_order_acquire) != 0; +} +} // namespace + +TEST(BlockAsyncPageStore, CrossExtentReadAfterReopenMatchesSync) +{ + struct ReadRange + { + uint64_t block_size; + uint64_t offset; + size_t length; + }; + + for (const auto range : { + ReadRange{.block_size = 64 * 1024 * 1024, .offset = 133259264, .length = 1212416}, + ReadRange{.block_size = 128 * 1024 * 1024, .offset = 133259264, .length = 1212416}, + ReadRange{.block_size = 8192, .offset = 4096, .length = 16384 }, + ReadRange{.block_size = 8192, .offset = 8192, .length = 12288 } + }) { + SCOPED_TRACE(::testing::Message() << range.block_size << ':' << range.offset << '+' << range.length); + crowdb::tree_test::TempDir directory; + auto uring = make_uring(); + std::unique_ptr store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 0, 0, range.block_size, 4096, &store).ok()); + std::vector expected(range.length); + for (size_t i = 0; i < expected.size(); ++i) { + expected[i] = static_cast(((i / 4096) + i) % 251); + } + ASSERT_TRUE(store->write_at(range.offset, expected.data(), expected.size()).ok()); + ASSERT_TRUE(store->sync().ok()); + store.reset(); + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 0, 0, range.block_size, 4096, &store).ok()); + for (int fd : store->all_extent_fds()) { + uring.register_fd(fd); + } + std::unique_ptr output(std::aligned_alloc(4096, range.length), &std::free); + ASSERT_NE(output, nullptr); + ASSERT_TRUE(store->read_at(range.offset, static_cast(output.get()), range.length).ok()); + ASSERT_TRUE(std::equal(expected.begin(), expected.end(), static_cast(output.get()))); + std::fill_n(static_cast(output.get()), range.length, 0); + BlockAsyncPageStore asynchronous(store.get(), &uring); + std::atomic completed{0}; + Status status; + asynchronous.submit_read(range.offset, output.get(), range.length, + crowdb::tree::detail::own_async_completion([&](Status result) { + status = std::move(result); + completed.fetch_add(1, std::memory_order_release); + })); + ASSERT_TRUE(wait_for(completed)); + ASSERT_TRUE(status.ok()) << status.to_string(); + EXPECT_EQ(completed.load(), 1U); + EXPECT_TRUE(std::equal(expected.begin(), expected.end(), static_cast(output.get()))); + } +} + +TEST(BlockAsyncPageStore, CrossExtentReadRejectsMissingAndTruncatedTail) +{ + for (bool missing : {false, true}) { + SCOPED_TRACE(missing); + crowdb::tree_test::TempDir directory; + auto uring = make_uring(); + std::unique_ptr store; + ASSERT_TRUE(BlockPageStore::open_blocks(directory.path, 0, 0, 8192, 4096, &store).ok()); + std::vector expected(12288, 0xA5); + ASSERT_TRUE(store->write_at(4096, expected.data(), expected.size()).ok()); + if (missing) { + ASSERT_TRUE(store->delete_block(1).ok()); + } + else { + uint64_t local = 0; + ASSERT_EQ(::ftruncate(store->fd_for_offset(8192, &local), 4096), 0); + } + for (int fd : store->all_extent_fds()) { + uring.register_fd(fd); + } + std::unique_ptr output(std::aligned_alloc(4096, expected.size()), &std::free); + ASSERT_NE(output, nullptr); + BlockAsyncPageStore asynchronous(store.get(), &uring); + std::atomic completed{0}; + Status status; + asynchronous.submit_read(4096, output.get(), expected.size(), + crowdb::tree::detail::own_async_completion([&](Status result) { + status = std::move(result); + completed.fetch_add(1, std::memory_order_release); + })); + ASSERT_TRUE(wait_for(completed)); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find(missing ? "no fd for offset" : "short read (4096 of 8192 bytes)"), + std::string::npos); + EXPECT_EQ(completed.load(), 1U); + } +} From 9cc2982664c6099f6fa02822ea3f1b4b1908e6c9 Mon Sep 17 00:00:00 2001 From: Gian Date: Mon, 2 Nov 2026 07:12:04 +0800 Subject: [PATCH 18/24] Keep monitor health responsive during authority reads --- container/crowdb-monitor/src/node/runtime.rs | 1 + .../crowdb-monitor/src/node/runtime/health.rs | 52 +- .../src/node/runtime/health/intents.rs | 96 +++ .../src/node/runtime/services.rs | 1 + .../src/node/runtime/services/execution.rs | 21 + .../src/node/runtime/services/test_support.rs | 8 +- .../tests/common/node_health.rs | 133 ++++ .../tests/node_health_authority_test.rs | 40 ++ .../tests/node_service_replacement_test.rs | 37 ++ doc/dev/physical-node-test-plan.md | 615 ++++++++++-------- doc/doc_index.md | 2 +- doc/working/plan-ci-stability-review.md | 37 +- doc/working/plan-physical-node-ci-handoff.md | 81 ++- 13 files changed, 826 insertions(+), 298 deletions(-) create mode 100644 container/crowdb-monitor/src/node/runtime/health/intents.rs create mode 100644 container/crowdb-monitor/tests/common/node_health.rs create mode 100644 container/crowdb-monitor/tests/node_health_authority_test.rs diff --git a/container/crowdb-monitor/src/node/runtime.rs b/container/crowdb-monitor/src/node/runtime.rs index 94bd094a0..ea54bcfe8 100644 --- a/container/crowdb-monitor/src/node/runtime.rs +++ b/container/crowdb-monitor/src/node/runtime.rs @@ -195,6 +195,7 @@ pub async fn run_node_with_inputs( } Ok::<(), Box>(()) }.await; + health.stop().await; drop(endpoint_recovery); initialization.stop().await; setup.stop().await; diff --git a/container/crowdb-monitor/src/node/runtime/health.rs b/container/crowdb-monitor/src/node/runtime/health.rs index 3b1f57158..05723786c 100644 --- a/container/crowdb-monitor/src/node/runtime/health.rs +++ b/container/crowdb-monitor/src/node/runtime/health.rs @@ -10,12 +10,15 @@ use crate::{ StatusStore, }; +mod intents; + pub(super) struct NodeHealth { store: StatusStore, status: MonitorStatus, attempts: BTreeMap, probe_failures: BTreeMap, probes: crate::ProbeExecutor, + intents: intents::ServiceIntents, } impl NodeHealth { @@ -27,6 +30,7 @@ impl NodeHealth { attempts: BTreeMap::new(), probe_failures: BTreeMap::new(), probes: crate::ProbeExecutor::new(true)?, + intents: intents::ServiceIntents::default(), }) } @@ -118,17 +122,19 @@ impl NodeHealth { profile: &DeploymentProfile, processes: &mut ProcessManager, ) -> Result<()> { - if !profile.paths.data_root.join("node-binding.json").exists() { - return Ok(()); + if let Some(intents) = self.intents.poll(profile).await { + self.apply_services(profile, processes, &intents).await?; } - let intents = match super::services::desired(profile).await { - Ok(intents) => intents, - Err(error) => { - tracing::debug!(%error, "service recovery awaits committed intents"); - return Ok(()); - } - }; - for intent in &intents { + Ok(()) + } + + async fn apply_services( + &mut self, + profile: &DeploymentProfile, + processes: &mut ProcessManager, + intents: &[crowdb_protocol::mgmt::node::NodeServiceIntent], + ) -> Result<()> { + for intent in intents { let pending = super::services::pending(profile, intent, processes)?; let restart_unhealthy = if !pending && matches!( @@ -143,17 +149,15 @@ impl NodeHealth { if !pending && !restart_unhealthy { continue; } - if let Err(error) = super::services::dependencies_ready(profile, intent, &intents).await { + if let Err(error) = super::services::dependencies_ready(profile, intent, intents).await { tracing::debug!(service = %intent.service_id, %error, "service recovery waits for dependencies"); continue; } if restart_unhealthy { - let process_id = if intent.kind == "kv" { - "kv" - } else { - &intent.service_id - }; - processes.stop(process_id, Duration::from_secs(10)).await?; + if let Err(error) = super::services::stop_unhealthy(profile, processes, intent).await { + tracing::debug!(service = %intent.service_id, %error, "unhealthy service recovery awaits current authority"); + continue; + } } match super::services::execute(profile, processes, intent.clone()).await { Ok(_) => { @@ -166,6 +170,20 @@ impl NodeHealth { Ok(()) } + pub(super) async fn stop(&mut self) { + self.intents.stop().await; + } + + #[cfg(feature = "test-util")] + pub(in crate::node::runtime) async fn reconcile_services_for_tests( + &mut self, + profile: &DeploymentProfile, + processes: &mut ProcessManager, + ) -> Result<()> { + let intents = super::services::desired(profile).await?; + self.apply_services(profile, processes, &intents).await + } + async fn restart_if_unhealthy( &mut self, profile: &DeploymentProfile, diff --git a/container/crowdb-monitor/src/node/runtime/health/intents.rs b/container/crowdb-monitor/src/node/runtime/health/intents.rs new file mode 100644 index 000000000..dab29c1d5 --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/health/intents.rs @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Read-only authority work cannot hold up current local health observations. + +use std::time::{Duration, Instant}; + +use crowdb_protocol::mgmt::node::{NodeBinding, NodeServiceIntent}; + +use super::super::{durable, services, Result}; +use crate::DeploymentProfile; + +struct Observation { + binding: NodeBinding, + started: Instant, + intents: Vec, +} + +#[derive(Default)] +pub(super) struct ServiceIntents { + task: Option>>, +} + +impl ServiceIntents { + pub(super) async fn poll(&mut self, profile: &DeploymentProfile) -> Option> { + let binding = match durable::read::(&profile.paths.data_root.join("node-binding.json")) { + Ok(Some(binding)) => binding, + Ok(None) => { + self.stop().await; + return None; + } + Err(error) => { + tracing::debug!(%error, "service intent observation awaits a valid local binding"); + return None; + } + }; + let mut ready = None; + if self + .task + .as_ref() + .is_some_and(tokio::task::JoinHandle::is_finished) + { + let task = self.task.take()?; + match task.await { + Ok(Ok(observation)) + if observation.binding == binding + && observation.started.elapsed() <= Duration::from_secs(3) => + { + ready = Some(observation.intents); + } + Ok(Ok(_)) => tracing::debug!("discarded expired or rebound service intent observation"), + Ok(Err(error)) => tracing::debug!(%error, "service recovery awaits committed intents"), + Err(error) => { + tracing::warn!(%error, "service intent worker failed; next health pass starts another read"); + } + } + } + if self.task.is_none() { + let profile = profile.clone(); + self.task = Some(tokio::spawn(async move { + let started = Instant::now(); + let intents = services::desired(&profile).await?; + if durable::read::(&profile.paths.data_root.join("node-binding.json"))?.as_ref() + != Some(&binding) + { + return Err("service intent binding changed during observation".into()); + } + Ok(Observation { + binding, + started, + intents, + }) + })); + } + ready + } + + pub(super) async fn stop(&mut self) { + if let Some(task) = self.task.take() { + task.abort(); + if let Err(error) = task.await { + if !error.is_cancelled() { + tracing::warn!(%error, "service intent worker stopped unexpectedly"); + } + } + } + } +} + +impl Drop for ServiceIntents { + fn drop(&mut self) { + if let Some(task) = &self.task { + task.abort(); + } + } +} diff --git a/container/crowdb-monitor/src/node/runtime/services.rs b/container/crowdb-monitor/src/node/runtime/services.rs index c34ec18e7..7fd9da347 100644 --- a/container/crowdb-monitor/src/node/runtime/services.rs +++ b/container/crowdb-monitor/src/node/runtime/services.rs @@ -32,6 +32,7 @@ mod observation; mod policy; mod receipt; pub(super) use execution::execute; +pub(super) use execution::stop_unhealthy; pub(super) use observation::observe; pub(super) fn retained(profile: &DeploymentProfile) -> Result> { diff --git a/container/crowdb-monitor/src/node/runtime/services/execution.rs b/container/crowdb-monitor/src/node/runtime/services/execution.rs index a489c62ac..5d4381c48 100644 --- a/container/crowdb-monitor/src/node/runtime/services/execution.rs +++ b/container/crowdb-monitor/src/node/runtime/services/execution.rs @@ -7,6 +7,27 @@ use crowdb_protocol::mgmt::node::{NodeBinding, NodeServiceAction, NodeServiceInt use serde_json::{json, Value}; use std::time::Duration; +pub(in crate::node::runtime) async fn stop_unhealthy( + profile: &DeploymentProfile, + processes: &mut ProcessManager, + intent: &NodeServiceIntent, +) -> Result<()> { + let binding: NodeBinding = + durable::read(&profile.paths.data_root.join("node-binding.json"))?.ok_or("node is unbound")?; + if binding.node_id != intent.node_id || binding.bootstrap.cluster_id != intent.cluster_id { + return Err("service intent belongs to another node or cluster".into()); + } + tokio::time::timeout(Duration::from_secs(3), super::super::verify_binding(&binding)).await??; + verify_intent(&binding, intent).await?; + let id = if intent.kind == "kv" { + "kv" + } else { + &intent.service_id + }; + processes.stop(id, Duration::from_secs(10)).await?; + Ok(()) +} + pub(in crate::node::runtime) async fn execute( profile: &DeploymentProfile, processes: &mut ProcessManager, diff --git a/container/crowdb-monitor/src/node/runtime/services/test_support.rs b/container/crowdb-monitor/src/node/runtime/services/test_support.rs index 8eb61c74f..6f72a17c1 100644 --- a/container/crowdb-monitor/src/node/runtime/services/test_support.rs +++ b/container/crowdb-monitor/src/node/runtime/services/test_support.rs @@ -91,6 +91,12 @@ impl NodeServiceForTests { Ok(()) } + /// # Errors + /// Rejects superseded recovery observations before stopping an owned process. + pub async fn stop_unhealthy_for_tests(&mut self, intent: &NodeServiceIntent) -> Result<()> { + super::stop_unhealthy(&self.profile, &mut self.processes, intent).await + } + /// # Errors /// Propagates production authority, process and readiness errors. pub async fn execute_for_tests(&mut self, intent: NodeServiceIntent) -> Result { @@ -108,7 +114,7 @@ impl NodeServiceForTests { self.health .as_mut() .ok_or("missing test health worker")? - .reconcile_services(&self.profile, &mut self.processes) + .reconcile_services_for_tests(&self.profile, &mut self.processes) .await } diff --git a/container/crowdb-monitor/tests/common/node_health.rs b/container/crowdb-monitor/tests/common/node_health.rs new file mode 100644 index 000000000..fb910fb87 --- /dev/null +++ b/container/crowdb-monitor/tests/common/node_health.rs @@ -0,0 +1,133 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{extract::State, http::StatusCode, routing::get, Router}; +use crowdb_monitor::{DeploymentProfile, MonitorStatus, NodeServiceForTests, StatusStore}; +use crowdb_protocol::mgmt::{node::NodeBinding, SystemBootstrapIdentity}; +use std::{ + os::unix::fs::PermissionsExt, + path::Path, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; + +pub struct TestHealth { + pub worker: NodeServiceForTests, + pub ready: Arc, + pub entered: Arc, + profile: DeploymentProfile, + servers: Vec>, + _directory: crowdb_test_harness::test_dirs::TestDir, +} + +impl TestHealth { + pub async fn new() -> Self { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-health-authority"); + let root = directory.path(); + let ready = Arc::new(AtomicBool::new(true)); + let entered = Arc::new(tokio::sync::Notify::new()); + let (health, health_task) = health_server(Arc::clone(&ready)).await; + let (authority, authority_task) = slow_authority(Arc::clone(&entered)).await; + let profile = profile(root, &health); + let binding = NodeBinding { + node_id: 2, + bootstrap: SystemBootstrapIdentity { + cluster_id: uuid::Uuid::new_v4().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + configuration_digest: "00".repeat(32), + }, + management_seeds: vec![authority], + }; + let binding_path = root.join("node-binding.json"); + std::fs::write(&binding_path, serde_json::to_vec(&binding).unwrap()).unwrap(); + std::fs::set_permissions(binding_path, std::fs::Permissions::from_mode(0o600)).unwrap(); + let mut worker = NodeServiceForTests::new_for_tests(profile.clone()).await.unwrap(); + worker.start_kv_child_for_tests().await.unwrap(); + Self { + worker, + ready, + entered, + profile, + servers: vec![health_task, authority_task], + _directory: directory, + } + } + + pub fn snapshot(&self) -> MonitorStatus { + StatusStore::open(&self.profile.paths.run_root) + .unwrap() + .read(Duration::from_secs(3)) + .unwrap() + } +} + +impl Drop for TestHealth { + fn drop(&mut self) { + for server in &self.servers { + server.abort(); + } + } +} + +fn profile(root: &Path, health: &str) -> DeploymentProfile { + let mut profile = + DeploymentProfile::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml")) + .unwrap(); + profile.paths.data_root = root.to_path_buf(); + profile.paths.log_root = root.join("log"); + profile.paths.run_root = root.join("run"); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + std::fs::create_dir_all(&profile.paths.run_root).unwrap(); + for service in &mut profile.services { + if matches!(service.id.as_str(), "web" | "kv") { + service.program = "/bin/sleep".into(); + service.args = vec!["60".into()]; + service.probe.target = format!("{health}/health"); + service.additional_probes.clear(); + } + } + profile +} + +async fn health_server(ready: Arc) -> (String, tokio::task::JoinHandle<()>) { + async fn health(State(ready): State>) -> StatusCode { + if ready.load(Ordering::Acquire) { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + } + } + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + axum::serve( + listener, + Router::new().route("/health", get(health)).with_state(ready), + ) + .await + .unwrap(); + }); + (origin, task) +} + +async fn slow_authority(entered: Arc) -> (String, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + let app = Router::new().route( + "/topology", + get(move || { + let entered = Arc::clone(&entered); + async move { + entered.notify_one(); + std::future::pending::().await + } + }), + ); + axum::serve(listener, app).await.unwrap(); + }); + (origin, task) +} diff --git a/container/crowdb-monitor/tests/node_health_authority_test.rs b/container/crowdb-monitor/tests/node_health_authority_test.rs new file mode 100644 index 000000000..45c39438d --- /dev/null +++ b/container/crowdb-monitor/tests/node_health_authority_test.rs @@ -0,0 +1,40 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/node_health.rs"] +mod fixture; + +use crowdb_monitor::MonitorPhase; +use fixture::TestHealth; +use std::{sync::atomic::Ordering, time::Duration}; + +#[tokio::test] +async fn slow_service_authority_does_not_freeze_current_local_health() { + let mut fixture = TestHealth::new().await; + tokio::time::timeout(Duration::from_secs(1), fixture.worker.tick_for_tests()) + .await + .expect("service authority must not block local health publication") + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), fixture.entered.notified()) + .await + .expect("the production authority read was actually started"); + let healthy = fixture.snapshot(); + assert_eq!(healthy.phase, MonitorPhase::Ready); + assert_eq!(healthy.services.len(), 2); + assert!(healthy.services.values().all(|service| service.healthy)); + + fixture.ready.store(false, Ordering::Release); + tokio::time::timeout(Duration::from_secs(1), fixture.worker.tick_for_tests()) + .await + .expect("an in-flight authority read must not hide changed local health") + .unwrap(); + let unavailable = fixture.snapshot(); + assert_eq!(unavailable.phase, MonitorPhase::Restarting); + assert!(unavailable.services.values().all(|service| !service.healthy)); + assert!(unavailable.revision > healthy.revision); + for (id, service) in &unavailable.services { + assert_eq!(service.pid, healthy.services[id].pid); + assert_eq!(service.restart_attempts, 0); + } + fixture.worker.stop_kv_for_tests().await.unwrap(); +} diff --git a/container/crowdb-monitor/tests/node_service_replacement_test.rs b/container/crowdb-monitor/tests/node_service_replacement_test.rs index 3a908fa0f..9b1161d9b 100644 --- a/container/crowdb-monitor/tests/node_service_replacement_test.rs +++ b/container/crowdb-monitor/tests/node_service_replacement_test.rs @@ -68,3 +68,40 @@ async fn supervised_kv_replacements_and_legacy_receipts_refresh_the_pid_without_ worker.stop_kv_for_tests().await.unwrap(); health.abort(); } + +#[tokio::test] +async fn superseded_recovery_observation_cannot_stop_the_current_process() { + let mut fixture = TestReplacement::new().await; + let mut current = fixture.intent.clone(); + current.operation_id = uuid::Uuid::new_v4().to_string(); + current.action = crowdb_protocol::mgmt::node::NodeServiceAction::Stop; + fixture + .authority + .kv + .put( + 0, + 0, + b"/deployment/services/2/kv-2", + &serde_json::to_vec(¤t).unwrap(), + None, + ) + .await + .unwrap(); + let before = std::fs::read(&fixture.state).unwrap(); + let error = fixture + .worker + .stop_unhealthy_for_tests(&fixture.intent) + .await + .unwrap_err(); + assert!(error.to_string().contains("superseded"), "{error}"); + assert_eq!( + fixture + .worker + .owned_processes_for_tests(¤t.cluster_id, current.node_id) + .unwrap()["processes"]["kv"], + fixture.original + ); + assert_eq!(std::fs::read(&fixture.state).unwrap(), before); + fixture.worker.stop_kv_for_tests().await.unwrap(); + fixture.health.abort(); +} diff --git a/doc/dev/physical-node-test-plan.md b/doc/dev/physical-node-test-plan.md index 33969ca4b..003ffd9c2 100644 --- a/doc/dev/physical-node-test-plan.md +++ b/doc/dev/physical-node-test-plan.md @@ -3,269 +3,352 @@ # Physical Node Test Plan -Validate five-node deployment, data correctness, sustained load, and recovery -across two physical hosts. This is a reusable plan. Store each run's results -separately; previous passes do not establish that a new run passed. - -Upstream: [deployment and recovery](../design/deploy/design-crowdb-deploy.md), -[OCI build and runtime](../../container/oci-image/README.md), -[container acceptance](../../e2e-test/README.md), and -[KV recovery](../design/kv/design-crowdb-kv-slot.md). - -## 1. Default environment and boundaries - -- Physical host A: `192.168.31.129`. -- Physical host B: `192.168.31.184`. -- Verify host identity, current interface, and address before each run. - A LAN IP locates a host but does not establish resource ownership. -- Use existing SSH configuration and Ed25519 authentication; verify - noninteractive `BatchMode`. Never record login usernames, passwords, private - keys, access tokens, or application secrets in this plan or committed reports. - Supply credentials through local SSH configuration/private inputs; redact logs. -- Use five fixed voters: A1/A2/A3 on A, B1/B2 on B. Port offsets are `0/1/2` - on A and `0/1` on B. Each node has a separate persistent volume and identity. - These are two physical host/rack failure domains, not five hosts. -- Use host networking, hard limits of `1 CPU / 2048 MiB` per container, and - one Tokio worker per service. Record cgroup limits and any deviations. -- Simulate host failure by stopping this run's containers on that host; record - the scope. Do not change shared firewalls, interfaces, routes, kernel settings, - or the Docker daemon. -- A reduced three-node run must explicitly declare A1/A2/B1 and three voters. - It cannot substitute for five-node coverage. -- Record each retained EC layout's fragment distribution and actual loss budget. - Three live KV voters do not establish payload availability. For example, - EC 8+4 with six fragments on B cannot reconstruct that data while B is offline. - Keep this limit separate from the acknowledged-data audit after B returns. - -## 2. Execution contract - -- Run every build, test, and executable through `pixi run`, including remotely. -- Create a unique run ID. Store scripts, source snapshots, receipts, clients, - logs, and results in `.crowdb-runtime/artifacts/physical-nodes//`. - Inventory volume ownership separately. Do not clean shared runtime directories - during an active run. Check both hosts' free disk space and memory first. -- No repository task currently implements this entire plan. Save a harness - using current public interfaces, its source, and actual commands. Do not depend - on earlier `/tmp` files, hardcoded node UUIDs, or deleted artifacts. Do not - present this plan as an existing one-command Pixi task. -- Finish shared builds before consumers; never build concurrently into the same - output directory. Freeze source during OCI builds. Record commit, dirty diff, - and relevant source identities. -- Reuse images only when service, dependency, and source inputs match. Import - the same verified OCI on both hosts; compare execution configuration and - ordered rootfs layers. Distinguish OCI manifest digests from Docker image IDs. -- Label resources with run ownership; verify it before faults, recreation, and - cleanup. -- Execute phases in order. Capture the first failure and stop dependent work. - After repair, rerun affected cases and resume unfinished work; retain valid - passes whose inputs have not changed. -- Do not increase timeouts, weaken assertions, or add caller retries to pass. - Poll only defined readiness/recovery conditions. Never replay timed-out - mutations; audit uncertain writes through read-only calls. -- Use `PASS / FAIL / NOT RUN`. An untriggered fault is a coverage gap even if a - later request succeeds. Any failed or unrun required case prevents all-pass. - -## 3. Deployment and baseline data - -- [ ] **P00 Environment, authentication, and image**: verify SSH key access, - clocks, Docker/cgroups, free space, management interfaces, cross-host listener - reachability, and existing resource owners. Save redacted inventory, source - identity, build receipts, and same-image import verification on both hosts. -- [ ] **P01 Five-node discovery and initialization**: form five voters through - normal startup policy, discovery, key admission, and production storage - preparation. Verify five distinct discovery UUIDs/node IDs, accepted - `initial_voters = 5`, fixed membership, and real listener addresses. Apply - offsets exactly once; check same-host conflicts and cross-host loopback. - Currently `run-node.py --initial-voters` accepts only `1/3`; use the supported - explicit node-policy/controlled container path for five voters and verify - Monitor's accepted result. Do not invent a working `--initial-voters 5` command. -- [ ] **P02 Whole-cluster readiness**: verify seven managed services per node, - 35 total, including counts, live processes, health, and dependencies. All five - Web endpoints must report Ready. Monitor observations must postdate this - container start and be no older than three seconds; old snapshots are invalid. - Measure age when receiving the snapshot and record that receipt time. Verify - current DiskDB ownership leases; live owners must renew expiring leases - without changing their process epoch. -- [ ] **P03 Baseline data and allocation**: check five-replica mappings for - Group 0, metadata Group 1, and explicit data Group 3. Wait for current leaders/ - leases before single-attempt operations. Verify exact KV single/batch writes; - deterministic S3 objects of at least 96 KiB with byte/hash checks; Iceberg - namespace/table/property commits and metadata locations; and native ChunkDB - allocation/query of the same ID. Cross-read through all five endpoints. Use - `crowdb-tpc-loader` for generated Iceberg and standalone S3 data, plus generated - Dataset samples with inline fields, multipart payloads, and complete readback. - Record nodes, groups, objects, and acknowledgments without application secrets. - -## 4. Write scale and 30 minutes of sustained load - -- [ ] **P04 Short 512-record reproduction**: use native ChunkKV defaults. - Write 32 batches of 16 records, each 64 KiB: 512 records/32 MiB across five - owners. Read the complete catalog first; generate run-unique keys within - actual ranges and verify routed owners. Exclude reserved S3, Iceberg, and - Dataset metadata prefixes; appending a test suffix to a protocol key still - corrupts that protocol's scan namespace. Select safe ranges covering all five - owners, or fail preparation. Uniform first bytes do not establish - coverage. Test the old key shape separately and report its coverage. - Preserve the default five-second deadline per batch. Verify every mutation - acknowledgment and full readback; save catalog split/cutover state, latency, - and results. Audit failed/uncertain writes read-only without replay. -- [ ] **P05 Larger working set**: after P04 passes, seed a fresh namespace with - at least 4096 deterministic 64 KiB records, 256 MiB total, across five Serving - owners. Budget both mirrors, metadata, WAL, reclamation, and artifacts; retain - at least 30% free test-media capacity. Expand only owned test media or fail - preparation if insufficient; do not silently shrink the working set. Record - key counts, acknowledged bytes, and verification digests, not just API success. -- [ ] **P06 Sustained workload**: run **30 continuous minutes** on P05's data, - excluding builds, deployment, and idle waiting. Mix native ChunkKV inserts, - overwrites, deletes, and readback across both hosts and all five owners. - Concurrently exercise S3 reads/writes/deletes, Iceberg commit/load/cleanup, and - Dataset sample reads/publication. Declare concurrency, ratios, pacing, and - working-set limits before starting. Continuously produce successful operations - while reusing/reclaiming bounded data; a fixed load followed by waiting does - not qualify. -- [ ] **P07 Soak data verification**: journal each mutation's operation ID, - key/version, expected length/hash, acknowledgment state, and timestamp. - Serialize overwrites per key or use an explicit version contract to prevent - verifier races. Continuously sample acknowledged data; finish with a complete - audit of live keys and expected deletions. Journal timeout/connection-loss as - `OutcomeUnknown` and audit it, rather than declaring it uncommitted. Any - mismatch, request failure, OOM, or unexpected process replacement fails the - normal load phase. -- [ ] **P08 Load observations**: sample throughput, success/failure/unknown - counts, p50/p95/p99/max latency, CPU, RSS/cgroup memory, free space, service - PIDs, catalog, and reclamation at least every five seconds. Report acknowledged - operations/bytes, actual active duration, and resource changes. Live containers - alone do not prove a pass. Keep different load-stage latencies separate. - -## 5. Faults and recovery using the soak data - -Before each case, save identities, PIDs, catalog, data inventory, and replica -status. After recovery, verify P03's baseline, all P07 data, and fresh -acknowledged writes from the current case. - -- [ ] **P09 Retained-volume recreation**: recreate all five containers with - original volumes, startup policies, and accepted identities. Verify unchanged - UUIDs/node IDs, 35 recovered services, readback through every endpoint, and - all replicas caught up. Repeat affected recovery after production image fixes. -- [ ] **P10 Unresponsive child process**: send `SIGSTOP` only to one verified - ChunkKV child. Monitor must recover it with a new PID within 60 seconds; the - other 34 service PIDs must remain unchanged. Verify old data and fresh - single-attempt operations. Clear injected faults on failure paths too. -- [ ] **P11 Cross-host KV lifecycle**: from a Web endpoint with an available - remote leader, stop/restart its local follower. Separately stop the current - leader. Verify Stop receipts, actual termination, continued remote management, - a new PID, and no collateral restarts. A completed Stop receipt must remain - readable after the operation stops the former Group-0 leader. -- [ ] **P12 Both B nodes offline**: stop B1/B2; A retains three-voter quorum. - Verify single-attempt KV writes/reads and catalog takeover onto surviving - Serving owners within 180 seconds. While B stays offline, submit fresh S3 - writes and Iceberg property commits through A, then cross-read. Use new - witnesses per fault to prove new writes occurred, and verify those witnesses - through all three A endpoints while B remains offline. Record temporary - unavailability of older EC data when the stopped fragments exceed its actual - loss budget; do not claim rack protection for such a layout. Restore B, then recreate - B1/B2 individually with retained volumes. Check original identities, all data - through five endpoints, and all five replicas caught up. -- [ ] **P13 All three A nodes offline**: stop A1/A2/A3; B loses quorum. - Observe current unavailability within 30 seconds; writes must not acknowledge - success. Audit unknown outcomes after recovery without asserting absence. - Restore A; verify every service, five-replica checkpoints, and complete data, - not just a new leader. -- [ ] **P14 Allocation-only safe retry**: inject bounded Group-1 leadership - step-down while preserving KV processes and the Group-0 lease. Within three - seconds, verify the leader accepted at least one step-down. Observe explicit - allocation-level `Unavailable` safe retry. After release, complete same-ID - allocation/query within the original five-retry/ten-second limit. Record - actual retry evidence and original limits; success alone is insufficient. - Use the production client; instrumentation must not change branches or policy. -- [ ] **P15 Large backlog and full cold restart**: use normal acknowledged - writes to leave a controlled offline KV replica more than 1024 slots behind. - Record group, start/end checkpoints, and actual gap. Restore it and verify - bounded-concurrency FetchGap/leader recovery progress, then cold-start all - five retained-volume nodes. Fix acknowledged targets for Groups 0/1/3; - require all five replicas per group to reach them with healthy engines and - exact data. Do not move targets with ongoing writes. Report actual coverage - of reclaimed WAL, empty sealed WAL, high promise/old chosen, and other recovery - boundaries; component regressions do not prove physical coverage. -- [ ] **P16 Final recovery gate**: require all catalog entries Serving, all five - endpoints Ready, and 35 healthy services. The next sample must show the same - normal PIDs. Audit all data; complete a fresh KV batch and fresh ChunkDB - allocation/query. Fully scan owned DiskDB occupancy read-only; verify no - cross-chunk overlap among live mirrored segments and no scan truncation. - -Measure recovery budgets independently: catalog takeover 180 seconds, service -recovery 180 seconds, and fixed-checkpoint catch-up 180 seconds per group. -Record each condition's own start and elapsed time. Whole-phase duration cannot -prove individual deadlines. Preserve shorter production-client limits. - -## 6. Failure evidence, repairs, and cleanup - -- [ ] **P17 Failure capture**: print failing cases, operations, and complete - errors. Use `CROWDB_E2E_PRINT_SERVICE_ERRORS=1` while debugging. Save client - output, service logs, current inspect data, exit/OOM state, health freshness, - KV roles/leases/checkpoints, catalog/transfers, DiskIO availability, and disk - usage. Export allowlisted/redacted fields without credentials. -- [ ] **P18 Repair verification**: locate the first divergence and root cause. - Add focused regressions and run affected component tests plus fmt/clippy. - Rebuild/verify images when production inputs change. Rerun affected cases - under the same resources and limits. After repairing P04, complete P05–P08; - 512 records alone do not finish testing. Repairs or harness restarts during - the soak require a complete new 30-minute run after the fix. -- [ ] **P19 Freeze and cleanup**: stop owned containers before freezing full - service logs. Delete only inventoried containers, volumes, and transfer files - with verified ownership. Check owned ports are closed on both hosts, no owned - resources remain, and SSH key access still works. Retain run artifacts. - Do not prune the daemon or clean other tasks' resources. Capture failures - before cleanup. Before retiring failed retained data, privately archive the - stopped owned volumes and verify archive integrity and checksums. A Docker - archive helper streaming binary tar output must use `--log-driver none` so - the daemon does not duplicate the archive into container JSON logs. - -## 7. Deliverables and the next run - -Save these artifacts and link them from the final summary: - -- `run.json`: run ID, source/image identities, both IPs, node/port/resource - mapping, ownership inventory, actual commands, and harness source locations. - No login credentials. -- `results.json`: P00–P19 states, times, limits, errors, evidence paths, and - unrun reasons. Preserve first failures separately from repaired results. -- `data-ledger.jsonl`, `metrics.jsonl`, and `checkpoints.json`: acknowledged/ - uncertain outcomes, load metrics, fixed targets, and actual catch-up. - Segment large files if needed. -- `cleanup.json` and `summary.md`: cleanup verification, actual passing - coverage, unresolved defects, and next starting point. Distinguish "run ended" - from "all required tests passed." CI does not replace physical results. -- `loader/`: loader source identity, generation/injection/verification commands - and reports, Iceberg tables/snapshots, S3 inventory, immutable Dataset snapshots, - and sample verification summaries. Confirm implementation and acceptance of - all three targets before use; mark unimplemented paths `NOT RUN`. - -Suggested request for a future run: - -> Execute the full five-node test in `doc/dev/physical-node-test-plan.md` with -> the two LAN hosts and existing SSH keys. Run continuously for 30 minutes. -> Resolve open issues first, capture and repair failures, then resume. -> Do not reduce coverage or relax limits. - -## 8. Implementation and regression entry points - -- Startup/images: `container/oci-image/{run-node,runtime,artifact,transfer}.py`, - `container/single-node-container/entrypoint.sh`, and - `container/crowdb-monitor/src/node/startup/policy.rs`. -- Formation/observations: `lib/crowdb-protocol/src/mgmt/node/`, - `app/crowdb-web/src/node/`, - `lib/crowdb-console-shared/src/ops/cluster/readiness/`, and - `container/crowdb-monitor/src/node/runtime/`. -- Data/recovery: `lib/crowdb-{kv,kv-client,chunk-kv-client,chunkdb-client}/`, - `app/crowdb-chunkdb/src/topology/`, and - `lib/crowdb-chunk-stream/src/production_chunk/`. -- Component regressions: - `lib/crowdb-kv/tests/group_test/{recovery_backlog,fetch_gap_durability}_test.rs`, - `lib/crowdb-chunk-stream/tests/production_chunk_test/finalized_empty.rs`, - `container/crowdb-monitor/tests/{member_discovery,node_service_receipt}_test.rs`, - and `app/crowdb-chunkdb/tests/{allocation_retry,topology_availability}_test.rs`. -- Container/protocol regressions: `e2e-test/tests/deployment/`, - `e2e-test/tests/console/`, `e2e-test/tests/access_iceberg/`, and corresponding - S3/native SDK acceptance entries. Select tests for actual changes and use - `pixi run`. Distinguish component, container, and real cross-host coverage. +这是可复用的物理测试用例目录。按“大类 → 用例 → 场景”组织，不记录本轮 +完成状态。每轮选择的场景、实际命令、源码/镜像身份和结果保存在独立 run +目录；历史 PASS 不能直接归给新镜像。 + +上游：[部署与恢复](../design/deploy/design-crowdb-deploy.md)、 +[OCI 构建与运行](../../container/oci-image/README.md)、 +[容器验收](../../e2e-test/README.md)、 +[KV 恢复](../design/kv/design-crowdb-kv-slot.md)。 + +## 1. 执行约定 + +### 环境与范围 + +- 默认两个物理故障域：A `192.168.31.129`，B `192.168.31.184`。 + 每轮重新核对主机身份、接口和地址；IP 不是资源归属证明。 +- A1/A2/A3、B1/B2 共五个固定 voter，分别使用独立持久卷和发现身份。 + A 的端口 offset 为 0/1/2，B 为 0/1；按当前启动接口核对实际接受值。 +- 每个容器固定 host network、1 CPU、2048 MiB、512 PIDs；每个服务一个 + Tokio worker。记录实际 cgroup 限制、存储容量、预留量和偏差。 +- 仅通过停止本轮指定主机上的容器模拟主机故障。不改变共享网络、路由、 + 防火墙、内核或 Docker daemon。三节点变体必须另立 profile，不能代替五节点。 +- 使用现有 SSH key 和非交互 BatchMode。凭据通过私有输入提供，不把密码、 + 私钥、token、登录身份或带认证参数的命令写入提交的文档/报告。 +- 所有构建、测试和可执行程序，包括远端命令，都通过 `pixi run`。 + +### 用例范围与结果 + +- P00–P19 保留为父用例编号；`.1`、`.2` 等是可分别核验的子场景。 + 默认主流程覆盖全部父用例及其普通场景。明确标为“扩展”的场景仅在所选 + profile 中要求执行；未选择不表示它已经通过。 +- 启动前冻结用例范围、并发、资源、数据规模、超时和验收条件。新增回归单独 + 记录；不能在失败后删去场景、放宽断言或改 profile 来取得 all-pass。 +- 每个子场景记录前置条件、注入对象/时刻、实际动作、断言、耗时、结果、证据 + 路径和输入身份。结果使用 `PASS / FAIL / NOT RUN`，未触发的注入计覆盖缺口。 +- 按依赖顺序运行，首错停止依赖它的后续工作。保留原失败和已通过结果；修复后 + 只重跑输入或行为受影响的场景，再继续未完成项。不要盲目重放部分执行的阶段。 +- mutation 在调用前落盘 operation/key/ID 和预期内容，成功后落盘 acknowledgment。 + 超时或断连记为 `OutcomeUnknown`，只读查证；测试控制器不得重放不确定写入。 +- 不增加 timeout，不削弱校验，不添加调用方重试。P14 仅观察生产 client 原有 + 明确安全重试策略，不为测试改变它。 +- 构建先于消费者；同一输出目录不并发构建。源码冻结后构建 OCI，保留 source + tree、commit/dirty diff、客户端摘要和镜像摘要。源码变更后重新判断受影响范围。 +- CI 与物理结果分别记录。用户要求不检测 CI 时，不查询、不推断新 run，也不把 + 等待 CI 作为继续物理测试的前置；本地受影响门禁仍须完成。 + +### 所有恢复用例共同遵守的断言 + +- 故障前保存容器 ID、卷名/CreatedAt、发现 UUID、node ID、bootstrap binding、 + 35 个服务的 container/PID/start-time 身份、catalog 和副本状态。 +- 连续观察覆盖“注入 → 恢复 → 完整数据审计”。完整采样间隔不超过五秒； + 单次缺少 `/proc` 的故障目标仍记录为 absent，不能丢掉整次兄弟服务样本。 +- 只允许该场景指定的进程变化。故障恢复期间，仍然存活的 monitor 应持续提供 + 新鲜的真实观察；故障目标离线可以不可观察，不能把旧 Ready 当作当前证据。 + 观察新鲜度和服务健康是两个断言，Unavailable 不能靠刷新旧健康时间戳伪装。 +- 恢复门禁必须取得所有节点当前、健康、晚于本容器启动且接收时不超过三秒的 + monitor 观察。此时固定全部 35 个进程身份，直到完整审计结束；不能再次设置 + 基线来容忍第二次重启。 +- 分别测量 catalog 接管 180 秒、服务恢复 180 秒、Groups 0/1/3 每组固定 + checkpoint catch-up 180 秒。每个门禁有自己的起点和耗时，不用整个阶段耗时 + 代替。P10 的 60 秒、P13 的 30 秒及生产 client 的更短限制另行保留。 +- 恢复后核对 P03 基线、P04/P05 数据、P07 已确认存活/删除记录和所有历史 + witness，并验证本场景的新写入。经全部五个恢复后的入口读取，完整校验内容。 +- EC 的实际 fragment 分布决定 payload 可用性；KV quorum 不能替代它。临时 + 超出丢失预算时的可用性结论，与恢复后的完整历史数据正确性分别报告。 + +## 2. 大类 A：主机、镜像与集群形成 + +### P00 环境、认证与镜像 + +- **P00.1 主机与资源**：验证两台真实主机、SSH key、时钟、接口、跨主机 + listener 可达性、Docker/cgroup、空闲磁盘和内存。清点已有资源的归属，记录 + 五节点限制与存储预算；准备不足即失败，不静默缩小数据集。 +- **P00.2 同一镜像**：两台主机导入同一已校验的镜像传输文件。校验传输摘要、 + manifest 引用的 config 内容哈希、平台、源码/runtime labels 和有序 rootfs + layers。Docker 可能分别显示 manifest ID 和 config ID，不能仅比较 ID 字符串。 +- **P00.3 拒绝错误输入**：离线回归覆盖损坏传输文件、源码/runtime label 不匹配、 + layer 顺序不一致和架构不匹配；失败时不能开始升级节点。 +- **证据**：脱敏主机清单、镜像/构建 receipts、两台主机实际 inspect、来源映射。 + +### P01 五节点发现、初始化与身份 + +- **P01.1 正常形成**：通过正常生产 policy、发现、key admission 和存储准备形成 + 五个 voter。验证五个不同 discovery UUID/node ID、实际 accepted initial_voters、 + 固定成员和真实 listener；使用当前支持的五 voter 路径，不杜撰 CLI 参数。 +- **P01.2 地址与端口**：逐节点核对 offset 只应用一次、同主机无冲突、跨主机 + 不使用对方不可达的 loopback，以及两个 rack/host 的归属。 +- **P01.3 保留身份**：已有节点重启只恢复原 binding、卷和身份，不重新 bootstrap。 + 过期绑定、外来卷和相同名称下被替换的容器必须拒绝采用。 +- **证据**：accepted policy、成员/监听地址、卷映射、bootstrap 和发现身份。 + +### P02 全集群 readiness 与持续观察 + +- **P02.1 基线**：五个 Web 入口均 Ready，每节点七个服务、合计 35 个真实存活 + 进程；同时验证健康、依赖和 DiskDB 当前 ownership lease 的持续续约。 +- **P02.2 新鲜度**：记录接收时刻，拒绝早于容器启动或年龄超过三秒的 monitor + 快照。区分进程存活、listener 存活和数据服务 Ready。 +- **P02.3 authority 退化**：Group0 切换/不可用或一个远端观察变慢时，本地 monitor + 仍能报告当前进程和真实健康。服务意图读取不能拖住本地健康发布或诊断控制。 + 不允许旧 healthy 值只更新时间；不因 readiness 暂退就重启仍存活的服务。 +- **P02.4 观察器负例**：回归旧快照、错误容器/PID/start-time、OOM、兄弟进程 + 丢失、超过五秒完整采样间隔，以及恢复后第二次身份变化；均必须失败。 +- **证据**：完整原始样本、各门禁 receipt、恢复身份 fence 和观察器结果。 + +## 3. 大类 B：原生数据、协议数据与规模 + +### P03 五入口基线数据和分配 + +- **P03.1 KV**：验证 Groups 0/1/3 的五副本映射和当前 leader/lease。执行单次 + single/batch 写入，从五入口读回完整 key/value；记录 acknowledgment。 +- **P03.2 ChunkDB**：生产 native client 分配并查询同一个 Chunk ID，核对位置、 + 大小和身份，不以一个新 ID 的成功查询替代原分配结果。 +- **P03.3 S3**：经 `crowdb-tpc-loader` 生成独立 S3 SF1 数据，并保存每个对象的 + URI、长度和摘要；经五入口完整读回八张表对应文件。另保留至少 96 KiB 的 + run-unique witness，核对完整字节和 hash。 +- **P03.4 Iceberg**：独立生成八表 SF1，核对 namespace/table UUID、snapshot、 + metadata location、manifest 文件集合、文件长度/摘要、所有行解码和独立扫描。 + schema、属性提交和数据文件分别验证；只读 footer 或十行 sample 不能代替全读。 +- **P03.5 Dataset**：生成含 inline fields、multipart payload 和不可变 sample + snapshot 的数据。逐份报告、逐字段、经五入口完整读取；核对实际报告数。 +- **P03.6 协议隔离**：原生压测 key 必须位于实际安全范围，避开 S3/Iceberg/ + Dataset 的保留元数据前缀；给协议 key 添加测试后缀仍可能破坏协议扫描。 +- **证据**：数据/ack ledger、loader 来源和命令、全部生成报告、五入口校验结果。 + +### P04 512 条短复现与五 owner 覆盖 + +- **P04.1 旧 key 形态**：独立运行历史 512 条复现，记录其真实 owner 覆盖； + 不把它自动计为五 owner 场景。 +- **P04.2 正式形态**：完整读取 catalog，在五个 Serving owner 的真实范围选择 + run-unique 安全 key。32 批 × 16 条 × 64 KiB，共 512 条/32 MiB；保留每批 + 默认五秒 deadline。统一首字节分布不能代替路由 owner 证明。 +- **P04.3 完整性**：逐批记录确认与未知结果，完整回读；保存 split/cutover、 + 各阶段延迟和错误。未知写入只读查证，不重放。 + +### P05 更大工作集与持久存储边界 + +- **P05.1 规模**：P04 通过后，新 namespace 至少 4096 条确定性 64 KiB 记录， + 共 256 MiB，覆盖五个实际 Serving owner。预算 mirrors、metadata、WAL、 + reclamation 和 artifacts，测试 media 保持至少 30% 空闲。 +- **P05.2 extent 边界**：根据实际 block 大小构造跨文件尾部的持久数据，冷打开后 + 触发异步读取跨一个及多个 extent。核对完整内容和一次 completion；记录实际 + offset/length/边界，不能只因工作集大就声称覆盖该路径。 +- **P05.3 错误传播**：组件回归覆盖缺失后续 extent、尾部截断和范围溢出，保留 + 真实 I/O 错误，拒绝短读成功、重复 completion 或自动重试掩盖错误。 +- **证据**：owner/key/字节数、全量摘要、实际 extent 路径和组件/物理覆盖分别列明。 + +## 4. 大类 C：30 分钟混合负载与最终一致性 + +### P06 持续混合负载 + +- **P06.1 参数**：开始前声明并发、比例、pacing、key/version 规则和有界数据量。 + 负载覆盖两个主机和五个 owner，复用/回收数据时仍持续产生成功操作。 +- **P06.2 原生与协议并发**：混合 native 插入、覆盖、删除、读回；同时执行 S3 + 读写删、Iceberg commit/load/cleanup 和 Dataset publication/readback。 +- **P06.3 有效时长**：至少 30 连续分钟成功混合工作重叠，不含构建、部署、空等。 + 固定 seed 后等待不算负载。修复或控制器重启打断负载后重新完成连续 30 分钟。 + +### P07 确认语义和数据审计 + +- **P07.1 ledger**：mutation 前后保存 operation、key/version、预期长度/hash、 + ack 状态和时间；同 key 覆盖串行化或有明确版本合同，避免 verifier 自身竞态。 +- **P07.2 在线抽查与收尾全查**：连续读取已确认数据，结束时核对所有 live keys、 + 覆盖后的值、完整协议 payload、witness，以及所有已确认删除的内容仍然不存在。 +- **P07.3 未知结果**：超时/断连保存 OutcomeUnknown；恢复后存在时验证预期内容， + 不能直接断言不存在，也不能为消除未知而重放 mutation。 +- 正常负载中的请求失败、内容不符、OOM 或未计划进程替换均使该负载阶段失败。 + +### P08 性能与资源观测 + +- **P08.1 完整采样**：至少每五秒记录吞吐、成功/失败/未知数、p50/p95/p99/max、 + CPU、RSS/cgroup memory、空闲空间、进程身份、catalog 和 reclamation。 +- **P08.2 分段统计**：分别列 build、启动、seed、持续负载、故障和审计耗时； + 保留成功操作/字节、有效重叠时长和不同阶段延迟，不混算或仅报告容器存活。 +- **P08.3 主机异常**：保存 diskstats、I/O/memory pressure、cgroup throttling/ + OOM 和有权限读取的内核错误。低 CPU/充足内存不能排除 fsync 或设备 I/O 停顿。 + +## 5. 大类 D：进程、容器与主机故障 + +### P09 原卷冷启动与镜像升级 + +- **P09.1 冷重建**：停止并重建全部五个容器，沿用原卷、policy、UUID/node ID + 和 bootstrap。验证 35 服务、所有固定 checkpoint 和五入口完整历史数据。 +- **P09.2 同卷升级**：生产修复后验证新镜像内容身份，再替换容器；所有卷的名称/ + CreatedAt 和数据身份不变。镜像传输 tar 是分发文件，不是数据卷备份。 +- **P09.3 操作竞态**：重建前再次核对旧容器 ID、stopped 状态/时间、挂载和归属； + 若准备期间曾被其他操作启动/替换，则停止，不盲目重复 remove/create。 + +### P10 单个无响应 ChunkKV + +- **P10.1 注入**：仅向一个已核对 container/PID/start-time 的 ChunkKV 发 SIGSTOP。 + 不暂停整容器或误伤同名服务。 +- **P10.2 恢复**：60 秒内旧进程退出并被新 PID 替换，其余 34 个服务身份不变。 + 验证旧数据和本场景新确认写入；恢复后全量审计期间不允许再次静默替换。 +- **P10.3 失败收尾**：保留错误后解除仍存活旧进程的 STOP，防止遗留注入污染后续。 + +### P11 跨主机 KV 生命周期 + +- **P11.1 Follower**：选择与当前 leader 不同物理主机的 follower，经它的 Web + stop 本地 KV；核对 completed Stop receipt 和实际退出。远端 authority 仍可 + 管理该节点，单次 restart 得到新 PID，其余 34 服务不变，完整历史审计通过。 +- **P11.2 Leader**：重新识别当前 leader，执行 Stop；即使它已退出，原 completed + receipt 仍可读取。等待存活节点的新 authority，再单次重启原节点并完整审计。 +- **P11.3 切换窗口**：全过程检查幸存节点的本地 monitor 新鲜度与真实进程身份。 + 不能只检查最终 Ready，从而漏掉服务意图读取阻塞、旧健康快照或兄弟服务重启。 +- **P11.4 退出证据**：出现 native abort、自 join、WAL drain 失败或其他异常退出， + 先保存对应 PID/容器代际的应用、RPC、tree、stderr 和 crash/core，再诊断。 + +### P12 B 主机两个节点同时离线 + +- **P12.1 真正接管**：故障前证明 B1/B2 都是当前 Serving owner，再停止它们。 + A 的三 voter 在 180 秒内接管 catalog；执行单次 KV 写/读，记录真实新 ack。 +- **P12.2 停机期间新 payload**：B 保持离线，经 A 新写 S3 witness 和 Iceberg + property commit；新 payload 的实际布局须能由 A 独立读取，三个 A 入口全读。 +- **P12.3 EC 边界**：记录旧对象每主机 fragment 数和丢失预算。例如 EC 8+4 在 + B 上有六片时，不能要求 B 离线期间该对象仍可重建，也不能宣称已有 rack + 保护。此时的新写可用性结论与旧 EC 暂不可用分别记录，不混为一个成功。 +- **P12.4 恢复与持久化**：恢复 B，再分别用原卷重建 B1/B2，验证原身份、五副本 + 固定 checkpoint 和五入口全部历史数据，包括停机时不可用的旧 EC 对象。 +- **扩展**：若要验证旧 EC 数据也能抗整机故障，单独选择满足丢片预算的真实布局， + 并在停机期间全读旧数据；不能由三 voter 存活推导这个保证。 + +### P13 A 主机三个节点同时离线 + +- **P13.1 失去 quorum**：停止 A1/A2/A3，B 在 30 秒内报告当前不可用；新 mutation + 不得确认成功。记录失败与未知结果，不把 connection loss 等同于没有写入。 +- **P13.2 租约安全退出**：因原租约到期而退出的幸存 ChunkKV 必须有同一进程的 + lease 失效、主动停止、RPC worker join 和退出证据。native abort 或未知原因 + 的替换不能归类为正常 lease fencing。 +- **P13.3 恢复与协议读取**：恢复 A，验证全部服务和固定五副本 checkpoint， + 再核对所有原生/协议历史。每个 Iceberg reader 独立保存 endpoint、PID、 + table/object/stage、时间、stdout 和退出码，再聚合结果。 +- **P13.4 错误归属**：区分真实 curl/GetObject 首错、其他 reader 的中止和后续未 + 执行项。S3 原文件通过不能替代 Iceberg 全读；不能按并发输出顺序猜测入口。 + +## 6. 大类 E：分配、积压恢复与空间正确性 + +### P14 分配级显式安全重试 + +- **P14.1 有界故障**：保留所有 KV 进程和 Group0 leader/term/lease，在 Group1 + 注入 leadership step-down，三秒内至少一次被当前 leader 接受。 +- **P14.2 实际触发**：单次启动生产分配 client，必须观察到 allocation-level + 明确 Unavailable safe retry。仅最终成功、transport timeout 或未触发故障均 + 不能代替该断言。 +- **P14.3 原策略完成**：释放注入，用同一 ID 在原五次重试/十秒限制内完成 + allocate/query。保存真实 retry/ack 证据；instrumentation 不改变分支或策略。 + +### P15 超过 1024 slots 的积压与全冷恢复 + +- **P15.1 实际 gap**：停止一个真实 KV follower，经存活副本确认至少 1100 次 + 单写；记录 offline checkpoint 上界、写前/写后目标，证明实际差值大于 1024。 +- **P15.2 catch-up 与冷启动**：恢复 follower，验证 FetchGap 有界并发和真实进度； + 然后全停五节点并沿原卷冷启动。故障前固定 Groups 0/1/3 的 ack target， + 每组五副本健康引擎都必须达到目标，再核对全部数据。不能随新写入移动目标。 +- **扩展 P15.3**：分别制造 reclaimed WAL、empty sealed WAL、high-promise/ + old-chosen 等物理边界，记录触发证据。未实际制造就保留覆盖缺口；组件回归 + 和普通大 gap 的成功不等于这些物理边界已经通过。 + +### P16 最终数据与 DiskDB 占用 + +- **P16.1 最终门禁**：catalog 全部 Serving，五入口 Ready，35 服务健康；下一样本 + 与最终审计结束时仍是相同进程身份。 +- **P16.2 新旧数据**：新 KV batch、新 ChunkDB allocation/query 与所有历史数据 + 分别校验，继续区分 acknowledgment、已确认删除和 OutcomeUnknown。 +- **P16.3 全量占用**：只读扫描所有本轮 DiskDB occupancy，覆盖完整 disk/zone/ + continuation，无截断；核对 live mirror segments 不跨 chunk 重叠，并保存 + 实际 live、allocated、free 和 reserve 数。 + +## 7. 大类 F：首错诊断、修复和清理 + +### P17 失败证据与分类 + +- **P17.1 首错**：打印失败用例、操作和完整错误，调试时启用 + `CROWDB_E2E_PRINT_SERVICE_ERRORS=1`。保留原始客户端输出，不仅保存包装层 panic。 +- **P17.2 同时间线**：保存 inspect/exit/OOM、当前 PID/start-time、健康时间戳、 + KV role/term/lease/checkpoint、catalog/transfer、DiskIO 和磁盘资源。 +- **P17.3 native 与主机**：进程异常先分析 crash/core；I/O 停顿补 diskstats/ + pressure 和可读的阻塞线程栈。观察事实、推断和未证实原因分开写；后续 PASS + 不会反向证明旧无栈 abort 或旧无入口标签超时的原因。 +- **P17.4 完整日志**：按实际 PID、容器启动时间和日志类型选择 application、RPC、 + metrics、tree 文件。历史 stderr 不能算作新进程错误，也不能因日志非空就丢弃 + 新错误；保存前后证据或能绑定本次执行的增量。 + +### P18 修复和续跑 + +- **P18.1 最小复现**：找第一次偏离，区分代码、时序/注入、观察器和环境原因； + 对代码缺陷补能够在旧代码失败的 focused regression，再运行受影响组和门禁。 +- **P18.2 实际输入**：生产输入变化后重新构建、校验、分发镜像及客户端；先修复 + 会使后续慢/挂/失败的依赖，不让队列继续反复撞同一问题。 +- **P18.3 续跑账本**：每次 attempt 使用独立身份和不可覆盖 receipt，保留原失败、 + 已确认写入和未知结果。观察器失败不能靠重设基线或移动 stop marker 自动消除； + 必须记录修复/环境核对和新审计范围，再继续未完成项。 +- **P18.4 覆盖边界**：组件、容器、物理和 soak 分开记；新镜像的局部 PASS 不 + 继承旧镜像的全部覆盖。负载被修复打断须新跑 30 分钟；后续恢复修复是否影响 + 旧 soak 输入须明确说明，不为了制造“同镜像 PASS”重复不受影响的全部测试。 + +### P19 停机留证与精确清理 + +- **P19.1 双主机预检**：先固定两台主机全部待清理对象；核对 run label、名称、 + container ID、原卷名称/CreatedAt、mount、最后 stopped 状态/时间。任一项改变 + 即停止，不能只因名称相同继续删除。 +- **P19.2 停机留证**：停止本轮节点后保存完整服务日志、crash、inspect 和数据 + ledger；两台主机全部留证成功后才开始第一次删除。 +- **P19.3 数据卷备份策略**：默认不新增整卷归档，依照用户本轮保留/清理要求执行； + 已存在的私有备份继续保留。只有用户明确要求时才做额外卷备份并验证完整性。 + 服务日志留证、镜像传输文件和数据卷备份是不同对象，不互相替代。 +- **P19.4 精确删除**：只删除已获授权且预检成功的归属对象，数量与清单完全 + 相符；不使用 daemon-wide prune，不清共享 runtime 或其他任务资源。 +- **P19.5 清理后验证**：两主机本轮目标资源均不存在、所属监听端口关闭、SSH key + 仍可用；保留 run artifacts、原始错误和约定保留文件。不得在报告中隐去之前 + 发生的越界删除或无法恢复的数据丢失。 + +## 8. 每轮交付与验收结论 + +统一目录：`.crowdb-runtime/artifacts/physical-nodes//`。保存 harness 的 +实际源码与命令；当前没有一个仓库 Pixi task 实现本目录的全部流程，不能依赖 +先前临时文件或把方案描述成已有的一键任务。 + +- `run.json`：所选父/子场景及扩展 profile、两主机、节点/端口/资源映射、归属 + 清单、源码/镜像/客户端身份、dirty diff、harness 来源和构建/传输 receipts。 +- `results.json`：各场景 PASS/FAIL/NOT RUN、独立门禁起点/限制/耗时、实际注入 + 是否触发、未执行原因、所属 attempt 和证据路径。旧结果不可覆盖。 +- `data-ledger.jsonl`、`metrics.jsonl`、`checkpoints.json`：数据及确认语义、分段 + 统计、固定目标和实际恢复进度；大文件分段保存。 +- `loader/` 与 `audits/`：各协议源版本、生成/注入/验证命令、完整报告和每个入口 + 的独立结果。未实现路径标 NOT RUN，不以其他协议或组件结果替代。 +- `observations/`、`first-failures/`：注入至完整审计结束的连续观察、身份 fence、 + 首错、完整日志、crash 和明确仍未证实的原因。 +- `cleanup.json` 与 `summary.md`：精确清理结果、实际源码代际覆盖、未完成/未触发 + 场景、数据保留/丢失事实和下次起点。明确区分“结束运行”与“所选必需场景全过”。 + +最终结论只覆盖本轮明确选择并实际通过的场景。人工共同 review 与测试执行分别 +记录；测试 PASS 不自动完成共同 review。 + +## 9. 实现和回归入口 + +- 启动/镜像：`container/oci-image/{run-node,runtime,artifact,transfer}.py`、 + `container/single-node-container/entrypoint.sh` 和 + `container/crowdb-monitor/src/node/startup/policy.rs`。 +- 形成/观察：`lib/crowdb-protocol/src/mgmt/node/`、`app/crowdb-web/src/node/`、 + `lib/crowdb-console-shared/src/ops/cluster/readiness/` 和 + `container/crowdb-monitor/src/node/runtime/`。 +- 数据/恢复：`lib/crowdb-{kv,kv-client,chunk-kv-client,chunkdb-client}/`、 + `app/crowdb-chunkdb/src/topology/`、`lib/crowdb-chunk-stream/src/production_chunk/` + 和 `lib/crowdb-tree/src/backend/local/`。 +- 组件回归：KV backlog/FetchGap durability、ChunkStream finalized-empty、Monitor + discovery/health/service receipts、ChunkDB allocation retry/topology availability、 + BlockAsyncPageStore 跨 extent 读取。按实际改动选择受影响组，通过 `pixi run` 执行。 +- 容器/协议回归：`e2e-test/tests/deployment/`、`e2e-test/tests/console/`、 + `e2e-test/tests/access_iceberg/` 和对应 S3/native SDK 入口；它们与真实跨主机 + 故障、连续 30 分钟负载分别计覆盖。 diff --git a/doc/doc_index.md b/doc/doc_index.md index 3de30bc2e..3b93da310 100644 --- a/doc/doc_index.md +++ b/doc/doc_index.md @@ -52,7 +52,7 @@ Temporary plans live under `doc/working/`; flow analyses live under | `doc/dev/crash_debugging.md` | Core collection, host setup, GDB, and exact-build symbols. | | `doc/dev/env_setup.md` | Benchmark commands, sentinels, prerequisites, and perf-counter setup. | | `doc/dev/hyper_fork.md` | Hyper fork branches, submodule, build, sync, validation, and recovery. | -| `doc/dev/physical-node-test-plan.md` | Repeatable two-host, five-container physical tests and 30-minute sustained load. | +| `doc/dev/physical-node-test-plan.md` | Physical-node test cases, scenarios, recovery gates and evidence. | | `tools/README.md` | Tool directories, Pixi task entry points, CI checks, and suite timing. | ## Component Acceptance diff --git a/doc/working/plan-ci-stability-review.md b/doc/working/plan-ci-stability-review.md index 67b052421..768308d60 100644 --- a/doc/working/plan-ci-stability-review.md +++ b/doc/working/plan-ci-stability-review.md @@ -85,7 +85,22 @@ Access Restart 前的实际依赖就绪观察；该测试修正仍需独立发 新 P09 在 A2 Group 0 复现真实的跨块异步短读，已停止五个节点保留原卷， P09–P19 尚未完成。最小回归重现完全相同的 958464/1212416 字节错误； 跨块读取修复、19 项 focused 检查及完整 C++/FFI、core、storage、fmt、 - tree-lint、clippy 门禁均通过；发布和新镜像物理验证继续进行。 + tree-lint、clippy 门禁均通过，已推送 `c063a169`；新镜像同卷升级通过。 + 新启动全部 35 服务在 37.439 秒恢复，原生历史与五端点 S3 SF1 通过。 + Iceberg 校验时 A 主机 NVMe 约十秒 I/O 停顿，随后 monitor 过期、Group0 + 超时和五个 ChunkKV 租约退出；OOM 为零，设备/文件系统根因尚未证实。 + 单对象原限制全读/校验通过；新的完整只读历史审计随后通过，1230.088 秒， + 597 次观察、最大间隔 3.843 秒、35 身份不变；187 份 Dataset 报告及全部 + 五端点协议/见证/删除记录通过。原失败不改写；c063 的新 P09 完整通过（1164.469 秒），P10 也通过 + （1023.819 秒）。P11 follower 完整审计通过；leader 恢复门禁通过但 B2 + monitor 观察过期 3519 ms，后续 P12–P19 未派发。 + 根因是本地健康发布前同步等待 Group0 desired-service scan。旧代码的慢 + authority 回归失败；改为一个有界后台只读任务并保留操作前权威 fence 后， + focused 八项通过；完整 fmt/clippy、Access/Monitor 2041、Console 426 及正常 + task 附属检查通过。未改动的 C++/core/storage 门禁保留 c063 输入证明。 + 发布、新镜像和受影响物理复验待完成；五节点已停止、原七卷保留，失败和 + 修复证据分别留存。可复用物理方案已按测试用例重组。 + 未查询新 CI。 这不是新增 CI job 失败，不并入原 16 项共同 review。 A 已完成的三个备份和全部原始失败证据继续保留，不再新增备份。 @@ -290,7 +305,8 @@ Access Restart 前的实际依赖就绪观察；该测试修正仍需独立发 - 上一版 P12 修复镜像输入 tree 为 `76f794e03757249f37efa96cd0bfcad064be4388`； 32 条历史容器失败记录所对应的 case 已全部在此镜像实际执行并通过， 精确映射见 `new-image-case-progress.json`。全量结束后的 scope guard 确认 - 原有 7 个停止容器及卷身份完全不变。物理恢复与新提交的远端 CI 仍须完成。 + 原有 7 个停止容器及卷身份完全不变。物理恢复仍须完成；用户随后取消 + 继续检测 CI，不再要求新提交的远端 CI 作为物理验收前置。 ## Run 41003611925：8 个失败 job @@ -584,8 +600,10 @@ Access Restart 前的实际依赖就绪观察；该测试修正仍需独立发 ## 最后逐项 review 的记录 -- [ ] **补齐最终验收证据**：更新新镜像 396-case 账本、受影响的物理恢复， - 以及本次新 SHA 的全部 27 个远端 job；保留各自输入身份和覆盖边界。 +- [ ] **补齐最终验收证据**：整理 396-case 历史账本、后续修复的受影响 + 组件/E2E 与物理恢复，保留各自源码/镜像身份和覆盖边界，不将早期 PASS + 归给当前镜像。远端仅保留已经取得的结果；用户已取消继续检测 CI， + 不再要求本次新 SHA 的 27 个远端 job。 - [ ] **逐项共同 review 01–16**：每项明确“原因确认程度、修复是否充分、 回归是否能捕获原问题、还需做什么”。保留 Go 旧原因未知的事实， 核对 browser batch 新增的未执行/级联错误归因及独立统计。 @@ -612,7 +630,8 @@ Access Restart 前的实际依赖就绪观察；该测试修正仍需独立发 Review 另发现 C ABI send/chain/slab 在同步 completion 释放 wrapper 后仍会 读它；四个确定性原生用例在修复前失败，保活 native client/connection/transport 至返回后均通过。完整 C++、owned tree transport、storage、fmt/tree-lint 已通过。 - 按新顺序先收尾此修复与 CI，再回到物理验证；P14 门禁继续保留。 + 生命周期修复的本地验证与发布已经完成；CI 按用户要求停止检测。 + 当前 c063 新镜像的 P12/P13 物理复验尚待完成，P14 门禁继续保留。 文件：`lib/crowdb-tree/ffi/src/chunk/transport.rs`、 `lib/crowdb-rpc/ffi/src/owned_route.rs`、 `lib/crowdb-rpc/src/client/client.cpp`、 @@ -630,6 +649,11 @@ Access Restart 前的实际依赖就绪观察；该测试修正仍需独立发 B1/B2 此次 ChunkKV 重启已有 lease 到期、主动退出、正常 join 的日志闭环， stderr 为空；不是已证实的同一 C++ 异常。下次每个入口单独保存 stdout、 endpoint、起止时间与退出码，再聚合状态，避免并发错误归属缺失。 + c063 启动审计另捕获 A 主机约十秒 NVMe I/O 停顿，随后 Group0 超时、 + 五个 ChunkKV 租约退出和五个具名 Iceberg checksum reader 的 curl28； + 无 OOM，设备/文件系统原因未证实，不能据此倒推旧 P13 的根因。 + 单对象原限制验证及新的完整五端点只读历史审计随后通过；新 P09 冷启动 + 及完整历史审计也已通过，后续仍须完成 P12/P13 的独立故障验证。 ## 文件与验证入口 @@ -640,4 +664,5 @@ Access Restart 前的实际依赖就绪观察；该测试修正仍需独立发 - 集成：完整 core/access/console/storage 组，以及精确的原始非容器失败 case。 - E2E：396 个 canonical scope/case、并发隔离、原失败 case 的实际执行、Go 故意注入后的 cleanup、GC 真实 overlap，以及新镜像上的物理恢复。 -- 远端：推送后关联本次 SHA 的新 run，核对 27 个展开后的 job，而非仅看汇总状态。 +- 远端：保留已记录的精确 SHA、attempt 和展开 job。用户已要求停止检测 CI， + 不查询或推断当前提交的新 run，也不将它作为后续物理验收条件。 diff --git a/doc/working/plan-physical-node-ci-handoff.md b/doc/working/plan-physical-node-ci-handoff.md index 7135782e8..8fbc222b4 100644 --- a/doc/working/plan-physical-node-ci-handoff.md +++ b/doc/working/plan-physical-node-ci-handoff.md @@ -11,9 +11,75 @@ Upstream: [Physical Node Test Plan](../dev/physical-node-test-plan.md), Goal: finish CI stability verification and publish CROWDB, then resolve P13 and complete the two-host physical recovery matrix without losing data or evidence. -## 1. Current checkpoint: cross-extent read repair locally verified - -Updated 2026-11-02, 00:55 Asia/Shanghai. The earlier repair is published as +## 1. Current checkpoint: P11 monitor freshness repair under validation + +Updated 2026-11-02, 07:05 Asia/Shanghai. On the published c063 image, P09 and P10 both passed +complete historical audits. P11 follower lifecycle and its complete audit passed; +P11 leader lifecycle recovered every original gate, but the continuous observer +rejected the unaffected B2 monitor at 3519 ms old. P12–P19 were not dispatched. +No I/O/memory stall accompanied this event, and no service identity changed +between the failure capture and the later repair preparation. + +The production health loop awaited a Group0 desired-service scan with a +three-second bound before publishing current local health. A slow-authority +regression fails on the old code. The repair moves that read into one bounded +background task, rejects expired/rebound observations, and retains current +binding/intent checks before any disruptive recovery action. Actual local probe +changes continue to publish with unchanged PIDs. Eight focused tests passed, +including rejection of an obsolete recovery intent before stopping a process. +Full fmt/clippy/access-monitor/console validation passed: 2041 Access/Monitor +Rust cases and 426 Console Rust cases, plus the normal task companion checks. +The first clippy semicolon failure and corrected continuation remain separate. +Unchanged C++/core/storage gates retain their c063 inputs and source proof. + +All five current nodes are stopped for repair with all seven original volumes +retained. Both hosts' complete stopped service/crash logs are frozen. Evidence: +first-failures/P11-monitor-freshness/ and repair/monitor-health-nonblocking/. +The repair still needs publication, a source-verified image, affected container +checks and a new physical epoch. Do not relabel c063 passes as new-image passes. + +The reusable physical plan now has six case categories, 20 parent cases and +71 named scenarios, with no checkboxes. It separates execution results from the +case catalog and cancels mandatory new volume archives. The original 16 human +joint-review boxes remain unchanged. CI inspection remains stopped. + +### Previous cross-extent repair checkpoint + +Updated 2026-11-02, 02:18 Asia/Shanghai. The cross-extent read repair is published +as `c063a169f5e400a44c48a5a66aa42fc48609b396`. The normal client/native/image +builds and same-volume deployment passed on both hosts. Image/runtime identity +is recorded in image/cross-extent-read-fixed/. Fresh startup recovered groups +0/1/3 within 1.887 seconds, catalog within 4.961 seconds and all 35 services +within 37.439 seconds, each under its original independent 180-second gate. +All native historical sets and five-endpoint S3 SF1 passed on this image. + +The subsequent five-endpoint Iceberg audit failed with GetObject curl 28 during +checksum reads. At 17:25:33 UTC the A host's nvme1n1 accumulated 209 outstanding +requests; reads/flushes nearly stopped for about ten seconds. All three A monitor +snapshots became stale, Group0 RPCs timed out, and all five ChunkKV processes +expired their leases and drained gracefully. OOM counters remained zero and +host memory/CPU were available. No kernel device error was recorded. The exact +device/filesystem cause remains unproven; do not attribute this event or the old +P13 timeout to the cross-extent repair without further evidence. Preserve the +failed startup receipt and observations/cross-extent-image-upgrade/ unchanged. + +The same 216-MB lineitem object passed isolated full decoding, checksum and +Iceberg sampling in 61.236 seconds with the original client limits. A new strict +complete read-only audit passed in 1230.088 seconds using +cross-extent-image-upgrade-audit and resume-start-audited.json. It includes all +five endpoints, 187 Dataset reports (935 endpoint/report results), all witnesses +and acknowledged deletions. All 35 identities remained fixed across 597 complete +samples; the maximum gap was 3.843 seconds. Additional 250-ms host I/O sampling +captured no new stall. No startup or mutation was replayed; no client timeout, +concurrency, assertion or lease deadline changed. Fresh P09 then passed in 1164.469 seconds, including all five original-volume +recreations, all recovery gates and full retained history. Its 35 expected service +replacements are the only identity changes; the recovered audit remained fixed. +P10 subsequently passed in 1023.819 seconds; P11 follower passed but its leader +observer failed as described above. The preserved dependent queue is +final-queue-cross-extent-read-fixed-after-io/. Both earlier P09 failures and the +new startup failure remain retained. See first-failures/cross-extent-upgrade-protocol/. + +Earlier resume history: the previous repair is published as 1814405e4dd1fe0fcd64f9f18ac7c71e98d64fd2. The user's latest instructions are "不要检测CI" and "归啥档，测试能过就好了". CI inspection is stopped; do not query its latest state or require it before physical work. No additional volume @@ -35,8 +101,9 @@ five Group-3 endpoints, baseline allocation, five-endpoint S3/Iceberg SF1, 187 Dataset reports, witnesses and acknowledged deletions. The audit observer passed 625 complete samples (maximum gap 3.823 seconds), with all 35 identities fixed. Fresh P09 found a production cross-extent read defect; P09–P19 remain -unfinished. All five current nodes are stopped for repair, with all seven -original volumes retained and no new archives or cleanup. +unfinished at that checkpoint. Those five nodes were stopped for repair; the +current c063 image is now running on the same volumes, with all seven original +volumes retained and no new archives or cleanup. - The first startup observer rejected monitor timestamps while managed services were still launching. P02 requires fresh observations at recovered readiness; @@ -66,8 +133,8 @@ original volumes retained and no new archives or cleanup. completing once after all parts or the first real error; no retry or lock was added. Cross-extent, missing/truncated-tail and existing async tests pass (19 focused tests). Complete C++/FFI, core and storage verification, fmt, - clean changed-file tree-lint and full clippy all passed. Publish this repair, - build a fresh verified image and resume retained-volume physical verification. + clean changed-file tree-lint and full clippy all passed. The repair was published + as c063a169; its verified new image is undergoing retained-volume verification. Evidence: first-failures/P09-retained-engine-unhealthy/ and repair/cross-extent-read-fixed/. Do not dispatch P10–P19 on the old image. - The new deployment controller is harness/physical_storage_repair.py with From f5a6ee07ce9487b3ea33bec7096a2e890a7ed5d7 Mon Sep 17 00:00:00 2001 From: Gian Date: Wed, 4 Nov 2026 14:28:22 +0800 Subject: [PATCH 19/24] Harden service recovery and stabilize integration tests --- .../src/s3/dependencies.rs | 46 +- app/crowdb-access-server/src/s3/health.rs | 10 + .../src/s3/health/observation.rs | 41 +- .../tests/access_probe_budget_test.rs | 136 +++ .../tests/common/access_probe_budget.rs | 255 ++++ .../tests/common/iceberg_client.py | 2 +- .../tests/dataset_batch_header_test.rs | 19 +- .../tests/iceberg_file_request_test.rs | 4 +- .../tests/iceberg_namespace_limits_test.rs | 4 +- .../tests/s3_object_inspection_test.rs | 4 +- app/crowdb-chunk-kv-server/src/server.rs | 7 - .../src/server/drain.rs | 11 + .../src/server/load_sampling.rs | 108 +- .../src/storage/root_catalog.rs | 13 +- .../tests/common/sampling_tree.rs | 147 +++ .../tests/shutdown_runtime_test.rs | 233 ++++ .../tests/transfer_storage_test.rs | 86 ++ app/crowdb-chunkdb/Cargo.toml | 1 + app/crowdb-chunkdb/src/lib.rs | 1 + app/crowdb-chunkdb/src/main.rs | 88 +- app/crowdb-chunkdb/src/readiness.rs | 142 +++ app/crowdb-chunkdb/src/storage/scope.rs | 28 +- .../tests/common/http_readiness.rs | 96 ++ .../tests/common/readiness_failures.rs | 47 + .../tests/common/readiness_registry.rs | 35 + .../tests/http_readiness_test.rs | 118 ++ .../tests/maintenance_domain_scan_test.rs | 128 ++ .../tests/readiness_probe_test.rs | 41 +- app/crowdb-cli/src/commands/cluster/local.rs | 2 +- .../src/commands/cluster/local/docker.rs | 29 +- .../src/commands/cluster/local/launch.rs | 2 +- .../tests/cluster_lifecycle_test.rs | 33 + app/crowdb-cli/tests/common/local_docker.py | 6 +- app/crowdb-kv-server/Cargo.toml | 1 + .../domain_monitor/chunk_kv/balance.rs | 32 +- .../chunk_kv/balance/materialization.rs | 63 + .../domain_monitor/chunk_kv/catalog.rs | 11 +- .../domain_monitor/chunk_kv/grants.rs | 21 +- .../tests/chunk_kv_balance_planning_test.rs | 2 + .../committed_transfer.rs | 143 +++ .../tests/common/grant_isolation.rs | 228 ++++ .../tests/common/split_materialization.rs | 3 + .../common/split_materialization/transfer.rs | 75 ++ .../tests/domain_monitor_test.rs | 2 + app/crowdb-web/src/lib.rs | 7 +- app/crowdb-web/src/node.rs | 2 +- app/crowdb-web/src/node/authority.rs | 47 +- .../src/node/authority/projection.rs | 61 + app/crowdb-web/src/services/observation.rs | 19 +- .../src/services/observation/cache.rs | 17 +- .../src/services/observation/mutation.rs | 1 + .../src/services/observation/sampling.rs | 32 +- .../src/services/observation/source.rs | 45 + .../tests/common/process_observed.rs | 175 +++ .../tests/common/process_transport.rs | 90 ++ .../tests/node_confirmed_projection_test.rs | 60 + .../tests/s3_inspection_proxy_test.rs | 2 +- .../service_inventory_observation_test.rs | 4 + .../tests/service_process_observed_test.rs | 230 ++++ .../tests/service_process_transport_test.rs | 67 ++ app/crowdb-web/ui/src/api.ts | 2 + .../ui/src/dataset/editor/Table.test.tsx | 2 +- app/crowdb-web/ui/src/kv/displayBytes.test.ts | 6 +- .../src/services/useNodeServicePlans.test.tsx | 16 +- .../ui/src/services/useNodeServicePlans.ts | 43 +- .../useNodeServicePlansObservation.test.tsx | 218 ++++ container/crowdb-monitor/src/monitor_log.rs | 29 + .../crowdb-monitor/src/node/runtime/health.rs | 64 +- .../src/node/runtime/health/observations.rs | 76 ++ .../src/node/runtime/services.rs | 1 + .../src/node/runtime/services/execution.rs | 37 +- .../src/node/runtime/services/test_support.rs | 7 + container/crowdb-monitor/src/process.rs | 42 +- .../src/process/exit_observation.rs | 82 ++ .../tests/common/managed_health.rs | 180 +++ .../tests/node_managed_health_test.rs | 73 ++ .../tests/node_service_readiness_test.rs | 25 +- .../crowdb-monitor/tests/process_test.rs | 139 +++ .../design-crowdb-access-server.md | 28 +- .../design-crowdb-chunkdb-range-binding.md | 31 +- .../chunkds/design-crowdb-chunk-kv-server.md | 27 +- .../console/design-crowdb-console-ui.md | 64 +- doc/design/deploy/design-crowdb-deploy.md | 17 + doc/design/diskio/design-crowdb-diskio.md | 6 + doc/design/kv/design-crowdb-kv-group0.md | 10 + doc/design/kv/design-crowdb-kv-rpc-client.md | 20 + doc/design/kv/design-crowdb-kv-wal.md | 19 +- .../tree/design-crowdb-tree-chunk-storage.md | 9 + doc/dev/physical-node-test-plan.md | 1049 +++++++++++------ doc/doc_index.md | 2 +- doc/working/plan-ci-stability-review.md | 738 ++---------- doc/working/plan-physical-node-ci-handoff.md | 710 ----------- .../e2e/fixtures/icebergInspection.py | 2 +- .../e2e/fixtures/icebergRoundTrip.py | 2 +- .../e2e/fixtures/nativeServiceLifecycle.ts | 33 +- .../console-ui/e2e/fixtures/parquetFiles.py | 2 +- .../flows/11-cluster-server-lifecycle.spec.ts | 17 +- .../e2e/flows/30-kv-ops-basic.spec.ts | 4 +- .../e2e/flows/55-chunk-kv-catalog.spec.ts | 7 +- .../console-ui/e2e/flows/71-s3-native.spec.ts | 2 +- .../e2e/flows/92-three-node-data.spec.ts | 2 +- e2e-test/clients/iceberg-python/listing.py | 4 +- e2e-test/clients/iceberg-python/namespace.py | 2 +- e2e-test/clients/s3/batch_delete.py | 4 +- e2e-test/clients/s3/default_client.py | 2 +- e2e-test/clients/s3/object_copy.py | 2 +- e2e-test/clients/s3/wire.py | 4 +- e2e-test/console_cli_readiness.py | 95 ++ e2e-test/console_cli_recovery.py | 15 +- e2e-test/tests/console_cli_readiness_test.py | 109 ++ e2e-test/tests/dataset_observation_test.py | 2 +- .../tests/file_json_test.rs | 2 +- .../tests/file_listing_test.rs | 8 +- .../tests/file_location_test.rs | 4 +- lib/crowdb-access-iceberg/tests/key_test.rs | 2 +- .../tests/manifest_variant_test.rs | 2 +- .../tests/namespace_create_test.rs | 8 +- .../tests/namespace_model_test.rs | 8 +- .../tests/namespace_record_test.rs | 8 +- .../tests/namespace_repository_test.rs | 10 +- .../tests/namespace_update_test.rs | 27 +- .../tests/native_storage_path_test.rs | 11 +- .../tests/record_test.rs | 2 +- .../tests/table_metadata_defaults_test.rs | 2 +- .../tests/table_metadata_document_test.rs | 2 +- lib/crowdb-access-s3/tests/copy_test.rs | 2 +- lib/crowdb-access-s3/tests/delete_test.rs | 7 +- .../tests/object_inspection_test.rs | 2 +- .../tests/user_metadata_test.rs | 2 +- lib/crowdb-chunk-client/src/client.rs | 2 +- .../src/client/disk_metrics.rs | 21 + .../src/client/readiness.rs | 157 ++- .../src/disk_io/disk_writer.rs | 11 + .../src/disk_io/routing.rs | 35 +- .../src/disk_io/routing/reads.rs | 44 + .../tests/common/readiness_peer.rs | 172 +++ .../tests/common/readiness_schedule.rs | 106 ++ .../tests/readiness_priority_test.rs | 103 ++ .../tests/readiness_test.rs | 230 +++- lib/crowdb-chunkdb-client/src/client.rs | 4 + lib/crowdb-chunkdb-client/src/readiness.rs | 9 +- lib/crowdb-chunkdb-client/src/retry.rs | 34 +- .../src/deployment/admission.rs | 3 + .../admission/process_observation.rs | 83 ++ .../tests/common/ssh_server.rs | 24 +- .../tests/process_observation_ssh_test.rs | 99 ++ lib/crowdb-kv-client/src/client/core.rs | 26 +- lib/crowdb-kv-client/src/client/retry.rs | 12 +- lib/crowdb-kv-client/src/client/topology.rs | 10 +- .../src/client/topology/recovery.rs | 34 + .../src/service/registry/diskio.rs | 19 +- .../src/transport/rpc_transport.rs | 202 +--- .../transport/rpc_transport/connections.rs | 152 +++ .../transport/rpc_transport/point_reads.rs | 83 ++ .../tests/common/point_read_peer.rs | 219 ++++ .../tests/common/topology_recovery.rs | 87 ++ .../tests/point_read_isolation_test.rs | 170 +++ .../tests/storage_exclusion_test.rs | 67 +- .../tests/topology_retry_discovery_test.rs | 88 ++ lib/crowdb-kv/Cargo.toml | 2 +- .../src/cluster/group_election_leader.rs | 9 +- lib/crowdb-kv/src/wal/block_backend.rs | 90 +- lib/crowdb-kv/src/wal/wal_file.rs | 16 +- .../tests/group_test/g3_leader_change_test.rs | 62 + lib/crowdb-kv/tests/wal_test.rs | 3 + .../tests/wal_test/block_runtime_test.rs | 141 +++ .../tests/chunk_task_value_test.rs | 2 +- .../include/crowdb-tree/btree/tree.h | 9 +- lib/crowdb-tree/src/snapshot/retry.cpp | 66 ++ lib/crowdb-tree/src/snapshot/write.cpp | 34 +- .../integration/chunk_snapshot_retry_test.cpp | 115 ++ 171 files changed, 7895 insertions(+), 2514 deletions(-) create mode 100644 app/crowdb-access-server/tests/access_probe_budget_test.rs create mode 100644 app/crowdb-access-server/tests/common/access_probe_budget.rs create mode 100644 app/crowdb-chunk-kv-server/tests/common/sampling_tree.rs create mode 100644 app/crowdb-chunk-kv-server/tests/shutdown_runtime_test.rs create mode 100644 app/crowdb-chunkdb/src/readiness.rs create mode 100644 app/crowdb-chunkdb/tests/common/http_readiness.rs create mode 100644 app/crowdb-chunkdb/tests/common/readiness_failures.rs create mode 100644 app/crowdb-chunkdb/tests/common/readiness_registry.rs create mode 100644 app/crowdb-chunkdb/tests/http_readiness_test.rs create mode 100644 app/crowdb-chunkdb/tests/maintenance_domain_scan_test.rs create mode 100644 app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/materialization.rs create mode 100644 app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/committed_transfer.rs create mode 100644 app/crowdb-kv-server/tests/common/grant_isolation.rs create mode 100644 app/crowdb-kv-server/tests/common/split_materialization/transfer.rs create mode 100644 app/crowdb-web/src/node/authority/projection.rs create mode 100644 app/crowdb-web/tests/common/process_observed.rs create mode 100644 app/crowdb-web/tests/common/process_transport.rs create mode 100644 app/crowdb-web/tests/node_confirmed_projection_test.rs create mode 100644 app/crowdb-web/tests/service_process_observed_test.rs create mode 100644 app/crowdb-web/tests/service_process_transport_test.rs create mode 100644 app/crowdb-web/ui/src/services/useNodeServicePlansObservation.test.tsx create mode 100644 container/crowdb-monitor/src/node/runtime/health/observations.rs create mode 100644 container/crowdb-monitor/src/process/exit_observation.rs create mode 100644 container/crowdb-monitor/tests/common/managed_health.rs create mode 100644 container/crowdb-monitor/tests/node_managed_health_test.rs delete mode 100644 doc/working/plan-physical-node-ci-handoff.md create mode 100644 e2e-test/console_cli_readiness.py create mode 100644 e2e-test/tests/console_cli_readiness_test.py create mode 100644 lib/crowdb-chunk-client/src/disk_io/routing/reads.rs create mode 100644 lib/crowdb-chunk-client/tests/common/readiness_peer.rs create mode 100644 lib/crowdb-chunk-client/tests/common/readiness_schedule.rs create mode 100644 lib/crowdb-chunk-client/tests/readiness_priority_test.rs create mode 100644 lib/crowdb-console-shared/src/deployment/admission/process_observation.rs create mode 100644 lib/crowdb-console-shared/tests/process_observation_ssh_test.rs create mode 100644 lib/crowdb-kv-client/src/client/topology/recovery.rs create mode 100644 lib/crowdb-kv-client/src/transport/rpc_transport/connections.rs create mode 100644 lib/crowdb-kv-client/src/transport/rpc_transport/point_reads.rs create mode 100644 lib/crowdb-kv-client/tests/common/point_read_peer.rs create mode 100644 lib/crowdb-kv-client/tests/common/topology_recovery.rs create mode 100644 lib/crowdb-kv-client/tests/point_read_isolation_test.rs create mode 100644 lib/crowdb-kv-client/tests/topology_retry_discovery_test.rs create mode 100644 lib/crowdb-kv/tests/wal_test/block_runtime_test.rs create mode 100644 lib/crowdb-tree/src/snapshot/retry.cpp create mode 100644 lib/crowdb-tree/tests/integration/chunk_snapshot_retry_test.cpp diff --git a/app/crowdb-access-server/src/s3/dependencies.rs b/app/crowdb-access-server/src/s3/dependencies.rs index 201cbf976..57f3a4e23 100644 --- a/app/crowdb-access-server/src/s3/dependencies.rs +++ b/app/crowdb-access-server/src/s3/dependencies.rs @@ -9,6 +9,8 @@ use crowdb_access_s3::{metadata::MetadataKey, metrics::DependencyHealth}; use super::operations::ProductionS3Operations; +pub(super) const PROBE_BUDGET: Duration = Duration::from_secs(2); + pub type DependencyProbeFuture = Pin> + Send + 'static>>; pub struct StorageReadiness { @@ -18,8 +20,8 @@ pub struct StorageReadiness { impl ProductionS3Operations { pub(super) async fn observe_storage(&self) -> StorageReadiness { - let (metadata, chunks) = tokio::join!( - bounded("metadata", async { + observe_storage_dependencies( + async { self.storage .metadata .get( @@ -29,24 +31,50 @@ impl ProductionS3Operations { .await .map_err(|error| error.to_string())?; Ok("Routed Chunk-KV bucket namespace read succeeded".into()) - }), - async { - Box::pin(self.storage.chunks.observe_readiness(Duration::from_millis(750))) + }, + |budget| async move { + Box::pin(self.storage.chunks.observe_readiness(budget)) .await .map_err(|error| error.to_string()) }, - ); - StorageReadiness { metadata, chunks } + ) + .await } + + #[cfg(feature = "test-util")] + /// Exercise the production dependency join and caller budget with controlled reads. + pub async fn observe_storage_dependencies_for_tests(metadata: M, chunks: C) -> StorageReadiness + where + M: Future>, + C: FnOnce(Duration) -> F, + F: Future>, + { + observe_storage_dependencies(metadata, chunks).await + } +} + +async fn observe_storage_dependencies(metadata: M, chunks: C) -> StorageReadiness +where + M: Future>, + C: FnOnce(Duration) -> F, + F: Future>, +{ + let (metadata, chunks) = tokio::join!(bounded("metadata", metadata), chunks(PROBE_BUDGET)); + StorageReadiness { metadata, chunks } } pub(super) async fn bounded( dependency: &str, probe: impl Future>, ) -> Result { - tokio::time::timeout(Duration::from_millis(750), probe) + tokio::time::timeout(PROBE_BUDGET, probe) .await - .unwrap_or_else(|_| Err(format!("{dependency} read exceeded its 750 ms deadline"))) + .unwrap_or_else(|_| { + Err(format!( + "{dependency} read exceeded its {} ms deadline", + PROBE_BUDGET.as_millis() + )) + }) } pub(super) fn health(result: &Result) -> DependencyHealth { diff --git a/app/crowdb-access-server/src/s3/health.rs b/app/crowdb-access-server/src/s3/health.rs index 549a28767..33269187b 100644 --- a/app/crowdb-access-server/src/s3/health.rs +++ b/app/crowdb-access-server/src/s3/health.rs @@ -41,6 +41,16 @@ impl S3HttpHandler for AccessHealthHandler { } } +impl S3Dispatcher { + #[cfg(feature = "test-util")] + /// Read the production cached response without transport or dependency work. + #[must_use] + pub fn health_response_for_tests(&self) -> Response { + let observed = self.observations.latest.load_full(); + readiness_response(observed.as_deref(), self.health.is_live(), false) + } +} + fn readiness_response( observed: Option<&HealthObservation>, live: bool, diff --git a/app/crowdb-access-server/src/s3/health/observation.rs b/app/crowdb-access-server/src/s3/health/observation.rs index 61fb9b834..f1b2fb499 100644 --- a/app/crowdb-access-server/src/s3/health/observation.rs +++ b/app/crowdb-access-server/src/s3/health/observation.rs @@ -17,6 +17,8 @@ use tokio::time::{Instant, MissedTickBehavior}; use super::super::{dependencies, S3Dispatcher, StorageReadiness}; +const OBSERVATION_BUDGET: Duration = dependencies::PROBE_BUDGET.saturating_add(Duration::from_millis(100)); + pub(super) const MAX_AGE: Duration = Duration::from_secs(3); #[derive(Default)] @@ -82,16 +84,43 @@ impl S3Dispatcher { u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) }); let observed = tokio::time::timeout( - Duration::from_millis(800), + OBSERVATION_BUDGET, Arc::clone(&self.operations).observe_dependencies(), ) .await .unwrap_or_else(|_| { Some(StorageReadiness { - metadata: Err("Dependency observation exceeded its 800 ms deadline".into()), - chunks: Err("Dependency observation exceeded its 800 ms deadline".into()), + metadata: Err(format!( + "Dependency observation exceeded its {} ms deadline", + OBSERVATION_BUDGET.as_millis() + )), + chunks: Err(format!( + "Dependency observation exceeded its {} ms deadline", + OBSERVATION_BUDGET.as_millis() + )), }) }); + let generation = self + .observations + .latest + .load() + .as_ref() + .map_or(1, |last| last.generation + 1); + if let Some(failed) = observed + .as_ref() + .filter(|value| value.metadata.is_err() || value.chunks.is_err()) + { + tracing::warn!( + generation, + configured_budget_ms = u64::try_from(dependencies::PROBE_BUDGET.as_millis()).unwrap_or(u64::MAX), + observation_budget_ms = u64::try_from(OBSERVATION_BUDGET.as_millis()).unwrap_or(u64::MAX), + total_elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX), + checked_at_unix_ms, + metadata_error = ?failed.metadata.as_ref().err(), + chunks_error = ?failed.chunks.as_ref().err(), + "Access dependency observation failed; background sampling continues" + ); + } let mut health = self .health .snapshot(&self.metrics, self.body_allocator.as_deref()); @@ -113,12 +142,6 @@ impl S3Dispatcher { DependencyHealth::Unavailable }; } - let generation = self - .observations - .latest - .load() - .as_ref() - .map_or(1, |last| last.generation + 1); HealthObservation { health, details, diff --git a/app/crowdb-access-server/tests/access_probe_budget_test.rs b/app/crowdb-access-server/tests/access_probe_budget_test.rs new file mode 100644 index 000000000..68694babf --- /dev/null +++ b/app/crowdb-access-server/tests/access_probe_budget_test.rs @@ -0,0 +1,136 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +#![cfg(feature = "s3")] + +#[path = "common/access_probe_budget.rs"] +mod fixture; +use fixture::{advance, failure_warnings, settle, TestSampler, HELD_OUTER}; +use std::sync::atomic::Ordering; + +#[tokio::test(start_paused = true)] +async fn production_metadata_and_chunk_probes_accept_reads_over_750ms_within_two_seconds() { + let (warnings, _subscriber) = failure_warnings(); + let fixture = TestSampler::new(1200); + settle().await; + assert_eq!(fixture.operations.calls.load(Ordering::SeqCst), 1); + advance(750).await; + assert_eq!( + fixture.generation(), + 0, + "a valid slower dependency must remain in progress" + ); + for _ in 0..32 { + assert_eq!(fixture.read(503).await["observation"]["state"], "pending"); + } + assert_eq!(fixture.operations.calls.load(Ordering::SeqCst), 1); + advance(450).await; + let ready = fixture.read(200).await; + assert_eq!(ready["metadata"], "ready"); + assert_eq!(ready["chunks"], "ready"); + assert_eq!(ready["observation"]["age_ms"], 1200); + assert_eq!(fixture.operations.metrics.snapshot().chunkdb.errors, 0); + assert_eq!(fixture.operations.metrics.snapshot().diskio.errors, 0); + assert_eq!(warnings.count.load(Ordering::SeqCst), 0); +} + +#[tokio::test(start_paused = true)] +async fn two_second_failure_is_visible_then_the_same_sampler_recovers_automatically() { + let (warnings, _subscriber) = failure_warnings(); + let fixture = TestSampler::new(3000); + settle().await; + advance(1999).await; + assert_eq!(fixture.generation(), 0); + fixture.operations.delay_ms.store(100, Ordering::SeqCst); + advance(1).await; + let failed = fixture.read(503).await; + assert_eq!(failed["observation"]["generation"], 1); + assert_eq!(failed["metadata"], "unavailable"); + assert_eq!(failed["chunks"], "unavailable"); + assert!(failed["details"]["metadata"] + .as_str() + .unwrap() + .contains("2000 ms")); + assert!(failed["details"]["chunks"] + .as_str() + .unwrap() + .contains("ChunkDB read exceeded")); + assert_eq!(fixture.operations.metrics.snapshot().chunkdb.errors, 1); + assert_eq!(warnings.count.load(Ordering::SeqCst), 1); + assert_eq!(warnings.generation.load(Ordering::SeqCst), 1); + assert_eq!(warnings.budget_ms.load(Ordering::SeqCst), 2000); + assert_eq!(warnings.elapsed_ms.load(Ordering::SeqCst), 2000); + assert!(warnings.checked_at_unix_ms.load(Ordering::SeqCst) > 0); + assert!(fixture.dispatcher.start_health_observer().is_err()); + advance(100).await; + let ready = fixture.read(200).await; + assert_eq!(ready["observation"]["generation"], 2); + assert_eq!(ready["ready"], true); + assert_eq!(fixture.operations.calls.load(Ordering::SeqCst), 2); + assert_eq!(fixture.operations.peak.load(Ordering::SeqCst), 1); + assert_eq!(fixture.operations.cancelled.load(Ordering::SeqCst), 0); + assert_eq!(warnings.count.load(Ordering::SeqCst), 1); +} + +#[tokio::test(start_paused = true)] +async fn outer_watchdog_cancels_at_2100ms_then_sampling_continues_without_overlap_or_burst() { + let (warnings, _subscriber) = failure_warnings(); + let mut fixture = TestSampler::new(HELD_OUTER); + settle().await; + advance(2099).await; + assert_eq!(fixture.generation(), 0); + fixture.operations.delay_ms.store(100, Ordering::SeqCst); + advance(1).await; + let failed = fixture.read(503).await; + assert!(failed["details"]["metadata"] + .as_str() + .unwrap() + .contains("2100 ms")); + assert_eq!(fixture.operations.cancelled.load(Ordering::SeqCst), 1); + assert_eq!(warnings.count.load(Ordering::SeqCst), 1); + assert_eq!(warnings.elapsed_ms.load(Ordering::SeqCst), 2100); + assert_eq!(fixture.operations.calls.load(Ordering::SeqCst), 2); + assert_eq!(fixture.operations.active.load(Ordering::SeqCst), 1); + advance(100).await; + assert_eq!(fixture.read(200).await["observation"]["generation"], 2); + advance(799).await; + assert_eq!(fixture.operations.calls.load(Ordering::SeqCst), 2); + advance(1).await; + assert_eq!(fixture.operations.calls.load(Ordering::SeqCst), 3); + assert_eq!(fixture.operations.peak.load(Ordering::SeqCst), 1); + fixture.observer.take(); + settle().await; + assert_eq!(fixture.operations.active.load(Ordering::SeqCst), 0); + assert_eq!(fixture.operations.disk.active.load(Ordering::SeqCst), 0); + assert_eq!(fixture.operations.metrics.snapshot().chunkdb.inflight, 0); + assert_eq!(fixture.operations.metrics.snapshot().diskio.inflight, 0); + assert_eq!(warnings.count.load(Ordering::SeqCst), 1); +} + +#[tokio::test(start_paused = true)] +async fn consecutive_slow_successes_keep_the_three_second_start_age_expiry_and_recover() { + let (warnings, _subscriber) = failure_warnings(); + let fixture = TestSampler::new(1800); + settle().await; + advance(1800).await; + let first = fixture.read(200).await; + assert_eq!(first["observation"]["generation"], 1); + assert_eq!(first["observation"]["max_age_ms"], 3000); + advance(1200).await; + assert_eq!(fixture.read(200).await["observation"]["age_ms"], 3000); + advance(1).await; + let stale = fixture.read(503).await; + assert_eq!(stale["observation"]["state"], "expired"); + assert_eq!(stale["metadata"], "unknown"); + assert_eq!(stale["chunks"], "unknown"); + assert_eq!( + stale["observation"]["checked_at_unix_ms"], + first["observation"]["checked_at_unix_ms"] + ); + assert_eq!(stale["observation"]["generation"], 1); + advance(599).await; + let recovered = fixture.read(200).await; + assert_eq!(recovered["observation"]["generation"], 2); + assert_eq!(recovered["observation"]["age_ms"], 1800); + assert_eq!(fixture.operations.peak.load(Ordering::SeqCst), 1); + assert_eq!(warnings.count.load(Ordering::SeqCst), 0); +} diff --git a/app/crowdb-access-server/tests/common/access_probe_budget.rs b/app/crowdb-access-server/tests/common/access_probe_budget.rs new file mode 100644 index 000000000..5bacf305b --- /dev/null +++ b/app/crowdb-access-server/tests/common/access_probe_budget.rs @@ -0,0 +1,255 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_access_s3::{ + auth::TrustedNetworkAuthenticator, + metrics::{DependencyHealth, S3Health, S3Metrics}, + route::S3Route, +}; +use crowdb_access_server::s3::{ + DependencyProbeFuture, HealthObserver, ProductionS3Operations, S3Dispatcher, S3Operations, + S3OperationsFuture, +}; +use crowdb_chunk_client::{metrics::ReadinessMetrics, ChunkIoClient, DiskWriter, Result}; +use crowdb_kv_client::DiskRecord; +use crowdb_protocol::{ + common::{DiskId, HwStatus}, + diskdb::rpc::{DiskValue, Segment}, +}; +use http_body_util::BodyExt; +use hyper::{body::Incoming, Request}; +use serde_json::Value; +use std::{ + sync::{ + atomic::{AtomicU64, AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::time::Instant; + +pub const HELD_OUTER: u64 = u64::MAX; + +#[derive(Default)] +pub struct TestDisk { + pub active: AtomicUsize, +} + +struct TestRead<'a>(&'a AtomicUsize); +impl Drop for TestRead<'_> { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} + +#[async_trait] +impl DiskWriter for TestDisk { + async fn write(&self, _: &Segment, _: u64, _: Bytes) -> Result<()> { + panic!("probe must not write"); + } + async fn write_at_byte_offset(&self, _: &Segment, _: u64, _: u64, _: Bytes) -> Result<()> { + panic!("probe must not write"); + } + async fn read_priority(&self, segment: &Segment, unit: u64, offset: u64, length: u32) -> Result { + assert!(segment.disk_id.is_some()); + assert_eq!((unit, offset, length), (4096, 0, 4096)); + self.active.fetch_add(1, Ordering::SeqCst); + let _read = TestRead(&self.active); + tokio::time::sleep(Duration::from_millis(100)).await; + Ok(Bytes::from(vec![7; 4096])) + } +} + +pub struct TestOperations { + pub delay_ms: AtomicU64, + pub calls: AtomicUsize, + pub active: AtomicUsize, + pub peak: AtomicUsize, + pub cancelled: AtomicUsize, + pub metrics: Arc, + pub disk: Arc, +} + +struct TestRound<'a> { + state: &'a TestOperations, + completed: bool, +} +impl Drop for TestRound<'_> { + fn drop(&mut self) { + self.state.active.fetch_sub(1, Ordering::SeqCst); + if !self.completed { + self.state.cancelled.fetch_add(1, Ordering::SeqCst); + } + } +} + +fn disks() -> Vec { + vec![DiskRecord { + rack_id: 1, + node_id: 1, + disk_group_id: 1, + disk_id: DiskId { high: 1, low: 1 }, + value: DiskValue { + status: HwStatus::Up as i32, + ..Default::default() + }, + }] +} + +impl S3Operations for TestOperations { + fn observe_dependencies(self: Arc) -> DependencyProbeFuture { + Box::pin(async move { + self.calls.fetch_add(1, Ordering::SeqCst); + let count = self.active.fetch_add(1, Ordering::SeqCst) + 1; + self.peak.fetch_max(count, Ordering::SeqCst); + let mut round = TestRound { + state: self.as_ref(), + completed: false, + }; + let delay = self.delay_ms.load(Ordering::SeqCst); + if delay == HELD_OUTER { + std::future::pending::<()>().await; + } + let metrics = self.metrics.clone(); + let disk = self.disk.clone(); + let observed = ProductionS3Operations::observe_storage_dependencies_for_tests( + async move { + tokio::time::sleep(Duration::from_millis(delay)).await; + Ok("current metadata read".into()) + }, + move |budget| async move { + let online = ChunkIoClient::observe_readiness_dependencies_for_tests( + Instant::now() + budget, + &metrics, + disk, + async { Ok(disks()) }, + async move { + tokio::time::sleep(Duration::from_millis(delay)).await; + Ok(()) + }, + ) + .await + .map_err(|error| error.to_string())?; + Ok(format!("current physical read on {online} disk")) + }, + ) + .await; + round.completed = true; + Some(observed) + }) + } + fn execute( + self: Arc, + _: S3Route, + _: Request, + _: String, + _: String, + ) -> S3OperationsFuture { + panic!("health must not execute application requests"); + } +} + +pub struct TestSampler { + pub dispatcher: Arc, + pub operations: Arc, + pub observer: Option, +} +impl TestSampler { + pub fn new(delay_ms: u64) -> Self { + let operations = Arc::new(TestOperations { + delay_ms: AtomicU64::new(delay_ms), + calls: AtomicUsize::new(0), + active: AtomicUsize::new(0), + peak: AtomicUsize::new(0), + cancelled: AtomicUsize::new(0), + metrics: Arc::default(), + disk: Arc::default(), + }); + let health = Arc::new(S3Health::starting(1)); + health.set_listener(DependencyHealth::Ready); + let dispatcher = Arc::new( + S3Dispatcher::new( + Arc::new(TrustedNetworkAuthenticator::new()), + operations.clone(), + Arc::new(S3Metrics::default()), + "probe-test".into(), + false, + ) + .with_health(health), + ); + let observer = Some(dispatcher.start_health_observer().unwrap()); + Self { + dispatcher, + operations, + observer, + } + } + pub async fn read(&self, status: u16) -> Value { + let response = self.dispatcher.health_response_for_tests(); + assert_eq!(response.status().as_u16(), status); + serde_json::from_slice(&response.into_body().collect().await.unwrap().to_bytes()).unwrap() + } + pub fn generation(&self) -> u64 { + self.dispatcher.health_generation_for_tests() + } +} +impl Drop for TestSampler { + fn drop(&mut self) { + self.observer.take(); + } +} + +pub async fn settle() { + for _ in 0..8 { + tokio::task::yield_now().await; + } +} +pub async fn advance(milliseconds: u64) { + tokio::time::advance(Duration::from_millis(milliseconds)).await; + settle().await; +} + +#[derive(Default)] +pub struct TestFailureWarnings { + pub count: AtomicU64, + pub generation: AtomicU64, + pub budget_ms: AtomicU64, + pub elapsed_ms: AtomicU64, + pub checked_at_unix_ms: AtomicU64, +} + +struct TestWarningLayer(Arc); +struct TestWarningFields<'a>(&'a TestFailureWarnings); +impl tracing::field::Visit for TestWarningFields<'_> { + fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {} + fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { + let target = match field.name() { + "generation" => &self.0.generation, + "configured_budget_ms" => &self.0.budget_ms, + "total_elapsed_ms" => &self.0.elapsed_ms, + "checked_at_unix_ms" => &self.0.checked_at_unix_ms, + _ => return, + }; + target.store(value, Ordering::SeqCst); + } +} +impl tracing_subscriber::Layer for TestWarningLayer { + fn on_event(&self, event: &tracing::Event<'_>, _: tracing_subscriber::layer::Context<'_, S>) { + if event.metadata().level() == &tracing::Level::WARN + && event.metadata().fields().field("configured_budget_ms").is_some() + { + self.0.count.fetch_add(1, Ordering::SeqCst); + event.record(&mut TestWarningFields(self.0.as_ref())); + } + } +} + +pub fn failure_warnings() -> (Arc, tracing::subscriber::DefaultGuard) { + use tracing_subscriber::prelude::*; + let observed = Arc::new(TestFailureWarnings::default()); + let subscriber = tracing_subscriber::registry().with(TestWarningLayer(observed.clone())); + let guard = tracing::subscriber::set_default(subscriber); + (observed, guard) +} diff --git a/app/crowdb-access-server/tests/common/iceberg_client.py b/app/crowdb-access-server/tests/common/iceberg_client.py index 684f5eb5d..0d1a7abe7 100644 --- a/app/crowdb-access-server/tests/common/iceberg_client.py +++ b/app/crowdb-access-server/tests/common/iceberg_client.py @@ -60,7 +60,7 @@ def main(): def verify_namespaces(uri, properties): writer = load_catalog("crowdb", **(properties | {"token": "w" * 32})) namespace = ("client-" + uuid.uuid4().hex,) - child = namespace + ("冰+a%2F",) + child = namespace + ("\u51b0+a%2F",) writer.create_namespace(namespace, {"owner": "original"}) assert writer.namespace_exists(namespace) assert writer.load_namespace_properties(namespace) == {"owner": "original"} diff --git a/app/crowdb-access-server/tests/dataset_batch_header_test.rs b/app/crowdb-access-server/tests/dataset_batch_header_test.rs index 603fdac1e..cb633204b 100644 --- a/app/crowdb-access-server/tests/dataset_batch_header_test.rs +++ b/app/crowdb-access-server/tests/dataset_batch_header_test.rs @@ -12,23 +12,24 @@ const SNAPSHOT: &str = "11111111111111111111111111111111"; #[test] fn ordered_cells_preserve_unicode_missing_tombstone_empty_and_chunk_wire_bytes() { - let (prefix, metadata) = DatasetHttpService::batch_header_for_tests(resolved(Some("游标\"/next"))) - .expect("bounded batch header"); + let (prefix, metadata) = + DatasetHttpService::batch_header_for_tests(resolved(Some("\u{6e38}\u{6807}\"/next"))) + .expect("bounded batch header"); let expected = concat!( "{\"snapshot\":\"11111111111111111111111111111111\",", "\"sample_ids\":[\"YQ==\",\"+/8=\"],\"fields\":[", - "{\"sample\":\"YQ==\",\"field\":\"空\",\"state\":\"value\",\"length\":0,", + "{\"sample\":\"YQ==\",\"field\":\"\u{7a7a}\",\"state\":\"value\",\"length\":0,", "\"md5\":\"d41d8cd98f00b204e9800998ecf8427e\"},", "{\"sample\":\"YQ==\",\"field\":\"erased\",\"state\":\"tombstone\",\"length\":0,\"md5\":null},", "{\"sample\":\"YQ==\",\"field\":\"absent\",\"state\":\"missing\",\"length\":0,\"md5\":null},", "{\"sample\":\"YQ==\",\"field\":\"blob\",\"state\":\"value\",\"length\":1,", "\"md5\":\"9dd4e461268c8034f5c8564e155c67a6\"},", - "{\"sample\":\"+/8=\",\"field\":\"空\",\"state\":\"missing\",\"length\":0,\"md5\":null},", + "{\"sample\":\"+/8=\",\"field\":\"\u{7a7a}\",\"state\":\"missing\",\"length\":0,\"md5\":null},", "{\"sample\":\"+/8=\",\"field\":\"erased\",\"state\":\"missing\",\"length\":0,\"md5\":null},", "{\"sample\":\"+/8=\",\"field\":\"absent\",\"state\":\"missing\",\"length\":0,\"md5\":null},", "{\"sample\":\"+/8=\",\"field\":\"blob\",\"state\":\"value\",\"length\":4294967297,", "\"md5\":\"abababababababababababababababab\"}],", - "\"next\":\"游标\\\"/next\",\"end\":false}" + "\"next\":\"\u{6e38}\u{6807}\\\"/next\",\"end\":false}" ); assert_wire(&prefix, &metadata, expected); } @@ -39,7 +40,7 @@ fn terminal_header_keeps_sample_and_field_order_without_a_cursor() { assert!(metadata.ends_with(b",\"next\":null,\"end\":true}")); let value: serde_json::Value = serde_json::from_slice(&metadata).unwrap(); assert_eq!(value["sample_ids"], serde_json::json!(["YQ==", "+/8="])); - assert_eq!(value["fields"][0]["field"], "空"); + assert_eq!(value["fields"][0]["field"], "\u{7a7a}"); assert_eq!(value["fields"][7]["sample"], "+/8="); assert_eq!(value["next"], serde_json::Value::Null); assert_eq!(value["end"], true); @@ -88,12 +89,14 @@ fn assert_wire(prefix: &Bytes, metadata: &Bytes, expected: &str) { fn resolved(next: Option<&str>) -> ResolvedReadPage { let mut value = empty(next.map(str::to_owned)); value.page.ids = vec![b"a".to_vec(), vec![0xfb, 0xff]]; - value.page.fields = ["空", "erased", "absent", "blob"].map(str::to_owned).to_vec(); + value.page.fields = ["\u{7a7a}", "erased", "absent", "blob"] + .map(str::to_owned) + .to_vec(); value.samples = vec![ SampleView { sample_id: b"a".to_vec(), fields: BTreeMap::from([ - ("空".into(), inline(b"")), + ("\u{7a7a}".into(), inline(b"")), ("erased".into(), FieldLocator::Tombstone), ("blob".into(), inline(b"x")), ]), diff --git a/app/crowdb-access-server/tests/iceberg_file_request_test.rs b/app/crowdb-access-server/tests/iceberg_file_request_test.rs index 739d7940d..7e073253b 100644 --- a/app/crowdb-access-server/tests/iceberg_file_request_test.rs +++ b/app/crowdb-access-server/tests/iceberg_file_request_test.rs @@ -117,7 +117,7 @@ fn native_list_requests_preserve_table_prefix_and_validate_all_selectors() { .unwrap() .unwrap(); assert_eq!(list.table, table); - assert_eq!(list.prefix, format!("{}data/雪%2B", table.object_prefix())); + assert_eq!(list.prefix, format!("{}data/\u{96ea}%2B", table.object_prefix())); assert_eq!(list.delimiter.as_deref(), Some("/")); assert!(list.encoding_url); assert_eq!(list.max_keys, 3); @@ -173,7 +173,7 @@ fn native_object_routes_decode_once_and_preserve_plus_percent_and_repeated_slash ("a+b", "a+b"), ("a%252Fb", "a%2Fb"), ("a//b", "a//b"), - ("%E5%86%B0/a%20b", "冰/a b"), + ("%E5%86%B0/a%20b", "\u{51b0}/a b"), ] { for (method, operation) in [ (Method::GET, FileOperation::Get), diff --git a/app/crowdb-access-server/tests/iceberg_namespace_limits_test.rs b/app/crowdb-access-server/tests/iceberg_namespace_limits_test.rs index a7d065a9a..a35be7263 100644 --- a/app/crowdb-access-server/tests/iceberg_namespace_limits_test.rs +++ b/app/crowdb-access-server/tests/iceberg_namespace_limits_test.rs @@ -79,8 +79,8 @@ async fn property_cardinality_replacement_and_overlap_are_atomic_over_http() { #[tokio::test] async fn property_utf8_key_and_value_byte_limits_fail_without_mutation() { let test = TestTableHttp::new().await; - let key = format!("{}a", "键".repeat(341)); - let value = "值".repeat(2730) + "ab"; + let key = format!("{}a", "\u{952e}".repeat(341)); + let value = "\u{503c}".repeat(2730) + "ab"; update(&test, json!({"updates":{key.clone():value.clone()}}), 200).await; assert_eq!(properties(&test).await, json!({key.clone():value.clone()})); for (invalid_key, invalid_value) in [ diff --git a/app/crowdb-access-server/tests/s3_object_inspection_test.rs b/app/crowdb-access-server/tests/s3_object_inspection_test.rs index bfde1d356..dea267ab2 100644 --- a/app/crowdb-access-server/tests/s3_object_inspection_test.rs +++ b/app/crowdb-access-server/tests/s3_object_inspection_test.rs @@ -163,7 +163,7 @@ async fn inspection_fixture() -> (Arc, S3Dispatcher) { .collect(); let object = ObjectRecord { bucket_id, - key: "folder/中文.bin".as_bytes().to_vec(), + key: "folder/\u{4e2d}\u{6587}.bin".as_bytes().to_vec(), logical_length: 184, checksum: vec![1; 16], etag: "etag".into(), @@ -222,7 +222,7 @@ async fn privileged_http_inspection_reads_two_metadata_keys_and_rejects_stale_ge ); let server = tokio::spawn(serve(listener, Arc::new(dispatcher), std::future::pending())); let http = reqwest::Client::new(); - let query = [("bucket", "bucket"), ("key", "folder/中文.bin")]; + let query = [("bucket", "bucket"), ("key", "folder/\u{4e2d}\u{6587}.bin")]; let response = http.get(&endpoint).query(&query).send().await.unwrap(); assert_eq!(response.status().as_u16(), 401); let response = http diff --git a/app/crowdb-chunk-kv-server/src/server.rs b/app/crowdb-chunk-kv-server/src/server.rs index e6e69884e..60480a628 100644 --- a/app/crowdb-chunk-kv-server/src/server.rs +++ b/app/crowdb-chunk-kv-server/src/server.rs @@ -299,13 +299,6 @@ impl ChunkKvService { self.authority.monotonic_ms() } - /// Stops new admission and revokes successful serving replies. Admitted - /// operations retain their partition handle and can finish durable work. - pub fn begin_drain(&self) { - self.admitting.store(false, Ordering::Release); - self.authority.clear(); - } - #[must_use] pub fn health(&self, now_monotonic_ms: u64) -> ServerHealth { let catalog = self.catalog.load(); diff --git a/app/crowdb-chunk-kv-server/src/server/drain.rs b/app/crowdb-chunk-kv-server/src/server/drain.rs index c489a6d3f..ea1b7b427 100644 --- a/app/crowdb-chunk-kv-server/src/server/drain.rs +++ b/app/crowdb-chunk-kv-server/src/server/drain.rs @@ -1,15 +1,26 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +use std::sync::atomic::Ordering; + use super::{ChunkKvError, ChunkKvService}; impl ChunkKvService { + /// Stops new admission and revokes successful serving replies. Admitted + /// operations retain their partition handle and can finish durable work. + pub fn begin_drain(&self) { + self.admitting.store(false, Ordering::Release); + self.load_sampling.close(); + self.authority.clear(); + } + /// Drain admitted mutations and release each hosted WAL writer for a new boot. /// Background assignment changes must be stopped before calling this method. /// # Errors /// Leaves uncertain or fenced writers subject to their normal lease exclusion. pub async fn drain_writers(&self) -> Result<(), ChunkKvError> { self.begin_drain(); + self.load_sampling.drain().await; let partitions = self.partitions.load_full(); let mut first_error = None; for partition in partitions.values() { diff --git a/app/crowdb-chunk-kv-server/src/server/load_sampling.rs b/app/crowdb-chunk-kv-server/src/server/load_sampling.rs index 04fc2e84a..90ec1d9a6 100644 --- a/app/crowdb-chunk-kv-server/src/server/load_sampling.rs +++ b/app/crowdb-chunk-kv-server/src/server/load_sampling.rs @@ -5,32 +5,82 @@ use std::collections::HashMap; use std::sync::{ - atomic::{AtomicBool, Ordering}, + atomic::{AtomicU8, Ordering}, Arc, }; use arc_swap::ArcSwap; use crowdb_chunk_kv::{ChunkKvError, Partition, PartitionLifecycle}; use crowdb_protocol::chunk_kv::Id128; +use tokio::sync::Notify; use tracing::warn; type Samples = Vec<(Vec, u64)>; +const RUNNING: u8 = 1; +const CLOSED: u8 = 2; + #[derive(Default)] pub(super) struct LoadSampling { - running: AtomicBool, + state: AtomicU8, cached: ArcSwap>, + completed: Notify, } struct SamplingJob(Arc); impl Drop for SamplingJob { fn drop(&mut self) { - self.0.running.store(false, Ordering::Release); + self.0.state.fetch_and(!RUNNING, Ordering::AcqRel); + self.0.completed.notify_waiters(); + } +} + +// Field order also keeps completion last if queued work is dropped before entry. +struct SamplingWork { + partitions: Arc>, + runtime: tokio::runtime::Handle, + completion: SamplingJob, +} + +impl SamplingWork { + fn run(self, limit: usize) { + // Declared first and completed last, including during unwinding. + let completion = self.completion; + let runtime_handle = self.runtime; + let owned_partitions = self.partitions; + runtime_handle.block_on(async { + let mut observations = HashMap::new(); + for (id, partition) in owned_partitions.iter() { + if completion.0.is_closed() { + return; + } + let snapshot = partition.snapshot(); + if snapshot.lifecycle != PartitionLifecycle::Serving { + continue; + } + match sample(partition, snapshot.ownership_epoch, limit).await { + Ok(samples) => { + observations.insert(*id, (snapshot.ownership_epoch, samples)); + } + Err(error) => warn!(partition_id_high = id.high, partition_id_low = id.low, %error, + "split observation failed; partition remains unsampled"), + } + } + if !completion.0.is_closed() { + completion.0.cached.store(Arc::new(observations)); + } + }); + drop(owned_partitions); + drop(runtime_handle); + drop(completion); } } impl LoadSampling { pub(super) fn samples(&self, id: Id128, epoch: u64) -> Samples { + if self.is_closed() { + return Vec::new(); + } self.cached .load() .get(&id) @@ -38,10 +88,33 @@ impl LoadSampling { .map_or_else(Vec::new, |(_, samples)| samples.clone()) } + pub(super) fn close(&self) { + self.state.fetch_or(CLOSED, Ordering::AcqRel); + } + + pub(super) async fn drain(&self) { + self.close(); + loop { + let completed = self.completed.notified(); + tokio::pin!(completed); + completed.as_mut().enable(); + if self.state.load(Ordering::Acquire) & RUNNING == 0 { + // Closed admission and completed payload teardown make this final. + self.cached.store(Arc::new(HashMap::new())); + return; + } + completed.await; + } + } + + fn is_closed(&self) -> bool { + self.state.load(Ordering::Acquire) & CLOSED != 0 + } + pub(super) fn launch(self: &Arc, partitions: Arc>, limit: usize) { if self - .running - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .state + .compare_exchange(0, RUNNING, Ordering::AcqRel, Ordering::Acquire) .is_err() { return; @@ -50,25 +123,12 @@ impl LoadSampling { let runtime = tokio::runtime::Handle::current(); // Native page reads may block on cold storage. One owned job per service // isolates that delay and prevents repeated heartbeat ticks spawning work. - tokio::task::spawn_blocking(move || { - runtime.block_on(async move { - let mut observations = HashMap::new(); - for (id, partition) in partitions.iter() { - let snapshot = partition.snapshot(); - if snapshot.lifecycle != PartitionLifecycle::Serving { - continue; - } - match sample(partition, snapshot.ownership_epoch, limit).await { - Ok(samples) => { - observations.insert(*id, (snapshot.ownership_epoch, samples)); - } - Err(error) => warn!(partition_id_high = id.high, partition_id_low = id.low, %error, - "split observation failed; partition remains unsampled"), - } - } - job.0.cached.store(Arc::new(observations)); - }); - }); + let work = SamplingWork { + partitions, + runtime, + completion: job, + }; + tokio::task::spawn_blocking(move || work.run(limit)); } } diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs index 6b56650fa..be867266b 100644 --- a/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs @@ -160,7 +160,7 @@ impl KvRootCatalogStore { && entry.owner_epoch == prepared.owner_epoch && entry.artifact.tree_id == prepared.artifact.tree_id && entry.artifact.stream_name == prepared.artifact.stream_name - && entry.transition_id == prepared.transition_id + && same_publication_dependency(entry, prepared) && same_recovery_base(entry, prepared) && (entry.state == ChunkKvRangeCatalogPartitionState::Serving || (allow_catching_up @@ -523,6 +523,17 @@ fn decode_u64(value: &[u8]) -> Option { value.try_into().ok().map(u64::from_be_bytes) } +fn same_publication_dependency( + entry: &ChunkKvRangeCatalogEntry, + prepared: &ChunkKvRangeCatalogEntry, +) -> bool { + entry.transition_id == prepared.transition_id + || (entry.state == ChunkKvRangeCatalogPartitionState::Serving + && entry.transition_id.is_none() + && entry.artifact.tail_overlay.is_none() + && prepared.artifact.tail_overlay.is_none()) +} + fn same_recovery_base(entry: &ChunkKvRangeCatalogEntry, prepared: &ChunkKvRangeCatalogEntry) -> bool { match (&entry.artifact.tail_overlay, &prepared.artifact.tail_overlay) { (None, None) => true, diff --git a/app/crowdb-chunk-kv-server/tests/common/sampling_tree.rs b/app/crowdb-chunk-kv-server/tests/common/sampling_tree.rs new file mode 100644 index 000000000..b942a3f77 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/common/sampling_tree.rs @@ -0,0 +1,147 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::{ + any::Any, + future::{poll_fn, Future}, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, + sync::Arc, + task::Poll, + time::Duration, +}; + +use crowdb_chunk_kv::{ + memory::MemoryPartitionTree, MutationOperation, PartitionRange, PartitionTree, Result, ScanEntry, + ValueRevision, +}; +use tokio::sync::Notify; + +pub struct TestSamplingTree { + memory: MemoryPartitionTree, + pub entered: Notify, + pub release: Notify, + pub finished: AtomicBool, + pub active: AtomicBool, + pub started: AtomicUsize, +} + +impl TestSamplingTree { + pub fn new() -> Self { + Self { + memory: MemoryPartitionTree::with_tree_id(71), + entered: Notify::new(), + release: Notify::new(), + finished: AtomicBool::new(false), + active: AtomicBool::new(false), + started: AtomicUsize::new(0), + } + } + + async fn hold_read(&self) { + let mut timer = std::pin::pin!(tokio::time::sleep(Duration::from_secs(30))); + let mut release = std::pin::pin!(self.release.notified()); + let mut announced = false; + poll_fn(|cx| { + // Keep a real registered timer in the production blocking job. + // Runtime shutdown wakes it; polling after shutdown must panic. + let timer = timer.as_mut().poll(cx); + let release = release.as_mut().poll(cx); + if timer.is_ready() || release.is_ready() { + return Poll::Ready(()); + } + if !announced { + announced = true; + self.entered.notify_one(); + } + Poll::Pending + }) + .await; + } +} + +#[async_trait::async_trait] +impl PartitionTree for TestSamplingTree { + fn as_any(&self) -> &dyn Any { + self + } + + fn tree_id(&self) -> u64 { + self.memory.tree_id() + } + + async fn get(&self, key: &[u8]) -> Result> { + self.memory.get(key).await + } + + async fn scan_forward( + &self, + start: Option<&[u8]>, + inclusive: bool, + end: Option<&[u8]>, + limit: usize, + bytes: usize, + ) -> Result<(Vec, bool)> { + self.started.fetch_add(1, Ordering::AcqRel); + self.active.store(true, Ordering::Release); + self.hold_read().await; + let result = self + .memory + .scan_forward(start, inclusive, end, limit, bytes) + .await; + self.active.store(false, Ordering::Release); + self.finished.store(true, Ordering::Release); + result + } + + async fn seek_reverse( + &self, + start: &[u8], + inclusive: bool, + begin: Option<&[u8]>, + ) -> Result> { + self.memory.seek_reverse(start, inclusive, begin).await + } + + async fn scan_reverse( + &self, + start: Option<&[u8]>, + begin: Option<&[u8]>, + limit: usize, + bytes: usize, + ) -> Result<(Vec, bool)> { + self.memory.scan_reverse(start, begin, limit, bytes).await + } + + async fn apply(&self, seq: u64, operation: &MutationOperation) -> Result<()> { + self.memory.apply(seq, operation).await + } + + async fn advance_noop(&self, seq: u64) -> Result<()> { + self.memory.advance_noop(seq).await + } + + async fn checkpoint(&self, offset: u64) -> Result<(u64, u64)> { + self.memory.checkpoint(offset).await + } + + async fn checkpoint_snapshot(&self, offset: u64) -> Result<(u64, u64, Arc)> { + self.memory.checkpoint_snapshot(offset).await + } + + async fn rebuild_range( + &self, + tree_id: u64, + range: &PartitionRange, + config: crowdb_tree_ffi::Config, + ) -> Result<(Arc, crowdb_tree_ffi::RangeRebuildStats)> { + self.memory.rebuild_range(tree_id, range, config).await + } + + fn checkpoint_state(&self) -> Result<(u64, u64)> { + self.memory.checkpoint_state() + } + + fn last_applied_seq(&self) -> u64 { + self.memory.last_applied_seq() + } +} diff --git a/app/crowdb-chunk-kv-server/tests/shutdown_runtime_test.rs b/app/crowdb-chunk-kv-server/tests/shutdown_runtime_test.rs new file mode 100644 index 000000000..7d8a4e22a --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/shutdown_runtime_test.rs @@ -0,0 +1,233 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/sampling_tree.rs"] +mod sampling_tree; +#[path = "common/split_storage.rs"] +mod split_storage; +#[path = "common/process.rs"] +mod test_process; + +use std::{ + future::{poll_fn, Future}, + process::Command, + sync::{atomic::Ordering, Arc, Barrier}, + task::Poll, + time::Duration, +}; + +use crowdb_chunk_kv::{Partition, PartitionId, PartitionLifecycle, PartitionRange}; +use crowdb_chunk_kv_server::ChunkKvService; +use sampling_tree::TestSamplingTree; +use split_storage::TestSplitStorage; + +const CHILD: &str = "CROWDB_TEST_SAMPLING_SHUTDOWN_CHILD"; + +#[test] +fn service_drain_owns_sampling_until_runtime_shutdown() { + isolated_child( + "service_drain_owns_sampling_until_runtime_shutdown", + run_shutdown_child, + ); +} + +fn isolated_child(name: &str, run: fn()) { + if std::env::var_os(CHILD).is_some() { + run(); + return; + } + let output = Command::new("pixi") + .arg("run") + .args(["timeout", "--signal=KILL", "20s"]) + .arg(std::env::current_exe().unwrap()) + .args(["--exact", name, "--nocapture"]) + .env(CHILD, "1") + .env("RUST_BACKTRACE", "full") + .output() + .expect("run isolated shutdown regression"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "shutdown child {name} failed: {}\nstdout:\n{stdout}\nstderr:\n{stderr}", + output.status + ); + assert!( + !stderr.contains("but it is being shutdown"), + "timer polled after shutdown:\n{stderr}" + ); +} + +fn run_shutdown_child() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .unwrap(); + let escaped = runtime.block_on(async { + let storage = TestSplitStorage::new(); + let tree = Arc::new(TestSamplingTree::new()); + let (partition, _) = storage + .partition( + PartitionId { high: 71, low: 1 }, + PartitionRange::default(), + 1, + tree.clone(), + ) + .await; + let service = ChunkKvService::new(test_process::TestProcess::authority(1), 8).unwrap(); + service.install_partition(&partition).unwrap(); + service + .registry_observation_with_load_samples(4096, 0, 2) + .await + .unwrap(); + tree.entered.notified().await; + println!("production sampler registered its timer"); + + service.begin_drain(); + // Finish the real journal release first, so unrelated writer scheduling + // cannot make a WAL-only service drain look like a sampler barrier. + partition.suspend_for_shutdown(1).await.unwrap(); + let mut drain = std::pin::pin!(service.drain_writers()); + let first = poll_fn(|cx| Poll::Ready(drain.as_mut().poll(cx))).await; + match first { + Poll::Ready(result) => { + result.unwrap(); + assert!(!tree.finished.load(Ordering::Acquire)); + println!("service drain returned with the production sampling read still active"); + true + } + Poll::Pending => { + tree.release.notify_one(); + drain.await.unwrap(); + assert!(tree.finished.load(Ordering::Acquire)); + assert_eq!( + partition.snapshot().lifecycle, + PartitionLifecycle::TransferQuiesced + ); + false + } + } + }); + drop(runtime); + assert!( + !escaped, + "service drain completed before its sampling job; runtime shutdown overtook owned storage work" + ); +} + +fn sampling_service( + runtime: &tokio::runtime::Runtime, +) -> (Arc, Partition, Arc) { + runtime.block_on(async { + let storage = TestSplitStorage::new(); + let tree = Arc::new(TestSamplingTree::new()); + let (partition, _) = storage + .partition( + PartitionId { high: 71, low: 1 }, + PartitionRange::default(), + 1, + tree.clone(), + ) + .await; + let service = Arc::new(ChunkKvService::new(test_process::TestProcess::authority(1), 8).unwrap()); + service.install_partition(&partition).unwrap(); + (service, partition, tree) + }) +} + +#[test] +fn observation_after_drain_admission_cannot_launch_sampling() { + isolated_child( + "observation_after_drain_admission_cannot_launch_sampling", + run_closed_observation_child, + ); +} + +fn run_closed_observation_child() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .unwrap(); + let (service, partition, tree) = sampling_service(&runtime); + // Keep the partition Serving: fencing its WAL first would let the old + // sampler skip it for an unrelated reason and conceal missing admission. + assert_eq!(partition.snapshot().lifecycle, PartitionLifecycle::Serving); + service.begin_drain(); + runtime + .block_on(service.registry_observation_with_load_samples(4096, 0, 2)) + .unwrap(); + drop(runtime); + assert_eq!(tree.started.load(Ordering::Acquire), 0); +} + +#[test] +fn concurrent_observation_and_close_cannot_escape_completed_drain() { + isolated_child( + "concurrent_observation_and_close_cannot_escape_completed_drain", + run_concurrent_close_child, + ); +} + +fn run_concurrent_close_child() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .unwrap(); + let (service, partition, tree) = sampling_service(&runtime); + let barrier = Arc::new(Barrier::new(3)); + let observer = { + let service = service.clone(); + let barrier = barrier.clone(); + let handle = runtime.handle().clone(); + std::thread::spawn(move || { + let _entered = handle.enter(); + barrier.wait(); + handle + .block_on(service.registry_observation_with_load_samples(4096, 0, 2)) + .unwrap(); + }) + }; + let closer = { + let service = service.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + service.begin_drain(); + }) + }; + barrier.wait(); + observer.join().unwrap(); + closer.join().unwrap(); + runtime.block_on(async { + // Remove WAL-release scheduling as a competing reason to be Pending. + partition.suspend_for_shutdown(1).await.unwrap(); + let mut drain = std::pin::pin!(service.drain_writers()); + if tree.active.load(Ordering::Acquire) { + let first = poll_fn(|cx| Poll::Ready(drain.as_mut().poll(cx))).await; + assert!(first.is_pending(), "drain escaped an active sampling read"); + } + // Also releases a read that passed lifecycle validation just before + // fencing but has not yet reached the injected storage boundary. + tree.release.notify_one(); + drain.await.unwrap(); + assert!(!tree.active.load(Ordering::Acquire)); + }); + drop(service); + drop(partition); + // The ordinary mutation worker also owns the tree until its closed + // receiver is polled. This is eventual fixture cleanup, not the sampler + // barrier oracle above. + runtime.block_on(async { + tokio::time::timeout(Duration::from_secs(1), async { + while Arc::strong_count(&tree) != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("ordinary partition worker did not release the fixture tree"); + }); + drop(runtime); +} diff --git a/app/crowdb-chunk-kv-server/tests/transfer_storage_test.rs b/app/crowdb-chunk-kv-server/tests/transfer_storage_test.rs index 51089e6ac..ddadaca8d 100644 --- a/app/crowdb-chunk-kv-server/tests/transfer_storage_test.rs +++ b/app/crowdb-chunk-kv-server/tests/transfer_storage_test.rs @@ -163,3 +163,89 @@ async fn only_published_matching_target_can_claim_mutable_root() { .await .is_err()); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn released_recovery_marker_does_not_block_the_first_target_checkpoint() { + let metadata = metadata::TestMetadata::start().await; + let kv = metadata.client.clone(); + let source_entry = source(); + let source = open_root_catalog_for_tests(kv.clone(), 0, 1, &source_entry) + .await + .unwrap(); + source.publish(42, 0, 1, 1, b"source-root").unwrap(); + publish_assignment(kv.clone(), source_entry.clone(), 1).await; + + // Lease-excluded recovery adopts the original stream without an overlay. + let mut recovered_entry = source_entry; + recovered_entry.owner.instance_id = 2; + recovered_entry.owner_epoch = 2; + recovered_entry.state = ChunkKvRangeCatalogPartitionState::Prepared; + recovered_entry.transition_id = Some(Id128 { high: 5, low: 6 }); + let recovered = open_root_catalog_for_tests(kv.clone(), 0, 1, &recovered_entry) + .await + .unwrap(); + assert_eq!( + recovered.publish(42, 1, 2, 2, b"target-root"), + Err(CtError::Unavailable) + ); + + // The monitor may release the completed marker before the target's first + // checkpoint; the exact Serving assignment still establishes ownership. + recovered_entry.state = ChunkKvRangeCatalogPartitionState::Serving; + recovered_entry.transition_id = None; + publish_assignment(kv.clone(), recovered_entry.clone(), 2).await; + recovered.publish(42, 1, 2, 2, b"target-root").unwrap(); + assert_eq!( + source.publish(42, 2, 1, 3, b"stale-source"), + Err(CtError::Unavailable) + ); + let reopened = open_root_catalog_for_tests(kv, 0, 1, &recovered_entry) + .await + .unwrap(); + assert_eq!( + reopened.load(42, RootCatalogObject::CurrentManifest).unwrap(), + Some(b"target-root".to_vec()) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn released_marker_on_another_assignment_does_not_authorize_the_prepared_root() { + for mismatch in ["owner", "epoch", "tree", "stream"] { + let metadata = metadata::TestMetadata::start().await; + let kv = metadata.client.clone(); + let source_entry = source(); + let source = open_root_catalog_for_tests(kv.clone(), 0, 1, &source_entry) + .await + .unwrap(); + source.publish(42, 0, 1, 1, b"source-root").unwrap(); + publish_assignment(kv.clone(), source_entry.clone(), 1).await; + let mut prepared = source_entry; + prepared.owner.instance_id = 2; + prepared.owner_epoch = 2; + prepared.state = ChunkKvRangeCatalogPartitionState::Prepared; + prepared.transition_id = Some(Id128 { high: 5, low: 6 }); + let target = open_root_catalog_for_tests(kv.clone(), 0, 1, &prepared) + .await + .unwrap(); + let mut published = prepared; + published.state = ChunkKvRangeCatalogPartitionState::Serving; + published.transition_id = None; + match mismatch { + "owner" => published.owner.instance_id = 3, + "epoch" => published.owner_epoch = 3, + "tree" => published.artifact.tree_id = 43, + "stream" => published.artifact.stream_name.low = 3, + _ => unreachable!(), + } + publish_assignment(kv, published, 2).await; + assert_eq!( + target.publish(42, 1, 2, 2, b"unpublished-target"), + Err(CtError::Unavailable), + "released marker must not bypass {mismatch} identity" + ); + assert_eq!( + source.load(42, RootCatalogObject::CurrentManifest).unwrap(), + Some(b"source-root".to_vec()) + ); + } +} diff --git a/app/crowdb-chunkdb/Cargo.toml b/app/crowdb-chunkdb/Cargo.toml index cacc1ccb0..a7b7bfffd 100644 --- a/app/crowdb-chunkdb/Cargo.toml +++ b/app/crowdb-chunkdb/Cargo.toml @@ -53,6 +53,7 @@ rand = "0.8" futures = "0.3" [dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } crowdb-console-shared = { path = "../../lib/crowdb-console-shared" } crowdb-kv = { path = "../../lib/crowdb-kv", features = ["test-util"] } crowdb-chunkdb-client = { path = "../../lib/crowdb-chunkdb-client" } diff --git a/app/crowdb-chunkdb/src/lib.rs b/app/crowdb-chunkdb/src/lib.rs index 0278dd1a6..50d5f5396 100644 --- a/app/crowdb-chunkdb/src/lib.rs +++ b/app/crowdb-chunkdb/src/lib.rs @@ -21,6 +21,7 @@ pub mod metrics; pub mod placement_rebalance; pub mod placement_repair; pub mod range_guard; +pub mod readiness; pub mod relocation; pub mod repair; pub mod routing; diff --git a/app/crowdb-chunkdb/src/main.rs b/app/crowdb-chunkdb/src/main.rs index d9d6b7a9d..8a8741091 100644 --- a/app/crowdb-chunkdb/src/main.rs +++ b/app/crowdb-chunkdb/src/main.rs @@ -165,11 +165,11 @@ async fn main() { .parse() .expect("valid rpc_listen_addr"); - // Build KV client for group-0 topology access. + // Share discovery and metadata while keeping point reads off scan frames. let mut kv_config = ClientConfig::new(config.server.kv_server_mgmt_seeds.clone()); kv_config.pool_size_per_endpoint = config.server.kv_pool_size; kv_config.rpc_workers = config.server.kv_rpc_workers; - let kv = Arc::new(CrowdbKvClient::new(kv_config)); + let kv = Arc::new(CrowdbKvClient::new_with_isolated_point_reads(kv_config)); let hw = HardwareClient::from_shared(Arc::clone(&kv)); let refresh_hw = HardwareClient::from_shared(Arc::clone(&kv)); let watch = WatchNotifyClient::from_shared(Arc::clone(&kv)); @@ -325,35 +325,10 @@ async fn main() { }, ); - let range_refresh_guard = Arc::clone(&range_guard); - let range_refresh_kv = Arc::clone(&kv); - let range_refresh_stop = stop_rx.clone(); - let range_refresh_instance = config - .server - .instance_id - .as_ref() - .and_then(|value| value.parse::().ok()); - let range_refresh_handle = tokio::spawn(async move { - let Some(instance_id) = range_refresh_instance else { - return; - }; - let mut stop = range_refresh_stop; - let mut ticker = tokio::time::interval(Duration::from_secs(1)); - loop { - tokio::select! { - _ = ticker.tick() => { - if let Err(error) = range_refresh_guard.load_from_group0(&range_refresh_kv, instance_id).await { - warn!(%error, instance_id, "periodic range guard refresh failed"); - } - } - changed = stop.changed() => { - if changed.is_ok() && *stop.borrow() { - return; - } - } - } - } - }); + let readiness = Arc::new(crowdb_chunkdb::readiness::RangeReadiness::new(Arc::clone( + &range_guard, + ))); + let range_refresh_handle = Arc::clone(&readiness).start(Arc::clone(&kv), instance_id, stop_rx.clone()); // Diskdb client pool + chunk allocator. let pool = Arc::new(DiskdbClientPool::with_transport( @@ -454,15 +429,6 @@ async fn main() { info!(%rpc_listen_addr, "crowdb-rpc server listening"); // Start HTTP health + metrics + cache invalidation server. - let readiness = HttpReadiness { - range_guard: Arc::clone(&range_guard), - kv: Arc::clone(&kv), - instance_id: config - .server - .instance_id - .as_ref() - .and_then(|value| value.parse().ok()), - }; let http_handle = tokio::spawn(run_http_server( http_listen_addr, readiness, @@ -605,30 +571,17 @@ fn spawn_chunkdb_keepalive( Some(handle) } -struct HttpReadiness { - range_guard: Arc, - kv: Arc, - instance_id: Option, -} - /// HTTP server — health, metrics, cache invalidation endpoints. async fn run_http_server( addr: SocketAddr, - readiness: HttpReadiness, + readiness: Arc, locks: Arc, conversion_metrics: Arc, repair_metrics: Arc, conversion: [Arc; 2], ) { - let app = axum::Router::new() - .route( - "/ready", - axum::routing::get(move || { - let range_guard = Arc::clone(&readiness.range_guard); - let kv = Arc::clone(&readiness.kv); - async move { ready_response(&range_guard, &kv, readiness.instance_id).await } - }), - ) + let app = readiness + .router() .route("/health", axum::routing::get(|| async { "ok" })) .route( "/metrics", @@ -714,29 +667,6 @@ async fn run_http_server( axum::serve(listener, app).await.expect("HTTP server error"); } -async fn ready_response( - range_guard: &RangeGuard, - kv: &Arc, - instance_id: Option, -) -> (axum::http::StatusCode, &'static str) { - if let Some(instance_id) = instance_id { - if range_guard.load_from_group0(kv, instance_id).await.is_err() { - return ( - axum::http::StatusCode::SERVICE_UNAVAILABLE, - "range ownership refresh failed", - ); - } - } - if range_guard.is_ready() { - (axum::http::StatusCode::OK, "ok") - } else { - ( - axum::http::StatusCode::SERVICE_UNAVAILABLE, - "range ownership pending", - ) - } -} - fn load_config(args: &Cli) -> ChunkdbConfig { let config_path = &args.config; let mut config = diff --git a/app/crowdb-chunkdb/src/readiness.rs b/app/crowdb-chunkdb/src/readiness.rs new file mode 100644 index 000000000..fbe4320e3 --- /dev/null +++ b/app/crowdb-chunkdb/src/readiness.rs @@ -0,0 +1,142 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! HTTP readers observe bounded ownership evidence without driving authority reads. + +use crate::range_guard::RangeGuard; +use arc_swap::ArcSwapOption; +use axum::{http::StatusCode, routing::get, Router}; +use crowdb_kv_client::CrowdbKvClient; +use std::{future::Future, sync::Arc, time::Duration}; +use tokio::{ + sync::watch, + task::JoinHandle, + time::{Instant, MissedTickBehavior}, +}; + +const MAX_AGE: Duration = Duration::from_secs(3); + +struct Observation { + started: Instant, + succeeded: bool, +} + +pub struct RangeReadiness { + guard: Arc, + latest: ArcSwapOption, +} + +impl RangeReadiness { + pub fn new(guard: Arc) -> Self { + Self { + guard, + latest: ArcSwapOption::empty(), + } + } + + /// Use the same independently refreshed evidence for every HTTP reader. + pub fn router(self: Arc) -> Router { + Router::new().route( + "/ready", + get(move || { + let readiness = Arc::clone(&self); + async move { readiness.response() } + }), + ) + } + + fn response(&self) -> (StatusCode, &'static str) { + let Some(observed) = self.latest.load_full() else { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "range ownership observation pending", + ); + }; + if observed.started.elapsed() > MAX_AGE { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "range ownership observation expired", + ); + } + if !observed.succeeded { + return (StatusCode::SERVICE_UNAVAILABLE, "range ownership refresh failed"); + } + if self.guard.is_ready() { + (StatusCode::OK, "ok") + } else { + (StatusCode::SERVICE_UNAVAILABLE, "range ownership pending") + } + } + + /// Own one bounded refresh at a time; callers never initiate extra reads. + pub fn start( + self: Arc, + kv: Arc, + instance_id: u64, + stop: watch::Receiver, + ) -> JoinHandle<()> { + let guard = Arc::clone(&self.guard); + self.start_source(stop, move || { + let guard = Arc::clone(&guard); + let kv = Arc::clone(&kv); + async move { + guard + .load_from_group0(&kv, instance_id) + .await + .map_err(|error| error.to_string()) + } + }) + } + + fn start_source(self: Arc, mut stop: watch::Receiver, source: F) -> JoinHandle<()> + where + F: Fn() -> Fut + Send + 'static, + Fut: Future> + Send, + { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(Duration::from_secs(1)); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + if *stop.borrow() { + return; + } + tokio::select! { + _ = ticker.tick() => {}, + _ = stop.changed() => return, + } + tokio::select! { + () = self.sample(source()) => {}, + _ = stop.changed() => return, + } + } + }) + } + + async fn sample(&self, source: impl Future>) { + let started = Instant::now(); + let result = tokio::time::timeout(MAX_AGE, source) + .await + .unwrap_or_else(|_| Err("range ownership refresh exceeded its three-second deadline".into())); + self.latest.store(Some(Arc::new(Observation { + started, + succeeded: result.is_ok(), + }))); + if let Err(error) = result { + tracing::warn!(%error, "periodic range guard refresh failed; readiness remains unavailable"); + } + } + + /// Exercise production scheduling, expiry and cancellation with deterministic I/O. + #[cfg(feature = "test-util")] + pub fn start_source_for_tests( + self: Arc, + stop: watch::Receiver, + source: F, + ) -> JoinHandle<()> + where + F: Fn() -> Fut + Send + 'static, + Fut: Future> + Send, + { + self.start_source(stop, source) + } +} diff --git a/app/crowdb-chunkdb/src/storage/scope.rs b/app/crowdb-chunkdb/src/storage/scope.rs index 864b1364f..abcd1f2a0 100644 --- a/app/crowdb-chunkdb/src/storage/scope.rs +++ b/app/crowdb-chunkdb/src/storage/scope.rs @@ -48,7 +48,8 @@ impl ChunkStore { } /// Keys begin with the owning chunk ID immediately after the prefix. - /// Continue past foreign slots until the requested owned result is filled. + /// Exclude other maintenance domains at KV, then continue past foreign + /// slots until the requested owned result is filled. pub(super) async fn scan_owned_records( &self, prefix: &[u8], @@ -58,6 +59,10 @@ impl ChunkStore { if max_keys == 0 || self.scope.as_ref().is_some_and(|(guard, _)| guard.is_empty()) { return Ok(Vec::new()); } + let (start, end) = self.scan_domain_bounds(prefix, start_after); + if !end.is_empty() && start.as_ref() >= end.as_slice() { + return Ok(Vec::new()); + } let table = self.bindings.snapshot(); if table.is_empty() { return Err(crate::routing::RouteError::NoBinding.into()); @@ -65,7 +70,7 @@ impl ChunkStore { let limit = usize::try_from(max_keys).unwrap_or(usize::MAX); let mut selected = Vec::new(); for route in table.bindings() { - let mut cursor = Bytes::copy_from_slice(start_after); + let mut cursor = start.clone(); let mut cutoff = 0; let mut matched = 0; loop { @@ -76,7 +81,7 @@ impl ChunkStore { route.kv_group_id, prefix, &cursor, - &[], + &end, 256, false, None, @@ -112,6 +117,23 @@ impl ChunkStore { } Ok(selected) } + + fn scan_domain_bounds(&self, prefix: &[u8], start_after: &[u8]) -> (Bytes, Vec) { + let Some((_, domain)) = &self.scope else { + return (Bytes::copy_from_slice(start_after), Vec::new()); + }; + // The purpose byte leads the big-endian ChunkId in both metadata + // namespaces. A short lower bound still includes its zero-valued ID. + let (first, end) = match domain { + ChunkDomain::System => (1, 4), + ChunkDomain::UserData => (4, 8), + }; + let mut lower = prefix.to_vec(); + lower.push(first); + let mut upper = prefix.to_vec(); + upper.push(end); + (Bytes::copy_from_slice(start_after.max(lower.as_slice())), upper) + } } fn key_chunk_id(key: &[u8], offset: usize) -> Result { diff --git a/app/crowdb-chunkdb/tests/common/http_readiness.rs b/app/crowdb-chunkdb/tests/common/http_readiness.rs new file mode 100644 index 000000000..f398ac4ca --- /dev/null +++ b/app/crowdb-chunkdb/tests/common/http_readiness.rs @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{body::Body, http::Request}; +use crowdb_chunkdb::{range_guard::RangeGuard, readiness::RangeReadiness}; +use crowdb_protocol::chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{sync::watch, task::JoinHandle}; +use tower::ServiceExt; + +pub struct TestReadiness { + pub guard: Arc, + pub readiness: Arc, + pub mode: Arc, + pub calls: Arc, + pub stop: watch::Sender, + pub worker: JoinHandle<()>, +} + +impl TestReadiness { + pub fn new(initialized: bool, instance: u64) -> Self { + let guard = Arc::new(RangeGuard::new()); + if initialized { + guard.install(&layout().service_map().unwrap(), instance).unwrap(); + } + let readiness = Arc::new(RangeReadiness::new(Arc::clone(&guard))); + let mode = Arc::new(AtomicUsize::new(0)); + let calls = Arc::new(AtomicUsize::new(0)); + let (stop, receiver) = watch::channel(false); + let worker = Arc::clone(&readiness).start_source_for_tests(receiver, { + let mode = Arc::clone(&mode); + let calls = Arc::clone(&calls); + move || { + let mode = mode.load(Ordering::Relaxed); + calls.fetch_add(1, Ordering::Relaxed); + async move { + match mode { + 1 => tokio::time::sleep(Duration::from_millis(1500)).await, + 2 => std::future::pending::<()>().await, + 3 => return Err("authority unavailable".into()), + _ => {} + } + Ok(()) + } + } + }); + Self { + guard, + readiness, + mode, + calls, + stop, + worker, + } + } + + pub async fn read(&self) -> u16 { + read(Arc::clone(&self.readiness)).await + } + + pub async fn settle() { + for _ in 0..16 { + tokio::task::yield_now().await; + } + } +} + +pub fn layout() -> ChunkSlotBootstrap { + ChunkSlotBootstrap { + service_instances: vec![1], + storage_groups: vec![ChunkStorageGroup { + store_id: 0, + group_id: 1, + }], + } +} + +pub async fn read(readiness: Arc) -> u16 { + tokio::time::timeout( + Duration::from_secs(1), + readiness + .router() + .oneshot(Request::builder().uri("/ready").body(Body::empty()).unwrap()), + ) + .await + .expect("HTTP readiness waited for authority I/O") + .unwrap() + .status() + .as_u16() +} diff --git a/app/crowdb-chunkdb/tests/common/readiness_failures.rs b/app/crowdb-chunkdb/tests/common/readiness_failures.rs new file mode 100644 index 000000000..c50dc1fd9 --- /dev/null +++ b/app/crowdb-chunkdb/tests/common/readiness_failures.rs @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunkdb::range_guard::RangeGuard; +use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbClientError}; +use crowdb_kv_client::CrowdbKvClient; +use crowdb_protocol::chunk_id::{ChunkIdParts, CHUNK_TYPE_S3}; +use crowdb_rpc_ffi::RpcServer; +use std::time::Duration; + +pub async fn assert_storage_and_owner_failures( + client: &ChunkdbClient, + data: &CrowdbKvClient, + control: &CrowdbKvClient, + guard: &RangeGuard, + server: &RpcServer, +) { + // Corruption is an error, not an authoritative absent key. + let mut key = b"/chunk/".to_vec(); + key.extend_from_slice( + &ChunkIdParts { + high: u64::from(CHUNK_TYPE_S3) << 56, + low: 0, + } + .to_bytes(), + ); + data.put(0, 1, &key, b"corrupt chunk record", None).await.unwrap(); + assert!(matches!( + Box::pin(client.observe_readiness()).await, + Err(ChunkdbClientError::Internal(_)) + )); + // A stale cached owner rejects the probe and requests authoritative refresh. + guard.replace_for_tests(&[]); + let _ = control.drain_window(); + let rejected = + tokio::time::timeout(Duration::from_millis(750), Box::pin(client.observe_readiness())).await; + let rescanned = control.drain_window().scan.len(); + server.stop(); + assert!( + !matches!(rejected, Ok(Ok(()))), + "a rejected owner cannot be Ready" + ); + assert!( + rescanned > 0, + "ownership rejection must still refresh the registry" + ); +} diff --git a/app/crowdb-chunkdb/tests/common/readiness_registry.rs b/app/crowdb-chunkdb/tests/common/readiness_registry.rs new file mode 100644 index 000000000..adf4896fc --- /dev/null +++ b/app/crowdb-chunkdb/tests/common/readiness_registry.rs @@ -0,0 +1,35 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunkdb_client::ChunkdbClient; +use crowdb_kv_client::CrowdbKvClient; +use crowdb_rpc_ffi::RpcServer; +use std::time::Duration; + +pub async fn assert_warm_registry_independence( + client: &ChunkdbClient, + control: &CrowdbKvClient, + data: &CrowdbKvClient, + server: &RpcServer, + endpoint: String, +) { + // Keep the real owner/storage healthy while the warmed client's Group0 + // route fails after ordinary registry refresh becomes due. + tokio::time::sleep(Duration::from_millis(5100)).await; + control.seed_leader(0, 0, "127.0.0.1:0".into()); + let _ = control.drain_window(); + let _ = data.drain_window(); + let observed = + tokio::time::timeout(Duration::from_millis(750), Box::pin(client.observe_readiness())).await; + control.seed_leader(0, 0, endpoint); + if !matches!(observed, Ok(Ok(()))) { + server.stop(); + } + assert!( + matches!(observed, Ok(Ok(()))), + "a validated route and current point reads must not wait for a registry scan: {observed:?}" + ); + let work = control.drain_window(); + assert_eq!(work.get.len() + work.scan.len(), 0); + assert_eq!(data.drain_window().get.len(), 2); +} diff --git a/app/crowdb-chunkdb/tests/http_readiness_test.rs b/app/crowdb-chunkdb/tests/http_readiness_test.rs new file mode 100644 index 000000000..ebaa33153 --- /dev/null +++ b/app/crowdb-chunkdb/tests/http_readiness_test.rs @@ -0,0 +1,118 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/http_readiness.rs"] +mod http_readiness; +#[path = "common/slot_groups.rs"] +#[allow(dead_code)] +mod slot_groups; + +use crowdb_chunkdb::{range_guard::RangeGuard, readiness::RangeReadiness}; +use crowdb_kv_client::ChunkSlotMapClient; +use http_readiness::TestReadiness; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +#[tokio::test] +async fn http_readers_share_real_validated_ownership_without_group_zero_work() { + let groups = slot_groups::TestGroups::start().await; + ChunkSlotMapClient::new(Arc::clone(&groups.kv)) + .initialize_service(&http_readiness::layout().service_map().unwrap()) + .await + .unwrap(); + let guard = Arc::new(RangeGuard::new()); + let readiness = Arc::new(RangeReadiness::new(guard)); + assert_eq!(http_readiness::read(Arc::clone(&readiness)).await, 503); + let (stop, receiver) = tokio::sync::watch::channel(false); + let worker = Arc::clone(&readiness).start(Arc::clone(&groups.kv), 1, receiver); + tokio::time::timeout(Duration::from_secs(3), async { + while http_readiness::read(Arc::clone(&readiness)).await != 200 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("background ownership sample did not finish"); + stop.send(true).unwrap(); + worker.await.unwrap(); + let _ = groups.kv.drain_window(); + let responses = + futures::future::join_all((0..32).map(|_| http_readiness::read(Arc::clone(&readiness)))).await; + assert!(responses.iter().all(|status| *status == 200)); + let work = groups.kv.drain_window(); + assert_eq!( + work.get.len() + work.scan.len() + work.put.len() + work.batch_write.len(), + 0 + ); + tokio::time::sleep(Duration::from_millis(3050)).await; + assert_eq!( + http_readiness::read(readiness).await, + 503, + "readers cannot renew evidence" + ); +} + +#[tokio::test(start_paused = true)] +async fn slow_ownership_refresh_is_independent_of_http_readers() { + let test = TestReadiness::new(true, 1); + test.mode.store(1, Ordering::Relaxed); + TestReadiness::settle().await; + assert_eq!(test.calls.load(Ordering::Relaxed), 1); + for _ in 0..32 { + assert_eq!(test.read().await, 503); + } + assert_eq!(test.calls.load(Ordering::Relaxed), 1); + tokio::time::advance(Duration::from_millis(1500)).await; + TestReadiness::settle().await; + assert_eq!( + test.read().await, + 200, + "a completed valid sample is readable within one second" + ); + test.stop.send(true).unwrap(); + test.worker.await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn stalled_authority_expires_without_renewal_and_recovers_without_reader_traffic() { + let test = TestReadiness::new(true, 1); + TestReadiness::settle().await; + assert_eq!(test.read().await, 200); + test.mode.store(2, Ordering::Relaxed); + tokio::time::advance(Duration::from_secs(1)).await; + TestReadiness::settle().await; + assert_eq!(test.read().await, 200); + tokio::time::advance(Duration::from_millis(2050)).await; + TestReadiness::settle().await; + assert_eq!(test.read().await, 503); + tokio::time::advance(Duration::from_secs(1)).await; + TestReadiness::settle().await; + assert_eq!( + test.read().await, + 503, + "deadline errors cannot publish healthy evidence" + ); + test.mode.store(0, Ordering::Relaxed); + tokio::time::advance(Duration::from_secs(3)).await; + TestReadiness::settle().await; + assert_eq!(test.read().await, 200); + test.stop.send(true).unwrap(); + test.worker.await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn authority_errors_and_uninitialized_maps_are_unready_but_idle_initialized_owners_are_ready() { + for (initialized, instance, expected) in [(false, 1, 503), (true, 2, 200)] { + let test = TestReadiness::new(initialized, instance); + TestReadiness::settle().await; + assert_eq!(test.read().await, expected); + assert!(test.guard.is_empty()); + test.mode.store(3, Ordering::Relaxed); + tokio::time::advance(Duration::from_secs(1)).await; + TestReadiness::settle().await; + assert_eq!(test.read().await, 503); + test.stop.send(true).unwrap(); + test.worker.await.unwrap(); + } +} diff --git a/app/crowdb-chunkdb/tests/maintenance_domain_scan_test.rs b/app/crowdb-chunkdb/tests/maintenance_domain_scan_test.rs new file mode 100644 index 000000000..0bd200843 --- /dev/null +++ b/app/crowdb-chunkdb/tests/maintenance_domain_scan_test.rs @@ -0,0 +1,128 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunkdb::range_guard::RangeGuard; +use crowdb_chunkdb::routing::{BindingCache, BindingTable}; +use crowdb_chunkdb::storage::ChunkStore; +use crowdb_protocol::chunk_domain::ChunkDomain; +use crowdb_protocol::chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, StripReservationGroup}; +use crowdb_protocol::common::ChunkId; +use std::sync::Arc; + +#[path = "common/slot_groups.rs"] +#[allow(dead_code)] +mod slot_groups; +use slot_groups::TestGroups; + +async fn seed(writer: &ChunkStore, purpose: u8, low: u64) -> ChunkId { + let id = ChunkId { + high: u64::from(purpose) << 56, + low, + }; + writer + .put_chunk(&Chunk { + id: Some(id), + modify_ts: 1, + state: ChunkState::Active as i32, + chunk_type: i32::from(purpose), + ..Default::default() + }) + .await + .unwrap(); + writer + .put_reservation_group(&StripReservationGroup { + chunk_id: Some(id), + group_id: Some(id), + ..Default::default() + }) + .await + .unwrap(); + id +} + +async fn assert_domain_scan(domain: ChunkDomain, foreign: u8, purposes: &[u8]) { + let test = TestGroups::start().await; + let layout = ChunkSlotBootstrap { + service_instances: vec![1], + storage_groups: vec![ChunkStorageGroup { + store_id: 0, + group_id: 1, + }], + }; + let bindings = BindingCache::new(); + bindings + .replace(BindingTable::new(layout.storage_map().unwrap())) + .unwrap(); + let guard = Arc::new(RangeGuard::new()); + guard.install(&layout.service_map().unwrap(), 1).unwrap(); + let writer = ChunkStore::new(Arc::clone(&test.kv), bindings.clone()); + let reader = ChunkStore::new(Arc::clone(&test.kv), bindings).with_scope(guard, domain); + for low in 1..=600 { + seed(&writer, foreign, low).await; + } + let mut expected = Vec::new(); + for purpose in purposes { + expected.push(seed(&writer, *purpose, 0).await); + } + let _ = test.kv.drain_window(); + let chunks = reader.list_chunks(None, 100).await.unwrap(); + assert_eq!( + chunks.iter().map(|chunk| chunk.id.unwrap()).collect::>(), + expected + ); + assert_eq!( + test.kv.drain_window().scan.len(), + 1, + "a scoped scan must not transfer pages belonging only to another maintenance domain" + ); + let reservations = reader.scan_reservation_groups(100).await.unwrap(); + assert_eq!( + reservations + .iter() + .map(|group| group.chunk_id.unwrap()) + .collect::>(), + expected + ); + assert_eq!(test.kv.drain_window().scan.len(), 1); + assert_eq!( + writer.list_chunks(None, 1000).await.unwrap().len(), + 600 + expected.len() + ); + for (index, id) in expected.iter().enumerate() { + let page = reader.list_chunks(Some(id), 1).await.unwrap(); + assert_eq!( + page.first().and_then(|chunk| chunk.id), + expected.get(index + 1).copied() + ); + let page = reader + .scan_reservation_groups_after(1, Some((id, id))) + .await + .unwrap(); + assert_eq!( + page.first().and_then(|group| group.chunk_id), + expected.get(index + 1).copied() + ); + } + let after_domain = ChunkId { + high: 8 << 56, + low: 0, + }; + let _ = test.kv.drain_window(); + assert!(reader + .list_chunks(Some(&after_domain), 1) + .await + .unwrap() + .is_empty()); + assert!(test.kv.drain_window().scan.is_empty()); +} + +#[tokio::test] +async fn user_data_scans_exclude_system_pages_before_transfer() { + Box::pin(assert_domain_scan(ChunkDomain::UserData, 1, &[4, 5, 6, 7])).await; +} + +#[tokio::test] +async fn system_scans_exclude_user_data_pages_before_transfer() { + Box::pin(assert_domain_scan(ChunkDomain::System, 5, &[1, 2, 3])).await; +} diff --git a/app/crowdb-chunkdb/tests/readiness_probe_test.rs b/app/crowdb-chunkdb/tests/readiness_probe_test.rs index eb9430b19..cc34224e1 100644 --- a/app/crowdb-chunkdb/tests/readiness_probe_test.rs +++ b/app/crowdb-chunkdb/tests/readiness_probe_test.rs @@ -8,17 +8,21 @@ use crowdb_chunkdb::{ routing::{BindingCache, BindingTable}, storage::ChunkStore, }; -use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbClientError, ChunkdbRpcTransport}; +use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; use crowdb_kv::cluster::kv_server::KvServer; use crowdb_kv_client::{ChunkSlotMapClient, ClientConfig, CrowdbKvClient, ServiceRegistryClient}; use crowdb_protocol::{ chunk_domain::ChunkDomain, - chunk_id::{ChunkIdParts, CHUNK_TYPE_S3}, + chunk_id::CHUNK_TYPE_S3, chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}, chunkdb::rpc::{Chunk, ChunkState}, common::ChunkId, }; +#[path = "common/readiness_failures.rs"] +mod readiness_failures; +#[path = "common/readiness_registry.rs"] +mod readiness_registry; #[path = "common/runtime_context.rs"] #[allow(dead_code)] mod runtime_context; @@ -46,7 +50,8 @@ async fn readiness_reads_both_domains_without_scanning_or_mutating_the_catalog() let guard = Arc::new(RangeGuard::new()); guard.install(&layout.service_map().unwrap(), 1).unwrap(); let store = ChunkStore::new(Arc::clone(&test.kv), bindings.clone()); - let (context, stop) = runtime_context::runtime_context(Arc::clone(&test.kv), bindings, guard); + let (context, stop) = + runtime_context::runtime_context(Arc::clone(&test.kv), bindings, Arc::clone(&guard)); let system = context.clone().start(ChunkDomain::System).await.unwrap(); let user = context.start(ChunkDomain::UserData).await.unwrap(); let service = Arc::new(user.rpc_service.with_system_runtime(Arc::new(system.rpc_service))); @@ -69,6 +74,7 @@ async fn readiness_reads_both_domains_without_scanning_or_mutating_the_catalog() .await .unwrap(); let client = ChunkdbClient::new(registry, Arc::new(ChunkdbRpcTransport::new())); + Box::pin(client.observe_readiness()).await.unwrap(); let reference = crowdb_kv_client::RangeBindingClient::from_shared(Arc::clone(&control)); let _ = control.drain_window(); reference.refresh().await.unwrap(); @@ -89,25 +95,16 @@ async fn readiness_reads_both_domains_without_scanning_or_mutating_the_catalog() assert_eq!(work.scan.len(), 0, "readiness must not enumerate catalog records"); assert_eq!(work.put.len() + work.batch_write.len() + work.delete.len(), 0); } - // Storage corruption must remain an unhealthy result, rather than being - // confused with an authoritative absent key in an empty catalog. - let mut key = b"/chunk/".to_vec(); - key.extend_from_slice( - &ChunkIdParts { - high: u64::from(CHUNK_TYPE_S3) << 56, - low: 0, - } - .to_bytes(), - ); - test.kv - .put(0, 1, &key, b"corrupt chunk record", None) - .await - .unwrap(); - assert!(matches!( - Box::pin(client.observe_readiness()).await, - Err(ChunkdbClientError::Internal(_)) - )); - server.stop(); + readiness_registry::assert_warm_registry_independence( + &client, + &control, + &test.kv, + &server, + test.server.listen_addr().unwrap().to_string(), + ) + .await; + + readiness_failures::assert_storage_and_owner_failures(&client, &test.kv, &control, &guard, &server).await; } async fn seed_catalog(store: &ChunkStore, count: u64) { diff --git a/app/crowdb-cli/src/commands/cluster/local.rs b/app/crowdb-cli/src/commands/cluster/local.rs index b16b55417..b8e87764c 100644 --- a/app/crowdb-cli/src/commands/cluster/local.rs +++ b/app/crowdb-cli/src/commands/cluster/local.rs @@ -216,7 +216,7 @@ async fn stop_inner(cli: &Cli) -> Result<()> { .ok_or_else(|| invalid("selected cluster has no local runtime ownership receipt"))?; binding(&record)?; Docker.check(&record).await?; - for item in Docker.objects("container").await? { + for item in Docker.objects(&record, "container").await? { if owned(&item, &record, "container") && item["State"]["Running"] == true { let identity = item["Id"] .as_str() diff --git a/app/crowdb-cli/src/commands/cluster/local/docker.rs b/app/crowdb-cli/src/commands/cluster/local/docker.rs index 40964c805..b4bc25b47 100644 --- a/app/crowdb-cli/src/commands/cluster/local/docker.rs +++ b/app/crowdb-cli/src/commands/cluster/local/docker.rs @@ -70,21 +70,30 @@ impl Docker { } Ok(()) } - pub async fn objects(&self, kind: &str) -> Result> { + pub async fn objects(&self, record: &Launch, kind: &str) -> Result> { let args = if kind == "container" { - strings(&["ps", "-aq"]) + strings(&["ps", "-a", "--format", "{{.Names}}"]) } else { - strings(&[kind, "ls", "-q"]) + strings(&[kind, "ls", "--format", "{{.Name}}"]) }; - let ids = self.call(&args).await?; - if ids.is_empty() { + let listed = self.call(&args).await?; + let names: Vec<_> = listed + .lines() + .filter(|name| { + if kind == "network" { + *name == record.resource_name(kind, 0) + } else { + (1..=record.nodes).any(|index| *name == record.resource_name(kind, index)) + } + }) + .collect(); + if names.is_empty() { return Ok(Vec::new()); } + // Other fixtures may remove their resources between list and inspect. + // Inspect only this receipt's exact names, then verify their ownership. let mut args = strings(&[kind, "inspect"]); - args.extend(ids.lines().map(String::from)); - if args.len() > 4098 { - return Err(invalid("Docker resource inventory exceeds bound")); - } + args.extend(names.into_iter().map(String::from)); serde_json::from_str(&self.call(&args).await?) .map_err(|_| invalid("invalid Docker resource observation")) } @@ -99,7 +108,7 @@ impl Docker { } pub async fn ensure(&self, record: &Launch, kind: &str, name: &str) -> Result { if let Some(item) = self - .objects(kind) + .objects(record, kind) .await? .into_iter() .find(|item| object_name(item) == name) diff --git a/app/crowdb-cli/src/commands/cluster/local/launch.rs b/app/crowdb-cli/src/commands/cluster/local/launch.rs index 3258e9802..3582497c8 100644 --- a/app/crowdb-cli/src/commands/cluster/local/launch.rs +++ b/app/crowdb-cli/src/commands/cluster/local/launch.rs @@ -67,7 +67,7 @@ pub(super) async fn run(docker: &Docker, record: &Launch) -> Result<()> { if bridge["Driver"] != "bridge" { return Err(invalid("local cluster requires its owned bridge")); } - let containers = docker.objects("container").await?; + let containers = docker.objects(record, "container").await?; for index in 1..=record.nodes { let name = record.resource_name("container", index); let volume = record.resource_name("volume", index); diff --git a/app/crowdb-cli/tests/cluster_lifecycle_test.rs b/app/crowdb-cli/tests/cluster_lifecycle_test.rs index d17ef6325..c871e83ab 100644 --- a/app/crowdb-cli/tests/cluster_lifecycle_test.rs +++ b/app/crowdb-cli/tests/cluster_lifecycle_test.rs @@ -144,6 +144,39 @@ async fn serve(cluster: &str, nodes: u8, ready: bool) -> (u16, tokio::task::Join ) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn concurrent_foreign_resource_removal_does_not_break_selected_cluster_replay() { + let test = TestDocker::new(); + let (port, server) = serve("local", 1, true).await; + let mut state = test.state(); + state["disappearing"] = json!({ + "container": ["crowdb-foreign-1"], + "volume": ["crowdb-foreign-1-data"], + "network": ["crowdb-foreign"], + }); + test.save(&state); + test.run( + None, + &["start", "--port-base", &port.to_string(), "--wait", "0"], + 0, + ); + let receipt = fs::read(test.receipt("local")).unwrap(); + test.run(None, &["stop"], 0); + test.run(None, &["start", "--wait", "0"], 0); + assert_eq!(fs::read(test.receipt("local")).unwrap(), receipt); + assert_eq!(test.state()["container"].as_object().unwrap().len(), 1); + assert!(test.state()["calls"].as_array().unwrap().iter().all(|args| { + args[1] != "inspect" + || args + .as_array() + .unwrap() + .iter() + .skip(2) + .all(|name| !name.as_str().unwrap().starts_with("crowdb-foreign")) + })); + server.abort(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn default_single_node_start_seals_identity_and_resumes_without_duplicate_resources() { let test = TestDocker::new(); diff --git a/app/crowdb-cli/tests/common/local_docker.py b/app/crowdb-cli/tests/common/local_docker.py index 74681286b..20a60ea5e 100644 --- a/app/crowdb-cli/tests/common/local_docker.py +++ b/app/crowdb-cli/tests/common/local_docker.py @@ -31,9 +31,11 @@ def one(flag): result = [{"Id": "sha256:" + "a" * 64, "Os": "linux", "Architecture": "amd64", "Config": {"Labels": {"org.opencontainers.image.source": "https://github.com/buzzcrow/crowdb"}}}] elif args[:2] == ["ps", "-aq"]: - result = "\n".join(state["container"]) + result = "\n".join([*state["container"], *state.get("disappearing", {}).get("container", [])]) +elif args[:3] == ["ps", "-a", "--format"]: + result = "\n".join([*state["container"], *state.get("disappearing", {}).get("container", [])]) elif len(args) > 1 and args[1] == "ls": - result = "\n".join(state[args[0]]) + result = "\n".join([*state[args[0]], *state.get("disappearing", {}).get(args[0], [])]) elif len(args) > 1 and args[1] == "inspect": result = [state[args[0]][name] for name in args[2:]] elif len(args) > 1 and args[1] == "create": diff --git a/app/crowdb-kv-server/Cargo.toml b/app/crowdb-kv-server/Cargo.toml index f06ee6a43..06389f473 100644 --- a/app/crowdb-kv-server/Cargo.toml +++ b/app/crowdb-kv-server/Cargo.toml @@ -39,6 +39,7 @@ sha2 = "0.10" uuid = { version = "1", features = ["v4"] } [dev-dependencies] +crowdb-kv = { path = "../../lib/crowdb-kv", features = ["test-util"] } reqwest = { version = "0.12", features = ["json"] } crowdb-rpc-ffi = { path = "../../lib/crowdb-rpc/ffi", features = ["test-util"] } crowdb-test-harness = { path = "../../lib/crowdb-test-harness" } diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs index eaee82913..61a3700f3 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs @@ -8,8 +8,8 @@ use std::collections::{HashMap, HashSet}; use bytes::Bytes; use crowdb_kv::cluster::group_operations::KvGroupOperationError; use crowdb_protocol::chunk_kv::{ - ChunkKvRangeBalancePolicy, ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, - DomainMonitorDescriptor, Id128, OwnerDescriptor, SplitPhase, TransferPhase, TransferTransition, + ChunkKvRangeBalancePolicy, ChunkKvRangeCatalogEntry, DomainMonitorDescriptor, Id128, OwnerDescriptor, + SplitPhase, TransferPhase, TransferTransition, }; use crowdb_protocol::common::{ChunkKvExtra, ChunkKvPartitionLoad, InstanceValue}; use crowdb_protocol::key::{ChunkKvSplitKey, ChunkKvTransferKey, TextKey}; @@ -21,6 +21,7 @@ use super::{ catalog, operation_error, read_instances, read_splits, read_transfers, transfer_id, wall_time_ms, }; +mod materialization; mod observation; mod selection; mod split; @@ -50,31 +51,8 @@ pub async fn plan(control: &Group0ControlPlane, descriptor: &DomainMonitorDescri if state.healthy.is_empty() { return Ok(()); } - let entries: Vec<_> = catalog - .pages - .iter() - .flat_map(|page| page.entries.iter()) - .collect(); - if let Some(entry) = entries.iter().copied().find(|entry| { - entry.state == ChunkKvRangeCatalogPartitionState::Serving - && entry.artifact.tail_overlay.is_some() - && partition_load(&state, entry).is_some_and(|load| load.independently_recoverable) - }) { - let committed_transfer = read_transfers(control).await?.into_iter().any(|(transition, _)| { - transition.phase == TransferPhase::CatalogCommitted - && entry.transition_id == Some(transition.transition_id) - && entry.partition_id == transition.partition_id - && entry.range == transition.range - && entry.owner.instance_id == transition.target.instance_id - && entry.owner_epoch == transition.target_epoch - && entry.artifact == transition.target_artifact - }); - if committed_transfer { - catalog::publish_materialized_transfer(control, entry.partition_id).await?; - } else { - catalog::publish_materialized_partition(control, entry.partition_id).await?; - } - // Materialization only releases an immutable parent-stream overlay. + if materialization::reconcile(control, &catalog, &state).await? { + // Materialization releases a completed marker or immutable overlay. // It does not affect request routing, so immediately plan from the // refreshed catalog instead of inserting a control-plane idle cycle // before the next independent split. diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/materialization.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/materialization.rs new file mode 100644 index 000000000..35debb5cf --- /dev/null +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance/materialization.rs @@ -0,0 +1,63 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Release completed handoff markers after current-owner recovery. + +use crowdb_protocol::chunk_kv::{ + ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, TransferPhase, TransferTransition, +}; + +use crate::group0_control_plane::Group0ControlPlane; + +use super::{catalog, partition_load, read_transfers, PlanningState}; + +pub(super) async fn reconcile( + control: &Group0ControlPlane, + catalog: &catalog::VersionedRangeCatalog, + state: &PlanningState, +) -> Result { + let candidates: Vec<_> = catalog + .pages + .iter() + .flat_map(|page| page.entries.iter()) + .filter(|entry| independently_recovered(entry, state)) + .collect(); + if candidates.is_empty() { + return Ok(false); + } + let transfers = read_transfers(control).await?; + for entry in candidates { + if transfers + .iter() + .any(|(transition, _)| committed_target(entry, transition)) + { + catalog::publish_materialized_transfer(control, entry).await?; + return Ok(true); + } + if entry.artifact.tail_overlay.is_some() { + catalog::publish_materialized_partition(control, entry.partition_id).await?; + return Ok(true); + } + } + Ok(false) +} + +fn independently_recovered(entry: &ChunkKvRangeCatalogEntry, state: &PlanningState) -> bool { + entry.state == ChunkKvRangeCatalogPartitionState::Serving + && (entry.transition_id.is_some() || entry.artifact.tail_overlay.is_some()) + && state + .hosted_epochs + .get(&(entry.owner.instance_id, entry.partition_id)) + == Some(&entry.owner_epoch) + && partition_load(state, entry).is_some_and(|load| load.independently_recoverable) +} + +fn committed_target(entry: &ChunkKvRangeCatalogEntry, transition: &TransferTransition) -> bool { + transition.phase == TransferPhase::CatalogCommitted + && entry.transition_id == Some(transition.transition_id) + && entry.partition_id == transition.partition_id + && entry.range == transition.range + && entry.owner.instance_id == transition.target.instance_id + && entry.owner_epoch == transition.target_epoch + && entry.artifact == transition.target_artifact +} diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs index 3d51546b8..c340cc986 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs @@ -212,7 +212,7 @@ async fn validate_materialized_split( pub async fn publish_materialized_transfer( control: &Group0ControlPlane, - partition_id: crowdb_protocol::chunk_kv::Id128, + expected: &ChunkKvRangeCatalogEntry, ) -> Result { let catalog = load_current(control) .await? @@ -221,18 +221,21 @@ pub async fn publish_materialized_transfer( .pages .iter() .flat_map(|page| &page.entries) - .find(|entry| entry.partition_id == partition_id) + .find(|entry| entry.partition_id == expected.partition_id) else { return Err("materialized transfer target is absent from the catalog".into()); }; if current.artifact.tail_overlay.is_none() && current.transition_id.is_none() { return Ok(catalog.head.generation); } + if current != expected { + return Err("materialized transfer assignment changed before publication".into()); + } let transition_id = current .transition_id .ok_or_else(|| "materialized transfer target has no transition identity".to_string())?; let (head, pages) = - release_materialized_transfer(catalog.head, catalog.pages, transition_id, partition_id)?; + release_materialized_transfer(catalog.head, catalog.pages, transition_id, expected.partition_id)?; publish(control, head, pages, catalog.head_revision).await } @@ -254,7 +257,7 @@ fn release_materialized_transfer( continue; } matched += 1; - if entry.partition_id != partition_id || entry.artifact.tail_overlay.is_none() { + if entry.partition_id != partition_id { return Err("materialized transfer target identity is inconsistent".into()); } entry.artifact.tail_overlay = None; diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs index 59ff76455..aa95956f9 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs @@ -13,7 +13,7 @@ use crowdb_protocol::{ common::InstanceValue, key::{InstanceKey, ServingGrantKey, StreamProcessLeaseKey, TextKey}, }; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; struct ReadyInstance { process_epoch: u64, @@ -40,7 +40,7 @@ pub(super) async fn issue( ) }) .collect(); - let mut assignments: HashMap> = HashMap::new(); + let mut assignments: BTreeMap> = BTreeMap::new(); for entry in catalog.pages.iter().flat_map(|page| &page.entries) { let assignment = ServingAssignment { partition_id: entry.partition_id, @@ -58,8 +58,9 @@ pub(super) async fn issue( .push(assignment); } } + let mut failures = Vec::new(); for (instance_id, assignments) in assignments { - publish( + if let Err(error) = publish( control, descriptor, instance_id, @@ -67,9 +68,19 @@ pub(super) async fn issue( catalog.head.generation, assignments, ) - .await?; + .await + { + failures.push(format!("instance {instance_id}: {error}")); + } + } + if failures.is_empty() { + Ok(()) + } else { + Err(format!( + "serving grant publication failed: {}", + failures.join("; ") + )) } - Ok(()) } async fn publish( diff --git a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs index d256bc2d7..9f9503ab8 100644 --- a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs +++ b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs @@ -5,6 +5,8 @@ mod balance_distribution; #[path = "common/balance_planning.rs"] mod balance_planning; +#[path = "chunk_kv_balance_planning_test/committed_transfer.rs"] +mod committed_transfer; #[path = "chunk_kv_balance_planning_test/recovering_owner.rs"] mod recovering_owner; diff --git a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/committed_transfer.rs b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/committed_transfer.rs new file mode 100644 index 000000000..de925de43 --- /dev/null +++ b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test/committed_transfer.rs @@ -0,0 +1,143 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_protocol::chunk_kv::{ + AuthorityReleaseProof, ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, + OwnerDescriptor, TargetReadinessProof, TransferPhase, TransferReadinessLimits, +}; +use crowdb_protocol::common::InstanceValue; +use crowdb_protocol::key::{ChunkKvRangeCatalogHeadKey, ChunkKvRangeCatalogPageKey, InstanceKey}; + +#[tokio::test] +async fn completed_recovery_does_not_starve_an_idle_owner_of_placement() { + let fixture = TestBalancePlanning::new().await; + seed_committed_recoveries(&fixture.control).await; + fixture.tick().await; + let rows = fixture + .control + .scan_all_prefix(Bytes::from(ChunkKvTransferKey::text_prefix_all()), 16) + .await + .unwrap(); + let planned: Vec = rows + .iter() + .map(|row| serde_json::from_slice(&row.value).unwrap()) + .filter(|transfer: &TransferTransition| transfer.phase == TransferPhase::Planned) + .collect(); + assert_eq!( + planned.len(), + 1, + "completed recovery must permit another placement" + ); + assert_eq!( + (planned[0].source.instance_id, planned[0].target.instance_id), + (1, 2) + ); + assert_eq!(planned[0].partition_id, Id128 { high: 41, low: 1 }); +} + +async fn seed_committed_recoveries(control: &Group0ControlPlane) { + let mut head: ChunkKvRangeCatalogHead = read(control, ChunkKvRangeCatalogHeadKey.to_path()).await; + let mut page: ChunkKvRangeCatalogPage = read( + control, + ChunkKvRangeCatalogPageKey { + generation: head.generation, + page_index: 0, + } + .to_path(), + ) + .await; + let mut records = Vec::new(); + for entry in &mut page.entries { + let transfer = committed_recovery(entry); + transfer.validate().unwrap(); + entry.owner_epoch = transfer.target_epoch; + entry.transition_id = Some(transfer.transition_id); + records.push(record( + ChunkKvTransferKey { + transition_id: transfer.transition_id, + } + .to_path(), + &transfer, + )); + } + page.generation += 1; + page.seal().unwrap(); + head.previous_generation = Some(head.generation); + head.generation = page.generation; + head.pages[0].page_generation = page.generation; + head.pages[0].page_checksum = page.checksum; + head.seal().unwrap(); + head.validate_pages(std::slice::from_ref(&page)).unwrap(); + records.push(record( + ChunkKvRangeCatalogPageKey { + generation: page.generation, + page_index: 0, + } + .to_path(), + &page, + )); + records.push(record(ChunkKvRangeCatalogHeadKey.to_path(), &head)); + let path = InstanceKey { + service: "chunk-kv".into(), + instance_id: 1, + } + .to_path(); + let mut instance: InstanceValue = read(control, path.clone()).await; + for hosted in &mut instance.extra.as_mut().unwrap().chunk_kv.as_mut().unwrap().hosted { + hosted.owner_epoch = 2; + } + records.push(record(path, &instance)); + control.put_batch(records).await.unwrap(); +} + +async fn read(control: &Group0ControlPlane, path: String) -> T { + let row = control.get(path.as_bytes()).await.unwrap(); + serde_json::from_slice(row.value.as_deref().unwrap()).unwrap() +} + +fn record(path: String, value: &T) -> (Bytes, Bytes) { + (Bytes::from(path), Bytes::from(serde_json::to_vec(value).unwrap())) +} + +fn committed_recovery(entry: &ChunkKvRangeCatalogEntry) -> TransferTransition { + let proof = TargetReadinessProof { + target_instance_id: entry.owner.instance_id, + target_epoch: entry.owner_epoch + 1, + artifact: entry.artifact.clone(), + durable_tail: 0, + }; + TransferTransition { + transition_id: Id128 { + high: 99, + low: entry.partition_id.low, + }, + partition_id: entry.partition_id, + range: entry.range.clone(), + source: OwnerDescriptor { + instance_id: 3, + rpc_endpoint: "127.0.0.1:17003".into(), + }, + source_epoch: entry.owner_epoch, + target: entry.owner.clone(), + target_epoch: proof.target_epoch, + artifact: entry.artifact.clone(), + target_artifact: entry.artifact.clone(), + readiness_limits: TransferReadinessLimits { + max_tail_records: 100, + max_tail_bytes: 1_000_000, + max_estimated_catchup_ms: 1_000, + prepare_deadline_ms: 1_000, + forwarding_grace_ms: 1_000, + }, + planned_at_ms: 0, + old_grant_expires_at_ms: 1, + phase: TransferPhase::CatalogCommitted, + release_proof: Some(AuthorityReleaseProof::LeaseExpired { + activation_not_before_ms: 2, + }), + readiness_proof: Some(proof.clone()), + catchup_proof: Some(proof), + failure: None, + } +} diff --git a/app/crowdb-kv-server/tests/common/grant_isolation.rs b/app/crowdb-kv-server/tests/common/grant_isolation.rs new file mode 100644 index 000000000..d8e2c251a --- /dev/null +++ b/app/crowdb-kv-server/tests/common/grant_isolation.rs @@ -0,0 +1,228 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; +use crowdb_protocol::{ + chunk_stream::{StreamProcessLease, StreamPurpose}, + key::StreamProcessLeaseKey, +}; + +struct TestGrantInstances { + store: Arc, + control: Group0ControlPlane, + policy: DomainMonitorDescriptor, +} + +fn lease_path(instance_id: u64) -> String { + StreamProcessLeaseKey { + purpose: StreamPurpose::Wal, + instance_id, + } + .to_path() +} + +impl TestGrantInstances { + async fn start() -> Self { + let (_, store) = registry_with_group_zero(PxLocalReplicaRole::Leader); + let control = Group0ControlPlane::acquire(&store).await.unwrap(); + let mut entries = Vec::new(); + for instance_id in 1..=2 { + let partition_id = Id128 { + high: 91, + low: instance_id, + }; + let endpoint = format!("127.0.0.1:{}", 17000 + instance_id); + entries.push(ChunkKvRangeCatalogEntry { + partition_id, + range: KeyRange { + start: if instance_id == 1 { vec![] } else { b"m".to_vec() }, + end: (instance_id == 1).then(|| b"m".to_vec()), + }, + owner: OwnerDescriptor { + instance_id, + rpc_endpoint: endpoint.clone(), + }, + owner_epoch: 1, + state: ChunkKvRangeCatalogPartitionState::Serving, + artifact: PartitionArtifact { + tree_id: 910 + instance_id, + stream_name: StreamName { + high: 92, + low: instance_id, + }, + tail_overlay: None, + }, + transition_id: None, + }); + test_process::TestProcessLease::seed(&control, instance_id).await; + let mut instance = test_process::TestProcessLease::idle_instance(instance_id, &endpoint); + instance.extra.as_mut().unwrap().chunk_kv.as_mut().unwrap().hosted = vec![HostedPartition { + partition_id, + owner_epoch: 1, + recovering: false, + }]; + put_json( + &control, + InstanceKey { + service: "chunk-kv".into(), + instance_id, + } + .to_path(), + &instance, + ) + .await; + } + Self::seed_catalog(&control, entries).await; + let mut policy = descriptor(); + policy.domain = "chunk-kv".into(); + policy.service_registry_name = "chunk-kv".into(); + policy.failure_policy = DomainFailurePolicy::OperatorOnly; + Self { + store, + control, + policy, + } + } + + async fn seed_catalog(control: &Group0ControlPlane, entries: Vec) { + let mut page = ChunkKvRangeCatalogPage { + generation: 1, + page_index: 0, + entries, + checksum: [0; 32], + }; + page.seal().unwrap(); + let mut head = ChunkKvRangeCatalogHead { + generation: 1, + previous_generation: None, + pages: vec![ChunkKvRangeCatalogPageRef { + page_generation: 1, + page_index: 0, + first_key: vec![], + page_checksum: page.checksum, + }], + checksum: [0; 32], + }; + head.seal().unwrap(); + put_json( + control, + ChunkKvRangeCatalogPageKey { + generation: 1, + page_index: 0, + } + .to_path(), + &page, + ) + .await; + put_json(control, ChunkKvRangeCatalogHeadKey.to_path(), &head).await; + } + + async fn assert_second_grant(&self) { + let path = ServingGrantKey { instance_id: 2 }.to_path(); + let observed = self.control.get(path.as_bytes()).await.unwrap(); + assert!( + observed.value.is_some(), + "one instance's failed grant publication must not skip an independent ready instance" + ); + let grant: ServingGrant = serde_json::from_slice(&observed.value.unwrap()).unwrap(); + let lease_record = self.control.get(lease_path(2).as_bytes()).await.unwrap(); + let lease: StreamProcessLease = serde_json::from_slice(&lease_record.value.unwrap()).unwrap(); + assert_eq!(grant.instance_id, 2); + assert_eq!(grant.process_epoch, 1); + assert_eq!(grant.catalog_generation, 1); + assert_eq!(grant.assignments.len(), 1); + assert_eq!(grant.assignments[0].partition_id, Id128 { high: 91, low: 2 }); + assert_eq!(grant.assignments[0].owner_epoch, 1); + assert_eq!(grant.lease_sequence, lease.grant_sequence); + assert!(lease.grant_expires_at_ms >= grant.expires_at_ms); + assert_eq!(observed.revision, lease_record.revision); + assert!(self + .control + .get(ServingGrantKey { instance_id: 1 }.to_path().as_bytes()) + .await + .unwrap() + .value + .is_none()); + } +} + +async fn until(condition: impl Fn() -> bool) { + tokio::time::timeout(Duration::from_secs(3), async { + while !condition() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn busy_instance_cas_does_not_skip_another_instances_grant() { + let test = TestGrantInstances::start().await; + let group = test.store.get_group(0).unwrap(); + let first_path = lease_path(1); + let current = test.control.get(first_path.as_bytes()).await.unwrap(); + let original_lease = current.value.clone(); + let permits = group.try_acquire_all_inflight_permits(); + assert_eq!(permits.len(), group.inflight_window_size()); + let contender = tokio::spawn({ + let control = test.control.clone(); + let path = first_path.clone(); + async move { + control + .compare_and_put(Bytes::from(path), current.value.unwrap(), current.revision) + .await + } + }); + until(|| group.conditional_write_pending_for_tests(first_path.as_bytes())).await; + let renewal = tokio::spawn({ + let control = test.control.clone(); + let policy = test.policy.clone(); + async move { ChunkKvRangeMonitorDriver::new().tick(&control, &policy).await } + }); + until(|| renewal.is_finished() || group.conditional_write_pending_for_tests(lease_path(2).as_bytes())) + .await; + drop(permits); + let result = tokio::time::timeout(Duration::from_secs(3), renewal) + .await + .unwrap() + .unwrap(); + tokio::time::timeout(Duration::from_secs(3), contender) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(result.unwrap_err().contains("admission is busy")); + assert_eq!( + test.control.get(first_path.as_bytes()).await.unwrap().value, + original_lease + ); + test.assert_second_grant().await; +} + +#[tokio::test] +async fn invalid_instance_grant_does_not_skip_another_instances_grant() { + let test = TestGrantInstances::start().await; + let path = lease_path(1); + let current = test.control.get(path.as_bytes()).await.unwrap(); + let mut lease: StreamProcessLease = serde_json::from_slice(¤t.value.unwrap()).unwrap(); + lease.grant_sequence = u64::MAX; + lease.grant_expires_at_ms = 1; + lease.validate().unwrap(); + test.control + .compare_and_put( + Bytes::from(path), + Bytes::from(serde_json::to_vec(&lease).unwrap()), + current.revision, + ) + .await + .unwrap(); + let result = ChunkKvRangeMonitorDriver::new() + .tick(&test.control, &test.policy) + .await; + assert!(result.unwrap_err().contains("grant sequence overflowed")); + let retained: StreamProcessLease = get_json(&test.control, &lease_path(1)).await; + assert_eq!(retained, lease); + test.assert_second_grant().await; +} diff --git a/app/crowdb-kv-server/tests/common/split_materialization.rs b/app/crowdb-kv-server/tests/common/split_materialization.rs index e02e1a7f2..4fdb544ec 100644 --- a/app/crowdb-kv-server/tests/common/split_materialization.rs +++ b/app/crowdb-kv-server/tests/common/split_materialization.rs @@ -4,6 +4,9 @@ use super::catalog_publication::{replace_json, tick}; use super::*; +#[path = "split_materialization/transfer.rs"] +mod transfer; + struct TestMaterialization { control: Group0ControlPlane, split: SplitTransition, diff --git a/app/crowdb-kv-server/tests/common/split_materialization/transfer.rs b/app/crowdb-kv-server/tests/common/split_materialization/transfer.rs new file mode 100644 index 000000000..c291da7e0 --- /dev/null +++ b/app/crowdb-kv-server/tests/common/split_materialization/transfer.rs @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test] +async fn committed_transfer_without_overlay_releases_its_catalog_marker() { + let fixture = TestMaterialization::transferred_parent().await; + let parent = &fixture.page.entries[0]; + assert!(parent.artifact.tail_overlay.is_none()); + assert!(parent.transition_id.is_some()); + observe_partition(&fixture.control, parent, true).await; + tick(&fixture.control).await.unwrap(); + + let (head, page) = current_catalog(&fixture.control).await; + let mut expected = parent.clone(); + expected.transition_id = None; + assert_eq!(page.entries[0], expected); + assert_eq!(page.entries[1], fixture.page.entries[1]); + assert_eq!(head.generation, fixture.head.generation + 1); + + let transfer: TransferTransition = get_json( + &fixture.control, + &ChunkKvTransferKey { + transition_id: parent.transition_id.unwrap(), + } + .to_path(), + ) + .await; + assert_eq!(transfer.phase, TransferPhase::CatalogCommitted); + tick(&fixture.control).await.unwrap(); + assert_eq!(current_catalog(&fixture.control).await.0, head); +} + +#[tokio::test] +async fn transfer_marker_waits_for_independent_recovery() { + let fixture = TestMaterialization::transferred_parent().await; + tick(&fixture.control).await.unwrap(); + let (head, page) = current_catalog(&fixture.control).await; + assert_eq!(head, fixture.head); + assert_eq!(page, fixture.page); +} + +#[tokio::test] +async fn obsolete_owner_observation_cannot_release_a_transfer_marker() { + let fixture = TestMaterialization::transferred_parent().await; + let mut obsolete = fixture.page.entries[0].clone(); + obsolete.owner_epoch += 1; + observe_partition(&fixture.control, &obsolete, true).await; + tick(&fixture.control).await.unwrap(); + let (head, page) = current_catalog(&fixture.control).await; + assert_eq!(head, fixture.head); + assert_eq!(page, fixture.page); +} + +#[tokio::test] +async fn transfer_marker_release_requires_the_exact_committed_assignment() { + for mismatch in 0..4 { + let mut fixture = TestMaterialization::transferred_parent().await; + let parent = &mut fixture.page.entries[0]; + match mismatch { + 0 => parent.owner_epoch += 1, + 1 => parent.artifact.tree_id += 1, + 2 => parent.artifact.stream_name.low += 1, + 3 => parent.transition_id = Some(Id128 { high: 999, low: 1 }), + _ => unreachable!(), + } + fixture.publish_page().await; + observe_partition(&fixture.control, &fixture.page.entries[0], true).await; + tick(&fixture.control).await.unwrap(); + let (head, page) = current_catalog(&fixture.control).await; + assert_eq!(head, fixture.head, "mismatch {mismatch}"); + assert_eq!(page, fixture.page, "mismatch {mismatch}"); + } +} diff --git a/app/crowdb-kv-server/tests/domain_monitor_test.rs b/app/crowdb-kv-server/tests/domain_monitor_test.rs index 8553aae15..1b1307719 100644 --- a/app/crowdb-kv-server/tests/domain_monitor_test.rs +++ b/app/crowdb-kv-server/tests/domain_monitor_test.rs @@ -29,6 +29,8 @@ use crowdb_protocol::key::{ DomainMonitorKey, InstanceKey, ServingGrantKey, TextKey, }; +#[path = "common/grant_isolation.rs"] +mod grant_isolation; #[path = "common/grant_process.rs"] mod grant_process; #[path = "common/process_lease.rs"] diff --git a/app/crowdb-web/src/lib.rs b/app/crowdb-web/src/lib.rs index 85d5676f8..510e09246 100644 --- a/app/crowdb-web/src/lib.rs +++ b/app/crowdb-web/src/lib.rs @@ -41,9 +41,14 @@ pub use services::observation::{ }; pub use services::observation::{start_service_observer, ServiceObserver}; #[cfg(feature = "test-util")] -pub use services::observation::{start_service_observer_for_tests, ServiceObservationSourceForTests}; +pub use services::observation::{ + start_service_observer_for_tests, ProcessObservationForTests, ServiceObservationSourceForTests, +}; pub use state::AppState; +#[cfg(feature = "test-util")] +pub use node::authority::projection::project_confirmed_servers_for_tests; + #[cfg(feature = "test-util")] pub use node::observation::{node_observation_time_for_tests, start_node_observer_for_tests}; diff --git a/app/crowdb-web/src/node.rs b/app/crowdb-web/src/node.rs index 8772d5669..f491fd2cb 100644 --- a/app/crowdb-web/src/node.rs +++ b/app/crowdb-web/src/node.rs @@ -9,7 +9,7 @@ pub(crate) mod preflight; pub(crate) mod setup; mod update; pub(crate) use update::update; -mod authority; +pub(crate) mod authority; pub(crate) mod cluster_status; mod hardware; pub(crate) mod initialization; diff --git a/app/crowdb-web/src/node/authority.rs b/app/crowdb-web/src/node/authority.rs index c4b077e2c..c562d3a86 100644 --- a/app/crowdb-web/src/node/authority.rs +++ b/app/crowdb-web/src/node/authority.rs @@ -16,6 +16,8 @@ use crate::{ error::{err_409, err_502, map_config_err, ErrorBody}, state::AppState, }; +pub(crate) mod projection; + type Failure = (StatusCode, Json); pub(crate) async fn status(State(state): State) -> Json { @@ -291,46 +293,11 @@ async fn confirmed_inner( return Err(err_409("Topology publication is incomplete")); }; let mut config = state.config.write().map_err(|error| err_502(error.to_string()))?; - if config.servers.is_empty() { - config.servers = operation.intent.to_config().servers; - } - for node in registry - .nodes - .iter() - .filter(|node| node.confirmed && !node.cancelled) - { - let host = if node.host.contains(':') { - format!("[{}]", node.host) - } else { - node.host.clone() - }; - if let Some(server) = config - .servers - .iter_mut() - .find(|server| server.node_id == Some(node.node_id)) - { - let management_port = reqwest::Url::parse(&server.url) - .ok() - .and_then(|url| url.port_or_known_default()) - .unwrap_or(10000); - let rpc_port = server - .rpc_url - .as_deref() - .and_then(|url| reqwest::Url::parse(&format!("http://{url}")).ok()) - .and_then(|url| url.port_or_known_default()) - .unwrap_or(10100); - server.url = format!("http://{host}:{management_port}"); - server.rpc_url = Some(format!("{host}:{rpc_port}")); - } else { - let mut server = crowdb_console_shared::config::ServerEntry::new( - format!("kv-{}", node.node_id), - format!("http://{host}:10000"), - ); - server.node_id = Some(node.node_id); - server.rpc_url = Some(format!("{host}:10100")); - config.servers.push(server); - } - } + projection::project( + &mut config.servers, + operation.intent.to_config().servers, + ®istry.nodes, + ); super::admission::save(&state.runtime_root.join("confirmed-nodes.json"), ®istry.nodes)?; Ok(()) } diff --git a/app/crowdb-web/src/node/authority/projection.rs b/app/crowdb-web/src/node/authority/projection.rs new file mode 100644 index 000000000..63d576306 --- /dev/null +++ b/app/crowdb-web/src/node/authority/projection.rs @@ -0,0 +1,61 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Project confirmed node addresses without replacing registered service identities. + +use crowdb_console_shared::{ + config::{ServerEntry, ServiceType}, + deployment::registry::NodeRecord, +}; + +pub(crate) fn project(servers: &mut Vec, bootstrap: Vec, nodes: &[NodeRecord]) { + for server in bootstrap { + if server.service_type == ServiceType::PaxosKv + && !servers.iter().any(|existing| { + existing.service_type == ServiceType::PaxosKv && existing.node_id == server.node_id + }) + { + servers.push(server); + } + } + for node in nodes.iter().filter(|node| node.confirmed && !node.cancelled) { + let host = if node.host.contains(':') { + format!("[{}]", node.host) + } else { + node.host.clone() + }; + if let Some(server) = servers.iter_mut().find(|server| { + server.node_id == Some(node.node_id) && server.service_type == ServiceType::PaxosKv + }) { + let management_port = reqwest::Url::parse(&server.url) + .ok() + .and_then(|url| url.port_or_known_default()) + .unwrap_or(10000); + let rpc_port = server + .rpc_url + .as_deref() + .and_then(|url| reqwest::Url::parse(&format!("http://{url}")).ok()) + .and_then(|url| url.port_or_known_default()) + .unwrap_or(10100); + server.url = format!("http://{host}:{management_port}"); + server.rpc_url = Some(format!("{host}:{rpc_port}")); + } else { + let mut server = crowdb_console_shared::config::ServerEntry::new( + format!("kv-{}", node.node_id), + format!("http://{host}:10000"), + ); + server.node_id = Some(node.node_id); + server.rpc_url = Some(format!("{host}:10100")); + servers.push(server); + } + } +} + +#[cfg(feature = "test-util")] +pub fn project_confirmed_servers_for_tests( + servers: &mut Vec, + bootstrap: Vec, + nodes: &[NodeRecord], +) { + project(servers, bootstrap, nodes); +} diff --git a/app/crowdb-web/src/services/observation.rs b/app/crowdb-web/src/services/observation.rs index 3b4eb70b7..f3fe88a34 100644 --- a/app/crowdb-web/src/services/observation.rs +++ b/app/crowdb-web/src/services/observation.rs @@ -13,6 +13,8 @@ mod worker; pub use access::remote_access_health_for_tests; pub(crate) use cache::ServiceObservations; #[cfg(feature = "test-util")] +pub use source::ProcessObservation as ProcessObservationForTests; +#[cfg(feature = "test-util")] pub use test_support::{invalidate_service_observation_for_tests, ServiceObservationMutationForTests}; pub use worker::{start_service_observer, ServiceObserver}; #[cfg(feature = "test-util")] @@ -46,6 +48,9 @@ pub struct ServerSummary { /// Live pid if the console currently tracks the process. #[serde(skip_serializing_if = "Option::is_none")] pub pid: Option, + /// Fresh accepted monitor evidence; absent for the legacy local inventory. + #[serde(skip_serializing_if = "Option::is_none")] + pub process_observed: Option, /// Accepted start awaits an owned PID observation and has not exhausted its launch budget. pub start_pending: bool, /// Node process namespace for a remote PID; never a Web-host PID. @@ -83,8 +88,10 @@ pub async fn http_list_servers(State(state): State) -> Json { @@ -99,7 +106,12 @@ pub async fn http_list_servers(State(state): State) -> Json) -> Json, pub(super) exhausted: bool, + pub(super) observed: bool, } #[derive(Clone)] @@ -100,9 +101,9 @@ impl Inventory { binding: Option<&NodeBinding>, node: Option<&NodeEntry>, entry: &ServerEntry, - ) -> (Option, NodeHealth, bool) { + ) -> (Option, NodeHealth, bool, bool) { let Some(node) = node.filter(|_| self.binding.as_ref() == binding && binding.is_some()) else { - return (None, NodeHealth::Unknown, false); + return (None, NodeHealth::Unknown, false, false); }; let key = (node.id, entry.id.clone()); let valid = |sample: &Sample| self.accepts(sample) && sample.node == *node && sample.entry == *entry; @@ -130,7 +131,7 @@ impl Inventory { ) }) && !process.is_some_and(|fact| fact.exhausted); - (pid, health, pending) + (pid, health, pending, process.is_some_and(|fact| fact.observed)) } } @@ -185,7 +186,14 @@ impl ServiceObservations { accepted } - pub(super) fn process(&self, sample: &Sample, started: Instant, pid: Option, exhausted: bool) { + pub(super) fn process( + &self, + sample: &Sample, + started: Instant, + pid: Option, + exhausted: bool, + observed: bool, + ) { self.latest.rcu(|current| { if !current.accepts(sample) { return Arc::clone(current); @@ -198,6 +206,7 @@ impl ServiceObservations { started, pid, exhausted, + observed, }, ); Arc::new(next) diff --git a/app/crowdb-web/src/services/observation/mutation.rs b/app/crowdb-web/src/services/observation/mutation.rs index b94a59b0b..e192078bb 100644 --- a/app/crowdb-web/src/services/observation/mutation.rs +++ b/app/crowdb-web/src/services/observation/mutation.rs @@ -139,6 +139,7 @@ impl Mutation { started: self.started, pid, exhausted: false, + observed: pid.is_some(), }, ); } diff --git a/app/crowdb-web/src/services/observation/sampling.rs b/app/crowdb-web/src/services/observation/sampling.rs index 3a42f3fb0..161c750e6 100644 --- a/app/crowdb-web/src/services/observation/sampling.rs +++ b/app/crowdb-web/src/services/observation/sampling.rs @@ -5,7 +5,7 @@ use super::{ cache::{Sample, MAX_AGE}, - source::ServiceObservationSource, + source::{ProcessObservation, ServiceObservationSource}, worker::phase_error, }; use crate::state::AppState; @@ -23,6 +23,13 @@ pub(super) async fn sample_node( binding: NodeBinding, node: NodeEntry, ) { + let mut transport = match source.process_observer(&state, &binding, &node) { + Ok(transport) => transport, + Err(error) => { + phase_error(&state, node.id, "processes", Instant::now(), &error); + return; + } + }; let mut interval = tokio::time::interval(Duration::from_secs(1)); interval.set_missed_tick_behavior(MissedTickBehavior::Skip); loop { @@ -40,19 +47,26 @@ pub(super) async fn sample_node( .filter(|entry| entry.node_id == Some(node.id)) .map(|entry| observed.sample(&binding, &node, entry)) .collect(); - sample_processes(&state, source.as_ref(), &binding, &node, &samples).await; + sample_processes(&state, source.as_ref(), &mut transport, &binding, &node, &samples).await; } } async fn sample_processes( state: &AppState, source: &dyn ServiceObservationSource, + observer: &mut Option>, binding: &NodeBinding, node: &NodeEntry, samples: &[Sample], ) { let started = Instant::now(); - let result = match tokio::time::timeout(MAX_AGE, source.processes(state, binding, node)).await { + let read = async { + match observer { + Some(observer) => observer.read().await, + None => source.processes(state, binding, node).await, + } + }; + let result = match tokio::time::timeout(MAX_AGE, read).await { Ok(Ok(value)) => Some(value), Ok(Err(error)) => { phase_error(state, node.id, "processes", started, &error); @@ -113,9 +127,19 @@ async fn sample_processes( value["service_failures"][&intent.service_id]["operation_id"] == intent.operation_id && value["service_failures"][&intent.service_id]["exhausted"] == true }); + let has_process_evidence = value + .as_ref() + .and_then(|value| value["processes"].as_object()) + .is_some_and(|processes| match processes.get(id) { + None => sample.intent.is_some() || sample.entry.service_type == ServiceType::PaxosKv, + Some(value) => value + .as_u64() + .and_then(|pid| u32::try_from(pid).ok()) + .is_some_and(|pid| pid > 0 && owned.is_some()), + }); state .service_observations - .process(sample, started, pid, exhausted); + .process(sample, started, pid, exhausted, has_process_evidence); } } diff --git a/app/crowdb-web/src/services/observation/source.rs b/app/crowdb-web/src/services/observation/source.rs index 9d3f0c549..e28c6c6ff 100644 --- a/app/crowdb-web/src/services/observation/source.rs +++ b/app/crowdb-web/src/services/observation/source.rs @@ -12,9 +12,35 @@ use crowdb_console_shared::{ use crowdb_protocol::mgmt::node::{NodeBinding, NodeControl, NodeServiceIntent}; use serde_json::Value; +/// One node sampler owns this reader throughout its bound configuration. +#[async_trait::async_trait] +pub trait ProcessObservation: Send { + /// Read one fresh monitor response; never return a retained observation. + async fn read(&mut self) -> Result; +} + +#[async_trait::async_trait] +impl ProcessObservation for crowdb_console_shared::deployment::admission::RemoteProcessObservation { + async fn read(&mut self) -> Result { + Self::read(self).await.map_err(|error| error.to_string()) + } +} + /// Observation transports; injected only by focused regression fixtures. #[async_trait::async_trait] pub trait ServiceObservationSource: Send + Sync + 'static { + /// Create an owned process transport when the source supports connection reuse. + /// + /// # Errors + /// Rejects an invalid bound transport configuration. + fn process_observer( + &self, + _state: &AppState, + _binding: &NodeBinding, + _node: &NodeEntry, + ) -> Result>, String> { + Ok(None) + } /// Read the committed service view without accepting mutations. async fn intents(&self, state: &AppState) -> Result, String>; /// Read authenticated, bound monitor-owned children. @@ -38,6 +64,25 @@ pub trait ServiceObservationSource: Send + Sync + 'static { pub(super) struct Production; #[async_trait::async_trait] impl ServiceObservationSource for Production { + fn process_observer( + &self, + state: &AppState, + binding: &NodeBinding, + node: &NodeEntry, + ) -> Result>, String> { + if node.id == binding.node_id { + return Ok(None); + } + let mut target = node.clone(); + target.ssh_password = None; + target.ssh_key = Some(crate::node::node_key_path(state).to_string_lossy().into_owned()); + let observer = crowdb_console_shared::deployment::admission::RemoteProcessObservation::new( + target, + binding.clone(), + ) + .map_err(|error| error.to_string())?; + Ok(Some(Box::new(observer))) + } async fn intents(&self, state: &AppState) -> Result, String> { crowdb_console_shared::deployment::services::list(state.kv_client().await.as_ref()) .await diff --git a/app/crowdb-web/tests/common/process_observed.rs b/app/crowdb-web/tests/common/process_observed.rs new file mode 100644 index 000000000..32e2974f7 --- /dev/null +++ b/app/crowdb-web/tests/common/process_observed.rs @@ -0,0 +1,175 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::fixture::{TestServiceInventory, TestSource}; +use crowdb_console_shared::{ + cluster::NodeHealth, + config::{NodeEntry, ServerEntry, ServiceType}, + ConsoleConfig, +}; +use crowdb_protocol::mgmt::node::{NodeBinding, NodeServiceIntent}; +use crowdb_web::{AppState, ServiceObservationSourceForTests}; +use serde_json::{json, Value}; +use std::{ + sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }, + time::Duration, +}; + +pub const RUNNING: u32 = 0; +pub const ABSENT: u32 = 1; +pub const READ_ERROR: u32 = 2; +pub const FOREIGN_BOOTSTRAP: u32 = 3; +pub const FOREIGN_OPERATION: u32 = 4; +pub const EXHAUSTED: u32 = 5; +pub const FOREIGN_EXHAUSTION: u32 = 6; +pub const MISSING_MAP: u32 = 7; +pub const NULL_MAP: u32 = 8; +pub const ARRAY_MAP: u32 = 9; +pub const ZERO_PID: u32 = 10; +pub const STRING_PID: u32 = 11; +pub const FOREIGN_NODE: u32 = 12; +pub const FOREIGN_CLUSTER: u32 = 13; + +struct TestProcessStateSource { + base: Arc, + mode: Arc, + bootstrap_pkv: bool, +} + +#[async_trait::async_trait] +impl ServiceObservationSourceForTests for TestProcessStateSource { + async fn intents(&self, _: &AppState) -> Result, String> { + Ok(self + .base + .intents + .load_full() + .iter() + .filter(|intent| !(self.bootstrap_pkv && intent.node_id == 1)) + .cloned() + .collect()) + } + + async fn processes( + &self, + _: &AppState, + binding: &NodeBinding, + node: &NodeEntry, + ) -> Result { + let mode = self.mode.load(Ordering::Relaxed); + if mode == READ_ERROR { + return Err("controlled authenticated read unavailable".into()); + } + let intent = self + .base + .intents + .load_full() + .iter() + .find(|i| i.node_id == node.id) + .unwrap() + .clone(); + let key = if self.bootstrap_pkv && node.id == 1 { + "kv".to_owned() + } else { + intent.service_id.clone() + }; + let mut bootstrap = binding.bootstrap.clone(); + if mode == FOREIGN_BOOTSTRAP { + bootstrap.operation_id = "foreign-bootstrap".into(); + } + let absent = [ABSENT, EXHAUSTED, FOREIGN_EXHAUSTION].contains(&mode); + let operation = if [FOREIGN_OPERATION, FOREIGN_EXHAUSTION].contains(&mode) { + "foreign-operation" + } else { + &intent.operation_id + }; + let mut value = json!({ + "cluster_id": binding.bootstrap.cluster_id, "node_id": node.id, "bootstrap": bootstrap, + "processes": if absent { json!({}) } else { json!({key.clone(): 100 + node.id}) }, + // Production monitor exposes execution ownership only for live children. + "process_operations": if absent || (self.bootstrap_pkv && node.id == 1) { json!({}) } else { json!({key.clone(): operation}) }, + "service_failures": if [EXHAUSTED, FOREIGN_EXHAUSTION].contains(&mode) { + json!({intent.service_id: {"operation_id": operation, "exhausted": true}}) + } else { json!({}) }, + }); + match mode { + MISSING_MAP => { + value.as_object_mut().unwrap().remove("processes"); + } + NULL_MAP => value["processes"] = Value::Null, + ARRAY_MAP => value["processes"] = json!([]), + ZERO_PID => value["processes"][&key] = json!(0), + STRING_PID => value["processes"][&key] = json!("not-a-pid"), + FOREIGN_NODE => value["node_id"] = json!(99), + FOREIGN_CLUSTER => value["cluster_id"] = json!("foreign-cluster"), + _ => {} + } + Ok(value) + } + + async fn health( + &self, + _: &AppState, + _: &ConsoleConfig, + _: &ServerEntry, + _: Option, + _: &[NodeServiceIntent], + ) -> NodeHealth { + NodeHealth::Up + } +} + +pub fn start(fixture: &mut TestServiceInventory, bootstrap_pkv: bool) -> Arc { + if bootstrap_pkv { + let mut config = fixture.state.config.write().unwrap(); + let entry = config.servers.iter_mut().find(|s| s.node_id == Some(1)).unwrap(); + entry.id = "kv-1".into(); + entry.service_type = ServiceType::PaxosKv; + entry.url = "http://127.0.0.1:10000".into(); + entry.rpc_url = Some("127.0.0.1:10100".into()); + entry.rest_port = Some(10000); + entry.rpc_port = Some(10100); + } + let mode = Arc::new(AtomicU32::new(RUNNING)); + fixture.observer = Some( + crowdb_web::start_service_observer_for_tests( + &fixture.state, + Arc::new(TestProcessStateSource { + base: fixture.source.clone(), + mode: mode.clone(), + bootstrap_pkv, + }), + ) + .unwrap(), + ); + mode +} + +pub async fn wait(fixture: &TestServiceInventory, expected: &str) -> Value { + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let rows = fixture.rows().await; + let row = rows + .as_array() + .unwrap() + .iter() + .find(|r| r["node_id"] == 1) + .unwrap(); + let observed = row["process_observed"].as_bool(); + let matches = match expected { + "running" => observed == Some(true) && row["pid"].as_u64().is_some() && row["health"] == "up", + "stopped" => observed == Some(true) && row["pid"].is_null(), + "unknown" => observed == Some(false) && row["pid"].is_null(), + _ => panic!("unexpected process evidence state"), + }; + if matches { + return row.clone(); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("production observation did not distinguish validity from PID presence") +} diff --git a/app/crowdb-web/tests/common/process_transport.rs b/app/crowdb-web/tests/common/process_transport.rs new file mode 100644 index 000000000..c9d4e76ae --- /dev/null +++ b/app/crowdb-web/tests/common/process_transport.rs @@ -0,0 +1,90 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::fixture::TestSource; +use crowdb_console_shared::{ + cluster::NodeHealth, + config::{NodeEntry, ServerEntry}, + ConsoleConfig, +}; +use crowdb_protocol::mgmt::node::{NodeBinding, NodeServiceIntent}; +use crowdb_web::{AppState, ProcessObservationForTests, ServiceObservationSourceForTests}; +use serde_json::Value; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +#[derive(Default)] +pub struct TestCounts { + pub created: [AtomicUsize; 3], + pub dropped: [AtomicUsize; 3], + pub reads: [AtomicUsize; 3], +} + +pub struct TestTransportSource { + pub source: Arc, + pub counts: Arc, +} + +struct TestProcessTransport { + source: Arc, + counts: Arc, + state: AppState, + binding: NodeBinding, + node: NodeEntry, +} + +impl Drop for TestProcessTransport { + fn drop(&mut self) { + self.counts.dropped[usize::try_from(self.node.id).unwrap()].fetch_add(1, Ordering::Relaxed); + } +} + +#[async_trait::async_trait] +impl ProcessObservationForTests for TestProcessTransport { + async fn read(&mut self) -> Result { + self.counts.reads[usize::try_from(self.node.id).unwrap()].fetch_add(1, Ordering::Relaxed); + self.source + .processes(&self.state, &self.binding, &self.node) + .await + } +} + +#[async_trait::async_trait] +impl ServiceObservationSourceForTests for TestTransportSource { + fn process_observer( + &self, + state: &AppState, + binding: &NodeBinding, + node: &NodeEntry, + ) -> Result>, String> { + self.counts.created[usize::try_from(node.id).unwrap()].fetch_add(1, Ordering::Relaxed); + Ok(Some(Box::new(TestProcessTransport { + source: self.source.clone(), + counts: self.counts.clone(), + state: state.clone(), + binding: binding.clone(), + node: node.clone(), + }))) + } + + async fn intents(&self, state: &AppState) -> Result, String> { + self.source.intents(state).await + } + + async fn processes(&self, _: &AppState, _: &NodeBinding, _: &NodeEntry) -> Result { + panic!("the owned process transport must handle process sampling") + } + + async fn health( + &self, + state: &AppState, + config: &ConsoleConfig, + entry: &ServerEntry, + pid: Option, + intents: &[NodeServiceIntent], + ) -> NodeHealth { + self.source.health(state, config, entry, pid, intents).await + } +} diff --git a/app/crowdb-web/tests/node_confirmed_projection_test.rs b/app/crowdb-web/tests/node_confirmed_projection_test.rs new file mode 100644 index 000000000..fb62cb88f --- /dev/null +++ b/app/crowdb-web/tests/node_confirmed_projection_test.rs @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_console_shared::{ + config::{ServerEntry, ServiceType}, + deployment::registry::NodeRecord, +}; +use crowdb_web::project_confirmed_servers_for_tests; + +#[test] +fn auxiliary_projection_before_confirmation_keeps_bootstrap_kv_and_accepted_ports() { + for host in ["192.0.2.42", "2001:db8::42"] { + let mut auxiliary = ServerEntry::new("access-42", "http://original:16000"); + auxiliary.node_id = Some(42); + auxiliary.service_type = ServiceType::AccessServer; + auxiliary.rpc_url = Some("original:16001".into()); + let mut kv = ServerEntry::new("kv-42", "http://original:10042"); + kv.node_id = Some(42); + kv.rpc_url = Some("original:10142".into()); + let nodes = vec![NodeRecord { + discovery_id: "discovery-42".into(), + node_id: 42, + physical_host_id: "host-42".into(), + rack_id: 1, + host: host.into(), + web_port: 9090, + ssh_port: 22, + ssh_user: "crowdb".into(), + operation_id: "admit-42".into(), + host_key: None, + confirmed: true, + cancelled: false, + }]; + let mut servers = vec![auxiliary.clone()]; + project_confirmed_servers_for_tests(&mut servers, vec![kv.clone()], &nodes); + assert_eq!( + servers.len(), + 2, + "auxiliary publication must not hide bootstrap KV" + ); + assert_eq!( + servers[0], auxiliary, + "KV recovery must not rewrite an auxiliary endpoint" + ); + let host = if host.contains(':') { + format!("[{host}]") + } else { + host.into() + }; + kv.url = format!("http://{host}:10042"); + kv.rpc_url = Some(format!("{host}:10142")); + assert_eq!(servers[1], kv); + let retained = servers.clone(); + project_confirmed_servers_for_tests(&mut servers, vec![kv], &nodes); + assert_eq!( + servers, retained, + "repeated confirmation must preserve service identities" + ); + } +} diff --git a/app/crowdb-web/tests/s3_inspection_proxy_test.rs b/app/crowdb-web/tests/s3_inspection_proxy_test.rs index 9df5cc72b..ea856bbdf 100644 --- a/app/crowdb-web/tests/s3_inspection_proxy_test.rs +++ b/app/crowdb-web/tests/s3_inspection_proxy_test.rs @@ -30,7 +30,7 @@ async fn upstream( r#"{"error":"Object generation changed"}"#.into(), ), "missing" => (StatusCode::NOT_FOUND, r#"{"error":"Object missing"}"#.into()), - "folder/中文.bin" => { + "folder/\u{4e2d}\u{6587}.bin" => { assert_eq!(query["limit"], "20"); assert_eq!(query["cursor"], "opaque+/="); (StatusCode::OK, r#"{"offset":"9007199254740993"}"#.into()) diff --git a/app/crowdb-web/tests/service_inventory_observation_test.rs b/app/crowdb-web/tests/service_inventory_observation_test.rs index 349397c05..1c177d022 100644 --- a/app/crowdb-web/tests/service_inventory_observation_test.rs +++ b/app/crowdb-web/tests/service_inventory_observation_test.rs @@ -19,6 +19,7 @@ async fn inventory_readers_do_not_wait_for_an_unrelated_monitor_or_refresh_evide TestServiceInventory::signalled(&fixture.source.process_entered).await; let row = fixture.wait_pid(1, Some(101)).await; assert_eq!(row["pid_node_id"], 1); + assert_eq!(row["process_observed"], true); assert_eq!(fixture.rows().await.as_array().unwrap().len(), 2); assert!(crowdb_web::start_service_observer(&fixture.state).is_err()); fixture.observer.take(); @@ -33,6 +34,7 @@ async fn inventory_readers_do_not_wait_for_an_unrelated_monitor_or_refresh_evide assert!(rows[0]["pid"].is_null()); assert!(rows[0]["pid_node_id"].is_null()); assert_eq!(rows[0]["health"], "unknown"); + assert_eq!(rows[0]["process_observed"], false); assert_eq!(fixture.source.calls.load(Ordering::Relaxed), calls); } @@ -163,6 +165,7 @@ async fn fresh_bound_processes_remain_observable_when_authority_expires_but_acce tokio::time::sleep(Duration::from_millis(3150)).await; let row = fixture.wait_pid(1, Some(101)).await; assert_eq!(row["pid_node_id"], 1); + assert_eq!(row["process_observed"], true); assert_eq!(row["health"], "unknown"); assert_eq!(row["start_pending"], false); } @@ -189,6 +192,7 @@ async fn changed_endpoint_wrong_bootstrap_and_wrong_executed_operation_never_reu .find(|row| row["node_id"] == 1) .unwrap(); assert!(row["pid"].is_null()); + assert_eq!(row["process_observed"], false); assert_eq!(row["health"], "unknown"); std::fs::remove_file(fixture.root.path().join("node-binding.json")).unwrap(); for row in fixture.rows().await.as_array().unwrap() { diff --git a/app/crowdb-web/tests/service_process_observed_test.rs b/app/crowdb-web/tests/service_process_observed_test.rs new file mode 100644 index 000000000..d818d4de3 --- /dev/null +++ b/app/crowdb-web/tests/service_process_observed_test.rs @@ -0,0 +1,230 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/process_observed.rs"] +mod evidence; +#[path = "common/service_inventory.rs"] +mod fixture; + +use crowdb_console_shared::{config::ServerEntry, ConsoleConfig}; +use crowdb_web::AppState; +use evidence::{ + start, wait, ABSENT, ARRAY_MAP, EXHAUSTED, FOREIGN_BOOTSTRAP, FOREIGN_CLUSTER, FOREIGN_EXHAUSTION, + FOREIGN_NODE, FOREIGN_OPERATION, MISSING_MAP, NULL_MAP, READ_ERROR, RUNNING, STRING_PID, ZERO_PID, +}; +use fixture::TestServiceInventory; +use serde_json::{json, Value}; +use std::{sync::atomic::Ordering, time::Duration}; + +#[tokio::test] +async fn unavailable_process_evidence_is_unknown_even_when_endpoint_health_is_up_then_same_pid_returns() { + let mut fixture = TestServiceInventory::new(false); + let initial = fixture.rows().await; + assert_eq!(initial[0]["process_observed"], false); + let mode = start(&mut fixture, false); + assert_eq!(wait(&fixture, "running").await["pid"], 101); + mode.store(READ_ERROR, Ordering::Relaxed); + let row = wait(&fixture, "unknown").await; + assert!(row["pid"].is_null()); + assert_eq!(row["health"], "up"); + mode.store(RUNNING, Ordering::Relaxed); + assert_eq!(wait(&fixture, "running").await["pid"], 101); +} + +#[tokio::test] +async fn fresh_empty_process_map_confirms_stopped_and_only_current_exhaustion_ends_pending_start() { + let mut fixture = TestServiceInventory::new(false); + let mode = start(&mut fixture, false); + wait(&fixture, "running").await; + mode.store(ABSENT, Ordering::Relaxed); + let row = wait(&fixture, "stopped").await; + assert!(row["pid"].is_null()); + assert_eq!(row["start_pending"], true); + mode.store(EXHAUSTED, Ordering::Relaxed); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let row = wait(&fixture, "stopped").await; + if row["start_pending"] == false { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + mode.store(FOREIGN_EXHAUSTION, Ordering::Relaxed); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let row = wait(&fixture, "stopped").await; + if row["start_pending"] == true { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn foreign_live_operation_or_bootstrap_is_unknown_never_observed_stopped_or_adopted() { + let mut fixture = TestServiceInventory::new(false); + let mode = start(&mut fixture, false); + wait(&fixture, "running").await; + for wrong in [FOREIGN_OPERATION, FOREIGN_BOOTSTRAP] { + mode.store(wrong, Ordering::Relaxed); + let row = wait(&fixture, "unknown").await; + assert!(row["pid"].is_null()); + mode.store(RUNNING, Ordering::Relaxed); + assert_eq!(wait(&fixture, "running").await["pid"], 101); + } +} + +#[tokio::test] +async fn expired_process_evidence_is_unknown_and_never_retains_a_pid() { + let mut fixture = TestServiceInventory::new(false); + start(&mut fixture, false); + wait(&fixture, "running").await; + fixture.observer.take(); + tokio::task::yield_now().await; + tokio::time::sleep(Duration::from_millis(3050)).await; + let row = wait(&fixture, "unknown").await; + assert!(row["pid"].is_null()); + assert_eq!(row["start_pending"], false); +} + +#[tokio::test] +async fn bound_bootstrap_pkv_without_intent_distinguishes_no_read_from_positive_process_absence() { + let mut fixture = TestServiceInventory::new(false); + let mode = start(&mut fixture, true); + let row = wait(&fixture, "running").await; + assert_eq!(row["pid"], 101); + assert_eq!(row["start_pending"], false); + mode.store(ABSENT, Ordering::Relaxed); + let row = wait(&fixture, "stopped").await; + assert!(row["pid"].is_null()); + assert_eq!(row["start_pending"], false); + mode.store(READ_ERROR, Ordering::Relaxed); + assert!(wait(&fixture, "unknown").await["pid"].is_null()); + mode.store(RUNNING, Ordering::Relaxed); + assert_eq!(wait(&fixture, "running").await["pid"], 101); +} + +#[tokio::test] +async fn missing_malformed_process_maps_and_invalid_pid_values_never_confirm_absence() { + let mut fixture = TestServiceInventory::new(false); + let mode = start(&mut fixture, false); + wait(&fixture, "running").await; + for malformed in [MISSING_MAP, NULL_MAP, ARRAY_MAP, ZERO_PID, STRING_PID] { + mode.store(malformed, Ordering::Relaxed); + assert!(wait(&fixture, "unknown").await["pid"].is_null()); + mode.store(RUNNING, Ordering::Relaxed); + assert_eq!(wait(&fixture, "running").await["pid"], 101); + } +} + +#[tokio::test] +async fn wrong_monitor_node_or_cluster_is_unknown_not_observed_stopped() { + let mut fixture = TestServiceInventory::new(false); + let mode = start(&mut fixture, false); + wait(&fixture, "running").await; + for foreign in [FOREIGN_NODE, FOREIGN_CLUSTER] { + mode.store(foreign, Ordering::Relaxed); + assert!(wait(&fixture, "unknown").await["pid"].is_null()); + mode.store(RUNNING, Ordering::Relaxed); + assert_eq!(wait(&fixture, "running").await["pid"], 101); + } +} + +#[tokio::test] +async fn accepted_no_pid_mutation_reply_does_not_manufacture_process_absence() { + let mut fixture = TestServiceInventory::new(false); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + fixture.source.hold_node.store(1, Ordering::Relaxed); + TestServiceInventory::signalled(&fixture.source.process_entered).await; + let intent = fixture.source.intents.load_full()[0].clone(); + let mutation = + crowdb_web::invalidate_service_observation_for_tests(&fixture.state, 1, &intent.service_id); + mutation.complete( + intent.clone(), + &json!({"operation_id": intent.operation_id, "pid": null}), + ); + let row = wait(&fixture, "unknown").await; + assert!(row["pid"].is_null()); + assert_eq!(row["start_pending"], true); + fixture.source.hold_node.store(0, Ordering::Relaxed); + fixture.source.process_release.notify_one(); + TestServiceInventory::signalled(&fixture.source.process_finished).await; + fixture.wait_pid(1, Some(101)).await; + assert_eq!(wait(&fixture, "running").await["pid"], 101); +} + +#[tokio::test] +async fn accepted_actual_pid_mutation_reply_is_observed_and_late_sample_remains_fenced() { + let mut fixture = TestServiceInventory::new(false); + fixture.start(); + fixture.wait_pid(1, Some(101)).await; + fixture.source.hold_node.store(1, Ordering::Relaxed); + TestServiceInventory::signalled(&fixture.source.process_entered).await; + let intent = fixture.source.intents.load_full()[0].clone(); + let mutation = + crowdb_web::invalidate_service_observation_for_tests(&fixture.state, 1, &intent.service_id); + fixture.source.pid.store(500, Ordering::Relaxed); + mutation.complete( + intent.clone(), + &json!({"operation_id": intent.operation_id, "pid": 501}), + ); + let row = wait(&fixture, "running").await; + assert_eq!(row["pid"], 501); + fixture.source.hold_node.store(0, Ordering::Relaxed); + fixture.source.process_release.notify_one(); + TestServiceInventory::signalled(&fixture.source.process_finished).await; + assert_eq!(wait(&fixture, "running").await["pid"], 501); +} + +#[tokio::test] +async fn changed_node_configuration_or_missing_binding_never_reuses_observed_process_evidence() { + let mut fixture = TestServiceInventory::new(false); + start(&mut fixture, false); + wait(&fixture, "running").await; + // Freeze the accepted old fact; a legitimate new-node read is outside this check. + fixture.observer.take(); + tokio::task::yield_now().await; + fixture.state.config.write().unwrap().nodes[0].host = "127.0.0.99".into(); + let row = wait(&fixture, "unknown").await; + assert!(row["pid"].is_null()); + std::fs::remove_file(fixture.root.path().join("node-binding.json")).unwrap(); + for row in fixture.rows().await.as_array().unwrap() { + assert_eq!(row["process_observed"], false); + assert!(row["pid"].is_null()); + } +} + +#[tokio::test] +async fn local_inventory_omits_monitor_observation_validity_for_legacy_consumers() { + use axum::{ + body::{to_bytes, Body}, + http::Request, + }; + use tower::ServiceExt; + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("local-process-observed"); + let mut config = ConsoleConfig::default(); + config + .servers + .push(ServerEntry::new("external", "http://127.0.0.1:1")); + let state = AppState::with_runtime_root(config, root.path().join("console")); + let response = crowdb_web::router(state) + .oneshot( + Request::builder() + .uri("/api/servers") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert!(response.status().is_success()); + let rows: Value = serde_json::from_slice(&to_bytes(response.into_body(), 65_536).await.unwrap()).unwrap(); + assert_eq!(rows.as_array().unwrap().len(), 1); + assert!(!rows[0].as_object().unwrap().contains_key("process_observed")); +} diff --git a/app/crowdb-web/tests/service_process_transport_test.rs b/app/crowdb-web/tests/service_process_transport_test.rs new file mode 100644 index 000000000..092e25977 --- /dev/null +++ b/app/crowdb-web/tests/service_process_transport_test.rs @@ -0,0 +1,67 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/service_inventory.rs"] +#[allow(dead_code)] // Shared fixture also supplies entry points used by the inventory suite. +mod fixture; +#[path = "common/process_transport.rs"] +mod transport; + +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; +use transport::{TestCounts, TestTransportSource}; + +async fn until(condition: impl Fn() -> bool) { + tokio::time::timeout(Duration::from_secs(2), async { + while !condition() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("process transport ownership did not converge"); +} + +#[tokio::test] +async fn process_transport_belongs_to_one_node_configuration_and_bootstrap() { + let mut fixture = fixture::TestServiceInventory::new(false); + let counts = Arc::new(TestCounts::default()); + let source = Arc::new(TestTransportSource { + source: fixture.source.clone(), + counts: counts.clone(), + }); + fixture.observer = Some(crowdb_web::start_service_observer_for_tests(&fixture.state, source).unwrap()); + fixture.wait_pid(1, Some(101)).await; + fixture.wait_pid(2, Some(102)).await; + until(|| counts.reads[2].load(Ordering::Relaxed) >= 2).await; + assert_eq!(counts.created[1].load(Ordering::Relaxed), 1); + assert_eq!(counts.created[2].load(Ordering::Relaxed), 1); + fixture.state.config.write().unwrap().nodes[1].host = "127.0.0.99".into(); + until(|| { + counts.created[2].load(Ordering::Relaxed) == 2 && counts.dropped[2].load(Ordering::Relaxed) == 1 + }) + .await; + assert_eq!(counts.created[1].load(Ordering::Relaxed), 1); + let path = fixture.root.path().join("node-binding.json"); + let mut binding: crowdb_protocol::mgmt::node::NodeBinding = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + std::fs::remove_file(&path).unwrap(); + until(|| { + counts.dropped[1].load(Ordering::Relaxed) == 1 && counts.dropped[2].load(Ordering::Relaxed) == 2 + }) + .await; + binding.bootstrap.operation_id = "replacement-bootstrap".into(); + std::fs::write(&path, serde_json::to_vec(&binding).unwrap()).unwrap(); + fixture.wait_pid(1, Some(101)).await; + fixture.wait_pid(2, Some(102)).await; + assert_eq!(counts.created[1].load(Ordering::Relaxed), 2); + assert_eq!(counts.created[2].load(Ordering::Relaxed), 3); + fixture.observer.take(); + until(|| { + (1..=2).all(|node| { + counts.created[node].load(Ordering::Relaxed) == counts.dropped[node].load(Ordering::Relaxed) + }) + }) + .await; +} diff --git a/app/crowdb-web/ui/src/api.ts b/app/crowdb-web/ui/src/api.ts index d9bb422fa..e6bf57ee5 100644 --- a/app/crowdb-web/ui/src/api.ts +++ b/app/crowdb-web/ui/src/api.ts @@ -1197,6 +1197,8 @@ export async function removeDisk(nodeId: number, dgId: number, diskId: string, o /** `GET /api/servers` — list all deployed server entries. */ export interface ServerSummary { + /** Fresh authenticated process evidence; omitted by legacy local inventories. */ + process_observed?: boolean; start_pending?: boolean; id?: string; node_id?: number; diff --git a/app/crowdb-web/ui/src/dataset/editor/Table.test.tsx b/app/crowdb-web/ui/src/dataset/editor/Table.test.tsx index c5f8f41f3..17b164edd 100644 --- a/app/crowdb-web/ui/src/dataset/editor/Table.test.tsx +++ b/app/crowdb-web/ui/src/dataset/editor/Table.test.tsx @@ -43,7 +43,7 @@ it('edits a Sample as one row, persists only metadata and does not publish on Sa it('patches a cell without losing other Fields and rejects unbounded authoring input', () => { const op = setCell(draft(), btoa('note'), 'label', textValue('keep')); expect(setCell(op, btoa('note'), 'content', { kind: 'upload', id: '3'.repeat(32), length: '12', md5: '4'.repeat(32) }).mapping!.samples[0]).toMatchObject({ fields: [{ name: 'label' }, { name: 'content' }] }); - expect(() => textValue('字'.repeat(22000))).toThrow('64 KiB'); + expect(() => textValue('\u5b57'.repeat(22000))).toThrow('64 KiB'); expect(() => checkLimits({ ...op, mapping: { ...op.mapping!, samples: Array(1001).fill(op.mapping!.samples[0]) } })).toThrow('1000'); }); it('projects new columns as Missing without asking the parent for unknown Fields', async () => { diff --git a/app/crowdb-web/ui/src/kv/displayBytes.test.ts b/app/crowdb-web/ui/src/kv/displayBytes.test.ts index 53c72e1d9..78c8dc96f 100644 --- a/app/crowdb-web/ui/src/kv/displayBytes.test.ts +++ b/app/crowdb-web/ui/src/kv/displayBytes.test.ts @@ -7,7 +7,7 @@ describe('mixed KV byte presentation', () => { it('keeps printable fields around control and invalid UTF-8 bytes', () => { expect(byteRuns(undefined, '616200ffe4b8ad6364')).toEqual([ { text: 'ab', binary: false }, { text: '00FF', binary: true }, - { text: '中cd', binary: false }, + { text: '\u4e2dcd', binary: false }, ]); expect(printableBytes(undefined, '616200ff')).toBe(false); }); @@ -18,8 +18,8 @@ describe('mixed KV byte presentation', () => { expect(displayBytes(undefined, 'e282')).toBe('E282'); }); it('distinguishes text that looks like hex and preserves valid Unicode', () => { - expect(byteRuns('FF00中😀', undefined)).toEqual([{ text: 'FF00中😀', binary: false }]); - expect(printableBytes('FF00中😀', undefined)).toBe(true); + expect(byteRuns('FF00\u4e2d😀', undefined)).toEqual([{ text: 'FF00\u4e2d😀', binary: false }]); + expect(printableBytes('FF00\u4e2d😀', undefined)).toBe(true); expect(byteRuns(undefined, 'efbbbf41')).toEqual([{ text: 'EFBBBF', binary: true }, { text: 'A', binary: false }]); }); }); diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx b/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx index b300d5a6b..9b7c073c8 100644 --- a/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx +++ b/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx @@ -320,20 +320,23 @@ describe('node service plans', () => { expect(result.current.plans[1]['access-server'].state).toBe('deployed'); expect(vi.mocked(serviceRequest).mock.calls.filter(([, method]) => method === 'POST')).toHaveLength(3); }); - it('does not retry failures automatically and stops queued deployments before reset', async () => { + it('observes failures without replaying mutations and stops observation before reset', async () => { vi.mocked(serviceRequest).mockImplementation(async path => { if (path === '/service-plans') return {}; if (path === '/chunk-storage-readiness' || path === '/group0-readiness') return { ready: true }; if (path.endsWith('/service-plan')) return { revision: 1, ready: true }; throw new Error('Unavailable'); }); const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); await act(async () => { result.current.start(1); }); expect(result.current.plans[1].chunkdb.state).toBe('failed'); - const attempts = vi.mocked(serviceRequest).mock.calls.length; + const mutations = vi.mocked(serviceRequest).mock.calls.filter(([, method]) => method === 'POST'); + const observations = vi.mocked(listServers).mock.calls.length; await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); - expect(serviceRequest).toHaveBeenCalledTimes(attempts); + expect(vi.mocked(listServers).mock.calls.length).toBeGreaterThan(observations); + expect(vi.mocked(serviceRequest).mock.calls.filter(([, method]) => method === 'POST')).toEqual(mutations); + const attempts = vi.mocked(serviceRequest).mock.calls.length; await act(async () => { await result.current.stop(); }); await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); expect(result.current.plans).toEqual({}); expect(serviceRequest).toHaveBeenCalledTimes(attempts); }); - it('recovers waiting progress and fences interrupted deployment until explicit reconciliation', async () => { + it('recovers waiting progress and observes an interrupted deployment without replaying it', async () => { const steps = Object.fromEntries(['paxos-kv', 'diskdb', 'chunkdb', 'diskio', 'chunk-kv', 'access-server'].map(kind => [kind, { state: kind === 'chunkdb' ? 'deploying' : kind === 'paxos-kv' || kind === 'diskdb' ? 'deployed' : 'waiting' }])); vi.mocked(serviceRequest).mockImplementation(async path => path === '/service-plans' ? { 1: { revision: 7, steps } } : path === '/chunk-storage-readiness' ? { ready: true } : { revision: 8, ready: true }); @@ -346,7 +349,7 @@ describe('node service plans', () => { { node_id: 1, service_type: 'paxos-kv', pid: 10 }, { node_id: 1, service_type: 'diskdb', pid: 11 }, { node_id: 1, service_type: 'chunkdb', pid: 12 }, ] as Awaited>); - await act(async () => { await result.current.start(1); }); + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); expect(result.current.plans[1].chunkdb.state).toBe('deployed'); expect(vi.mocked(serviceRequest).mock.calls.some(([path, method, body]) => path.endsWith('/services/deploy') && method === 'POST' && (body as { kind?: string })?.kind === 'chunkdb')).toBe(false); expect(serviceRequest).toHaveBeenCalledWith('/nodes/1/service-plan', 'PUT', expect.objectContaining({ revision: 7 })); @@ -414,7 +417,8 @@ describe('node service plans', () => { vi.mocked(listServers).mockRejectedValue(new Error('Registry unavailable')); const { result } = renderHook(() => useNodeServicePlans([], {}, async () => {}, true)); await act(async () => { await result.current.start(1, ['paxos-kv']); }); - expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + expect(result.current.plans[1]['paxos-kv'].state).toBe('waiting'); + expect(result.current.plans[1]['paxos-kv'].detail).toContain('inventory'); for (const kind of serviceOrder.filter(kind => kind !== 'paxos-kv')) expect(result.current.plans[1][kind].state).toBe('disabled'); }); it('rechecks registered DiskIO ownership promptly without requiring disk rows to be expanded', async () => { diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlans.ts b/app/crowdb-web/ui/src/services/useNodeServicePlans.ts index 4f6b6c8c8..f6e9b4643 100644 --- a/app/crowdb-web/ui/src/services/useNodeServicePlans.ts +++ b/app/crowdb-web/ui/src/services/useNodeServicePlans.ts @@ -11,6 +11,8 @@ export const serviceOrder = ['access-server', 'chunk-kv', 'chunkdb', 'diskdb', ' export type ServiceKind = typeof serviceOrder[number]; const deploymentOrder: ServiceKind[] = ['paxos-kv', 'diskdb', 'diskio', 'chunkdb', 'chunk-kv', 'access-server']; const startingService = 'Waiting: target monitor is starting the registered service'; +const observingService = 'Waiting: current process observation is unavailable'; +const observingInventory = 'Waiting: current service inventory is unavailable'; export const serviceLabels = { 'access-server': 'crowdb-access-server', 'chunk-kv': 'crowdb-chunk-kv', @@ -24,8 +26,22 @@ export type NodeServicePlan = Record; export type ServiceOverrides = Partial> & { keepalive_interval_secs?: number }>>; export const newPlan = (): NodeServicePlan => Object.fromEntries(serviceOrder.map(kind => [kind, { state: 'pending' }])) as NodeServicePlan; +function observationDelay(plans: Record, hasDisks: boolean, waitingForStorage: boolean, immediate: boolean) { + if (immediate) return 0; + const steps = Object.values(plans).flatMap(plan => serviceOrder.map(kind => plan[kind])); + const queued = steps.some(step => step.state === 'waiting'); + const observing = steps.some(step => step.state === 'failed' + || step.detail === startingService || step.detail === observingService + || step.detail?.startsWith(observingInventory) + || step.detail?.startsWith('Waiting: restart registered DiskIO services')); + const waitingForAuthority = steps.some(step => step.detail === 'Waiting: initialize Group 0 in Paxos KV'); + // Missing Capacity data does not delay observations of registered services. + return queued && !hasDisks && !observing && !waitingForAuthority && !waitingForStorage ? 10000 : 2000; +} + /** Serializes default deployments across nodes so each step gets fresh port reservations. - * Plans outlive dialogs; dependency changes resume waiting steps, failures need Retry. + * Plans outlive dialogs; dependency changes and observed recovery update progress. + * Failed lifecycle mutations need explicit Retry; observation never replays them. * Progress is durable; interrupted mutations require registration reconciliation. */ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record, refresh: () => Promise, enabled: boolean) { @@ -75,7 +91,7 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< ? { ...savedStep, state: 'warning' as const } : savedStep; return [kind, normalizedStep.state === 'deploying' - ? { state: 'failed', detail: 'Deployment was interrupted; verify the registered service, then Retry to reconcile.' } : normalizedStep]; + ? { state: 'failed', detail: 'Deployment was interrupted; checking the registered service. Retry only if it still needs to be started.' } : normalizedStep]; })) as NodeServicePlan; update(id, steps); } @@ -132,7 +148,7 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< if (disposed || stopped.current) return; for (const [key, initial] of Object.entries(plansRef.current)) { if (disposed || stopped.current) break; - if (!serviceOrder.some(kind => ['pending', 'waiting'].includes(initial[kind].state))) continue; + if (!serviceOrder.some(kind => ['pending', 'waiting', 'failed'].includes(initial[kind].state))) continue; const id = Number(key); let plan = plansRef.current[id]; const write = async (kind: ServiceKind, step: ServiceStep) => { plan = { ...plan, [kind]: step }; await persist(id, plan); }; @@ -140,7 +156,9 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< try { existing = await listServers(); } catch (error) { if (disposed || stopped.current) return; - for (const kind of serviceOrder) if (!['deployed', 'disabled'].includes(plan[kind].state)) await write(kind, { state: 'failed', detail: String(error) }); + for (const kind of serviceOrder) if (['pending', 'waiting'].includes(plan[kind].state)) { + await write(kind, { state: 'waiting', detail: `${observingInventory}: ${String(error)}` }); + } continue; } let authorityReady = false; @@ -152,8 +170,12 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< for (const kind of deploymentOrder) { if (disposed || stopped.current || !input.current.enabled) break; if (plan[kind].state === 'disabled') continue; - if (plan[kind].state === 'failed' || plan[kind].state === 'warning') continue; + if (plan[kind].state === 'warning') continue; const registered = existing.find(server => server.node_id === id && server.service_type === kind); + if (plan[kind].state === 'failed') { + if (registered?.pid && registered.process_observed !== false) await write(kind, { state: 'deployed' }); + continue; + } if (registered) { const retryKey = `${id}/${kind}`; if (!registered.pid && retryRegistered.current.has(retryKey)) { @@ -174,7 +196,9 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< retryRegistered.current.delete(retryKey); const step: ServiceStep = registered.pid ? { state: 'deployed' } : registered.start_pending ? { state: 'waiting', detail: startingService } - : { state: 'failed', detail: 'Registered service is stopped; restart it from the Node menu before resuming.' }; + : registered.process_observed === false + ? { state: 'waiting', detail: observingService } + : { state: 'failed', detail: 'Registered service is stopped; restart it from the Node menu before resuming.' }; if (plan[kind].state !== step.state || plan[kind].detail !== step.detail) await write(kind, step); continue; } @@ -250,13 +274,8 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< } finally { busy.current = false; } } if (!disposed) { - const hasQueuedWork = Object.values(plansRef.current).some(plan => serviceOrder.some(kind => plan[kind].state === 'waiting')); - const waitingForAuthority = Object.values(plansRef.current).some(plan => serviceOrder.some(kind => plan[kind].detail === 'Waiting: initialize Group 0 in Paxos KV')); - const waitingForRestart = Object.values(plansRef.current).some(plan => serviceOrder.some(kind => plan[kind].detail?.startsWith('Waiting: restart registered DiskIO services'))); - const waitingForStart = Object.values(plansRef.current).some(plan => serviceOrder.some(kind => plan[kind].detail === startingService)); const hasDisks = Object.values(input.current.groups).some(node => Object.values(node.disksByDg).some(disks => disks.length > 0)); - // Collapsed disk rows are not evidence that registered storage is absent. - const delay = wakePending.current ? 0 : hasQueuedWork && !hasDisks && !waitingForAuthority && !waitingForRestart && !waitingForStart && !waitingForStorage ? 10000 : 2000; + const delay = observationDelay(plansRef.current, hasDisks, waitingForStorage, wakePending.current); wakePending.current = false; timer = setTimeout(() => { running.current = tick(); }, delay); } diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlansObservation.test.tsx b/app/crowdb-web/ui/src/services/useNodeServicePlansObservation.test.tsx new file mode 100644 index 000000000..861b7de82 --- /dev/null +++ b/app/crowdb-web/ui/src/services/useNodeServicePlansObservation.test.tsx @@ -0,0 +1,218 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +import { act, renderHook } from '@testing-library/react'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { deployServer, deployDiskdb, listServers, restartServer, restartDiskdb } from '../api'; +import type { ServerSummary } from '../api'; +import { serviceRequest } from './client'; +import { newPlan, serviceOrder, useNodeServicePlans } from './useNodeServicePlans'; +import type { EnrichedStoreView } from '../types'; + +vi.mock('../api', () => ({ listServers: vi.fn(), deployServer: vi.fn(), deployDiskdb: vi.fn(), restartServer: vi.fn(), restartDiskdb: vi.fn() })); +vi.mock('./client', () => ({ serviceNames: {}, serviceRequest: vi.fn() })); +type ProcessState = 'running' | 'stopped' | 'unknown'; +type ObservedServer = ServerSummary & { process_observed: boolean }; +const stores = [{ store_id: '0', groups: [{ group_id: '0' }, { group_id: '1' }] }] as unknown as EnrichedStoreView[]; +const kv = (state: ProcessState): ObservedServer => ({ id: 'kv-1', node_id: 1, service_type: 'paxos-kv', + pid: state === 'running' ? 126 : undefined, process_observed: state !== 'unknown', start_pending: false, health: 'up' }); +const diskdb = (running: boolean, pending: boolean): ObservedServer => ({ id: 'diskdb-1', node_id: 1, + service_type: 'diskdb', pid: running ? 272 : undefined, process_observed: !pending, + start_pending: pending, health: 'up' }); +function savedPlan() { + const steps = newPlan(); + for (const kind of serviceOrder) steps[kind] = { state: 'disabled' }; + steps['paxos-kv'] = { state: 'deployed' }; + steps.diskdb = { state: 'waiting', detail: 'Waiting: target monitor is starting the registered service' }; + vi.mocked(serviceRequest).mockImplementation(async path => path === '/service-plans' + ? { 1: { revision: 7, steps } } : { revision: 8, ready: true }); +} +function noLifecycleMutation() { + expect(deployServer).not.toHaveBeenCalled(); + expect(deployDiskdb).not.toHaveBeenCalled(); + expect(restartServer).not.toHaveBeenCalled(); + expect(restartDiskdb).not.toHaveBeenCalled(); + expect(vi.mocked(serviceRequest).mock.calls.some(([path, method]) => method === 'POST' + && (path.endsWith('/deploy') || path.endsWith('/restart')))).toBe(false); +} +beforeEach(() => { vi.useFakeTimers(); savedPlan(); }); +afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); + +it('observes an unknown current PKV process fact and recovers the same PID without Retry', async () => { + let state: ProcessState = 'running'; + let finished = false; + vi.mocked(listServers).mockImplementation(async () => [kv(state), diskdb(finished, !finished)]); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => {}); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + state = 'unknown'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('waiting'); + expect(result.current.plans[1]['paxos-kv'].detail).toMatch(/process|observation/i); + noLifecycleMutation(); + state = 'running'; finished = true; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + expect(result.current.plans[1].diskdb.state).toBe('deployed'); + noLifecycleMutation(); +}); + +it.each([['confirmed observation', true], ['legacy field omitted', undefined]] as const)('observes a stopped PKV and reconciles its recovery without a lifecycle mutation with %s', async (_label, processObserved) => { + let state: ProcessState = 'running'; + vi.mocked(listServers).mockImplementation(async () => [{ ...kv(state), process_observed: processObserved }, diskdb(false, true)]); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => {}); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + state = 'stopped'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + await act(async () => { await vi.advanceTimersByTimeAsync(4000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + noLifecycleMutation(); + state = 'running'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + noLifecycleMutation(); +}); + +it('observes an exhausted startup without restarting it and reconciles a later owned PID', async () => { + let exhausted = false; + let laterPid = false; + vi.mocked(listServers).mockImplementation(async () => [kv('running'), diskdb(laterPid, !exhausted)]); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => {}); + expect(result.current.plans[1].diskdb.state).toBe('waiting'); + exhausted = true; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1].diskdb.state).toBe('failed'); + await act(async () => { await vi.advanceTimersByTimeAsync(4000); }); + expect(result.current.plans[1].diskdb.state).toBe('failed'); + noLifecycleMutation(); + laterPid = true; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1].diskdb.state).toBe('deployed'); + noLifecycleMutation(); +}); + +it('observes unknown process state every two seconds without loaded Capacity data', async () => { + let state: ProcessState = 'running'; + let finished = false; + vi.mocked(listServers).mockImplementation(async () => [kv(state), diskdb(finished, !finished)]); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => {}); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + // Finish the only other queued service before observing the unknown process. + // The next timer must observe this process rather than use the disk backoff. + state = 'unknown'; finished = true; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('waiting'); + expect(result.current.plans[1].diskdb.state).toBe('deployed'); + state = 'running'; + await act(async () => { await vi.advanceTimersByTimeAsync(1999); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('waiting'); + await act(async () => { await vi.advanceTimersByTimeAsync(1); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + noLifecycleMutation(); +}); + +function failedPlan() { + const steps = newPlan(); + for (const kind of serviceOrder) steps[kind] = { state: 'disabled' }; + steps['paxos-kv'] = { state: 'failed', detail: 'Previously reported stopped' }; + return steps; +} +function recoverPlan(steps = failedPlan(), second?: ReturnType) { + vi.mocked(serviceRequest).mockImplementation(async path => path === '/service-plans' + ? { 1: { revision: 7, steps }, ...(second ? { 2: { revision: 3, steps: second } } : {}) } + : { revision: 8, ready: true }); +} + +it('observes a persisted failed-only plan and recovers only after current process evidence returns', async () => { + recoverPlan(); + let state: ProcessState = 'unknown'; + vi.mocked(listServers).mockImplementation(async () => [kv(state)]); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => {}); + expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + noLifecycleMutation(); + state = 'running'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + noLifecycleMutation(); +}); + +it('does not deploy a failed missing service while observing its later registered recovery', async () => { + recoverPlan(); + let present = false; + vi.mocked(listServers).mockImplementation(async () => present ? [kv('running')] : []); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => { await vi.advanceTimersByTimeAsync(4000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + noLifecycleMutation(); + present = true; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + noLifecycleMutation(); +}); + +it('keeps failed-service observation at two seconds while another plan waits for unloaded disks', async () => { + const second = failedPlan(); + second['paxos-kv'] = { state: 'disabled' }; + second['chunk-kv'] = { state: 'waiting', detail: 'Waiting: deploy DiskIO on at least two nodes for journal mirrors' }; + recoverPlan(failedPlan(), second); + let state: ProcessState = 'stopped'; + vi.mocked(listServers).mockImplementation(async () => [kv(state)]); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => {}); + await act(async () => { await vi.advanceTimersByTimeAsync(0); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + state = 'running'; + await act(async () => { await vi.advanceTimersByTimeAsync(1999); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('failed'); + await act(async () => { await vi.advanceTimersByTimeAsync(1); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + noLifecycleMutation(); +}); + + +it('preserves an existing failure and fixed-slot warning across inventory read errors and observes recovery', async () => { + const steps = failedPlan(); + steps.chunkdb = { state: 'warning', detail: 'Healthy service outside the fixed slot plan' }; + recoverPlan(steps); + let unavailable = true; + vi.mocked(listServers).mockImplementation(async () => { + if (unavailable) throw new Error('Controlled inventory unavailable'); + return [kv('running')]; + }); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv']).toEqual(steps['paxos-kv']); + expect(result.current.plans[1].chunkdb).toEqual(steps.chunkdb); + noLifecycleMutation(); + unavailable = false; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + expect(result.current.plans[1].chunkdb).toEqual(steps.chunkdb); + noLifecycleMutation(); +}); + +it('keeps unattempted work waiting across an unavailable inventory and resumes observation within two seconds', async () => { + const steps = failedPlan(); + steps['paxos-kv'] = { state: 'pending' }; + recoverPlan(steps); + let unavailable = true; + vi.mocked(listServers).mockImplementation(async () => { + if (unavailable) throw new Error('Controlled inventory unavailable'); + return [kv('running')]; + }); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => { await vi.advanceTimersByTimeAsync(0); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('waiting'); + expect(result.current.plans[1]['paxos-kv'].detail).toContain('inventory'); + noLifecycleMutation(); + unavailable = false; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); + noLifecycleMutation(); +}); diff --git a/container/crowdb-monitor/src/monitor_log.rs b/container/crowdb-monitor/src/monitor_log.rs index 70b00dd5f..e8bf10800 100644 --- a/container/crowdb-monitor/src/monitor_log.rs +++ b/container/crowdb-monitor/src/monitor_log.rs @@ -125,4 +125,33 @@ impl MonitorLog { } Ok(()) } + /// Additive diagnostic record; legacy lifecycle event fields stay unchanged. + pub(crate) async fn record_process_exit( + &mut self, + service: &str, + pid: u32, + observation: &serde_json::Value, + ) -> Result<(), MonitorLogError> { + let timestamp_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| MonitorLogError::Clock)? + .as_millis(); + let warning = observation.get("success").and_then(serde_json::Value::as_bool) != Some(true); + let mut body = serde_json::to_vec(&serde_json::json!({ + "timestamp_ms": timestamp_ms, + "level": if warning { "warn" } else { "info" }, + "kind": "child_exit_observed", + "service": service, + "pid": pid, + "attempt": null, + "process_exit": observation, + }))?; + body.push(b'\n'); + self.output.write_record(&body).await?; + self.output.sync().await?; + if warning && self.mirror_warnings_to_stderr { + eprint!("{}", String::from_utf8_lossy(&body)); + } + Ok(()) + } } diff --git a/container/crowdb-monitor/src/node/runtime/health.rs b/container/crowdb-monitor/src/node/runtime/health.rs index 05723786c..6d9e14c53 100644 --- a/container/crowdb-monitor/src/node/runtime/health.rs +++ b/container/crowdb-monitor/src/node/runtime/health.rs @@ -5,12 +5,10 @@ use std::collections::BTreeMap; use std::time::Duration; use super::{durable, AcceptedNode, Result}; -use crate::{ - DeploymentProfile, MonitorPhase, MonitorStatus, ProcessManager, ServiceProfile, ServiceStatus, - StatusStore, -}; +use crate::{DeploymentProfile, MonitorPhase, MonitorStatus, ProcessManager, ServiceProfile, StatusStore}; mod intents; +mod observations; pub(super) struct NodeHealth { store: StatusStore, @@ -60,61 +58,7 @@ impl NodeHealth { } } self.reconcile_services(profile, processes).await?; - self.status.services.clear(); - let retained = super::services::retained(profile)?; - let mut ids = vec!["web".to_owned(), "kv".to_owned()]; - ids.extend(retained.iter().cloned().map(|intent| intent.service_id)); - for id in &ids { - if processes.owns(id) { - let mut healthy = processes.alive(id)?; - if healthy { - if id == "web" || id == "kv" { - let service = if id == "web" { - web.clone() - } else { - super::service(profile, "kv")? - }; - healthy = self - .probes - .probe_service(&service, &BTreeMap::new()) - .await - .is_ok(); - } else if let Some(intent) = retained - .iter() - .find(|intent| intent.service_id == *id && intent.kind != "kv") - { - healthy = match super::probes::for_intent(profile, intent) { - Ok(service) => self - .probes - .probe_service(&service, &BTreeMap::new()) - .await - .is_ok(), - Err(_) => false, - }; - } - } - self.status.services.insert( - id.clone(), - ServiceStatus { - operation_id: retained - .iter() - .find(|intent| intent.service_id == *id || (id == "kv" && intent.kind == "kv")) - .map(|intent| intent.operation_id.clone()), - pid: processes.pid(id), - generation: 1 + u64::from(*self.attempts.get(id.as_str()).unwrap_or(&0)), - healthy, - restart_attempts: *self.attempts.get(id.as_str()).unwrap_or(&0), - }, - ); - } - } - self.status.phase = if self.status.services.values().all(|service| service.healthy) { - MonitorPhase::Ready - } else { - MonitorPhase::Restarting - }; - self.store.publish(&mut self.status)?; - Ok(()) + self.observe(profile, web, processes).await } pub(in crate::node::runtime) async fn reconcile_services( @@ -159,7 +103,7 @@ impl NodeHealth { continue; } } - match super::services::execute(profile, processes, intent.clone()).await { + match super::services::reconcile(profile, processes, intent.clone()).await { Ok(_) => { *self.attempts.entry(intent.service_id.clone()).or_default() += 1; self.probe_failures.remove(&intent.service_id); diff --git a/container/crowdb-monitor/src/node/runtime/health/observations.rs b/container/crowdb-monitor/src/node/runtime/health/observations.rs new file mode 100644 index 000000000..f9b170c2f --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/health/observations.rs @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Read-only probes share a health pass instead of serializing their deadlines. + +use super::{NodeHealth, Result}; +use crate::{DeploymentProfile, MonitorPhase, ProcessManager, ServiceProfile, ServiceStatus}; +use std::collections::BTreeMap; + +impl NodeHealth { + pub(super) async fn observe( + &mut self, + profile: &DeploymentProfile, + web: &ServiceProfile, + processes: &mut ProcessManager, + ) -> Result<()> { + let retained = super::super::services::retained(profile)?; + let mut ids = vec!["web".to_owned(), "kv".to_owned()]; + ids.extend(retained.iter().map(|intent| intent.service_id.clone())); + let mut jobs = Vec::new(); + for id in ids { + if !processes.owns(&id) { + continue; + } + let intent = retained + .iter() + .find(|intent| intent.service_id == id || (id == "kv" && intent.kind == "kv")); + let service = match id.as_str() { + "web" => Some(web.clone()), + "kv" => Some(super::super::service(profile, "kv")?), + _ => intent.and_then(|intent| super::super::probes::for_intent(profile, intent).ok()), + }; + let attempts = *self.attempts.get(&id).unwrap_or(&0); + jobs.push(( + id.clone(), + service, + ServiceStatus { + operation_id: intent.map(|intent| intent.operation_id.clone()), + pid: processes.pid(&id), + generation: 1 + u64::from(attempts), + healthy: processes.alive(&id)?, + restart_attempts: attempts, + }, + )); + } + let probes = &self.probes; + let observations = + futures::future::join_all(jobs.into_iter().map(|(id, service, mut status)| async move { + status.healthy = if status.healthy { + match service { + Some(service) => probes.probe_service(&service, &BTreeMap::new()).await.is_ok(), + None => false, + } + } else { + false + }; + (id, status) + })) + .await; + self.status.services.clear(); + for (id, mut status) in observations { + let alive = processes.alive(&id)?; + let current_pid = processes.pid(&id); + status.healthy &= alive && current_pid == status.pid; + status.pid = current_pid; + self.status.services.insert(id, status); + } + self.status.phase = if self.status.services.values().all(|service| service.healthy) { + MonitorPhase::Ready + } else { + MonitorPhase::Restarting + }; + self.store.publish(&mut self.status)?; + Ok(()) + } +} diff --git a/container/crowdb-monitor/src/node/runtime/services.rs b/container/crowdb-monitor/src/node/runtime/services.rs index 7fd9da347..d242000b1 100644 --- a/container/crowdb-monitor/src/node/runtime/services.rs +++ b/container/crowdb-monitor/src/node/runtime/services.rs @@ -32,6 +32,7 @@ mod observation; mod policy; mod receipt; pub(super) use execution::execute; +pub(super) use execution::reconcile; pub(super) use execution::stop_unhealthy; pub(super) use observation::observe; diff --git a/container/crowdb-monitor/src/node/runtime/services/execution.rs b/container/crowdb-monitor/src/node/runtime/services/execution.rs index 5d4381c48..283a46a52 100644 --- a/container/crowdb-monitor/src/node/runtime/services/execution.rs +++ b/container/crowdb-monitor/src/node/runtime/services/execution.rs @@ -32,6 +32,23 @@ pub(in crate::node::runtime) async fn execute( profile: &DeploymentProfile, processes: &mut ProcessManager, intent: NodeServiceIntent, +) -> Result { + execute_with_readiness(profile, processes, intent, true).await +} + +pub(in crate::node::runtime) async fn reconcile( + profile: &DeploymentProfile, + processes: &mut ProcessManager, + intent: NodeServiceIntent, +) -> Result { + execute_with_readiness(profile, processes, intent, false).await +} + +async fn execute_with_readiness( + profile: &DeploymentProfile, + processes: &mut ProcessManager, + intent: NodeServiceIntent, + wait_initial_ready: bool, ) -> Result { validate(&intent)?; super::policy::validate(&profile.paths.data_root, &intent)?; @@ -93,7 +110,16 @@ pub(in crate::node::runtime) async fn execute( .await .map(|()| processes.pid(process_id)) } else { - apply(profile, processes, &binding, &intent, process_id, &directory).await + apply( + profile, + processes, + &binding, + &intent, + process_id, + &directory, + wait_initial_ready, + ) + .await }; let (complete, pid, error) = match &result { Ok(pid) => (true, *pid, None), @@ -169,6 +195,7 @@ async fn apply( intent: &NodeServiceIntent, process_id: &str, directory: &std::path::Path, + wait_initial_ready: bool, ) -> Result> { // Fence the admitted operation immediately before disrupting its service. // KV may provide the authority being read, so verify while it is still live. @@ -219,7 +246,13 @@ async fn apply( let pid = processes .start(&service, &std::collections::BTreeMap::new()) .await?; - wait_ready(profile, processes, intent).await?; + if wait_initial_ready { + wait_ready(profile, processes, intent).await?; + } else { + // Recovery observes once per pass. A live child retains its + // durable launch attempt while health and control make progress. + observe_ready(profile, intent).await?; + } Some(pid) } } else { diff --git a/container/crowdb-monitor/src/node/runtime/services/test_support.rs b/container/crowdb-monitor/src/node/runtime/services/test_support.rs index 6f72a17c1..00a83ad01 100644 --- a/container/crowdb-monitor/src/node/runtime/services/test_support.rs +++ b/container/crowdb-monitor/src/node/runtime/services/test_support.rs @@ -91,6 +91,13 @@ impl NodeServiceForTests { Ok(()) } + /// # Errors + /// Propagates failure to stop one fixture-owned managed child. + pub async fn stop_child_for_tests(&mut self, id: &str) -> Result<()> { + self.processes.stop(id, std::time::Duration::from_secs(1)).await?; + Ok(()) + } + /// # Errors /// Rejects superseded recovery observations before stopping an owned process. pub async fn stop_unhealthy_for_tests(&mut self, intent: &NodeServiceIntent) -> Result<()> { diff --git a/container/crowdb-monitor/src/process.rs b/container/crowdb-monitor/src/process.rs index b47f9758f..3d359ab1f 100644 --- a/container/crowdb-monitor/src/process.rs +++ b/container/crowdb-monitor/src/process.rs @@ -1,3 +1,4 @@ +mod exit_observation; pub(crate) mod log; mod retention; @@ -7,7 +8,6 @@ use std::path::PathBuf; use std::process::Stdio; use std::time::Duration; -use rustix::process::{kill_process, Pid, Signal}; use thiserror::Error; use tokio::process::{Child, Command}; use tokio::task::JoinHandle; @@ -30,6 +30,7 @@ pub enum ProcessError { } struct ManagedProcess { + spawn_pid: u32, child: Child, logger: JoinHandle>, } @@ -108,8 +109,14 @@ impl ProcessManager { let policy = self.log_policy.clone(); let logger = tokio::spawn(async move { Box::pin(log::pump(stdout, stderr, &log_directory, policy)).await }); - self.processes - .insert(service.id.clone(), ManagedProcess { child, logger }); + self.processes.insert( + service.id.clone(), + ManagedProcess { + spawn_pid: pid, + child, + logger, + }, + ); if let Err(error) = self .events .record(&MonitorEvent { @@ -198,24 +205,7 @@ impl ProcessManager { .processes .remove(id) .ok_or(ProcessError::Invalid("service is not owned"))?; - if process.child.try_wait()?.is_none() { - if let Some(pid) = process - .child - .id() - .and_then(|value| i32::try_from(value).ok()) - .and_then(Pid::from_raw) - { - if let Err(error) = kill_process(pid, Signal::TERM) { - if error != rustix::io::Errno::SRCH { - return Err(io::Error::from_raw_os_error(error.raw_os_error()).into()); - } - } - } - if timeout(grace, process.child.wait()).await.is_err() { - process.child.start_kill()?; - process.child.wait().await?; - } - } + let observation = exit_observation::stop(&mut process.child, process.spawn_pid, grace).await?; let log_result = timeout(Duration::from_secs(5), &mut process.logger).await; if let Ok(joined) = log_result { joined?.map_err(ProcessError::Io)?; @@ -223,6 +213,16 @@ impl ProcessManager { process.logger.abort(); return Err(ProcessError::Invalid("child log pipes did not close after exit")); } + if let Err(error) = self + .events + .record_process_exit(id, process.spawn_pid, &observation) + .await + { + eprintln!( + "process exit evidence was not retained: service={id} pid={} error={error}", + process.spawn_pid + ); + } self.events .record(&MonitorEvent { kind: MonitorEventKind::ChildStopped, diff --git a/container/crowdb-monitor/src/process/exit_observation.rs b/container/crowdb-monitor/src/process/exit_observation.rs new file mode 100644 index 000000000..c6ca809dc --- /dev/null +++ b/container/crowdb-monitor/src/process/exit_observation.rs @@ -0,0 +1,82 @@ +use std::io; +use std::os::unix::process::ExitStatusExt; +use std::process::ExitStatus; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use rustix::process::{kill_process, Pid, Signal}; +use tokio::process::Child; +use tokio::time::timeout; + +use super::ProcessError; + +pub(super) async fn stop( + child: &mut Child, + original_pid: u32, + grace: Duration, +) -> Result { + let mut status = child.try_wait()?; + let mut stop_path = "already_exited_before_stop"; + let mut term_sent_at_unix_ms = None; + let mut term_clock = None; + let mut kill_requested_at_unix_ms = None; + let mut wait_error = None; + if status.is_none() { + stop_path = "owned_stop_requested"; + if let Some(pid) = child + .id() + .and_then(|value| i32::try_from(value).ok()) + .and_then(Pid::from_raw) + { + let attempted_clock = Instant::now(); + let attempted_at = unix_ms(); + match kill_process(pid, Signal::TERM) { + Ok(()) => { + term_sent_at_unix_ms = attempted_at; + term_clock = Some(attempted_clock); + } + Err(error) if error == rustix::io::Errno::SRCH => {} + Err(error) => return Err(io::Error::from_raw_os_error(error.raw_os_error()).into()), + } + } + // No diagnostic I/O runs between TERM and this original grace wait. + match timeout(grace, child.wait()).await { + Ok(Ok(observed)) => status = Some(observed), + // The original stop path ignored this inner wait error. Retain + // that behavior, but do not manufacture a successful exit status. + Ok(Err(error)) => wait_error = Some(error.to_string()), + Err(_) => { + stop_path = "owned_stop_grace_elapsed"; + child.start_kill()?; + kill_requested_at_unix_ms = unix_ms(); + status = Some(child.wait().await?); + } + } + } + let wait_returned_at_unix_ms = unix_ms(); + let elapsed_since_term_ms = term_clock.map(|clock| clock.elapsed().as_millis()); + let observation = serde_json::json!({ + "original_pid": original_pid, + "stop_path": stop_path, + "status_observed": status.is_some(), + "exit_code": status.as_ref().and_then(ExitStatus::code), + "exit_signal": status.as_ref().and_then(ExitStatusExt::signal), + "core_dumped": status.as_ref().map(ExitStatusExt::core_dumped), + "success": status.as_ref().map(ExitStatus::success), + "raw_wait_status": status.as_ref().map(|value| (*value).into_raw()), + "term_sent_at_unix_ms": term_sent_at_unix_ms, + "kill_requested_at_unix_ms": kill_requested_at_unix_ms, + "wait_returned_at_unix_ms": wait_returned_at_unix_ms, + "elapsed_since_term_ms": elapsed_since_term_ms, + "grace_ms": grace.as_millis(), + "wait_error": wait_error, + "caller_stop_reason": "not supplied by existing ProcessManager::stop API", + }); + Ok(observation) +} + +fn unix_ms() -> Option { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .ok() + .map(|value| value.as_millis()) +} diff --git a/container/crowdb-monitor/tests/common/managed_health.rs b/container/crowdb-monitor/tests/common/managed_health.rs new file mode 100644 index 000000000..6773f0e3c --- /dev/null +++ b/container/crowdb-monitor/tests/common/managed_health.rs @@ -0,0 +1,180 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::node_setup; +use crowdb_monitor::{DeploymentProfile, NodeIdentity, NodeServiceForTests}; +use crowdb_protocol::mgmt::node::{ + setup::{NodeSetupChoices, NodeSetupRequest}, + NodeServiceAction, NodeServiceIntent, +}; +use std::{ + collections::BTreeMap, + os::unix::fs::PermissionsExt, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; + +pub struct TestManagedHealth { + pub worker: NodeServiceForTests, + pub ready: Arc, + pub slow: Arc, + pub intent: NodeServiceIntent, + pub profile: DeploymentProfile, + _authority: node_setup::TestAuthority, + server: tokio::task::JoinHandle<()>, + _directory: crowdb_test_harness::test_dirs::TestDir, +} + +impl TestManagedHealth { + pub async fn new() -> Self { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("managed-health"); + let data = directory.path().join("data"); + std::fs::create_dir(&data).unwrap(); + let root = data.as_path(); + let request = NodeSetupRequest { + version: 1, + discovery_id: NodeIdentity::load_or_create(root).unwrap().uuid().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + choices: NodeSetupChoices::default(), + }; + let authority = node_setup::TestAuthority::start(root, &request).await; + assert_eq!(authority.registry.nodes[0].node_id, 2); + let ready = Arc::new(AtomicBool::new(true)); + let slow = Arc::new(AtomicBool::new(false)); + let (address, server) = health_server(Arc::clone(&ready), Arc::clone(&slow)).await; + let mut profile = DeploymentProfile::load( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml"), + ) + .unwrap(); + profile.paths.data_root = root.to_path_buf(); + profile.paths.install_root = root.parent().unwrap().to_path_buf(); + profile.paths.bin_root = root.join("bin"); + profile.paths.template_root = root.join("templates"); + profile.paths.log_root = root.join("log"); + profile.paths.run_root = directory.path().join("run"); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + std::fs::create_dir_all(&profile.paths.run_root).unwrap(); + std::fs::create_dir_all(&profile.paths.bin_root).unwrap(); + let program = profile.paths.bin_root.join("owned-child"); + std::fs::write(&program, "#!/bin/sh\nexec sleep 60\n").unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap(); + for service in &mut profile.services { + service.program.clone_from(&program); + service.config_template = None; + if matches!(service.id.as_str(), "web" | "kv" | "chunkdb") { + service.probe.target = format!("http://{address}/health"); + service.additional_probes.clear(); + // Keep the service executor and real Group-0 authority; this + // fixture isolates readiness from storage dependency assembly. + service.dependencies.clear(); + } + } + profile.validate().unwrap(); + let intent = NodeServiceIntent { + cluster_id: authority.binding.bootstrap.cluster_id.clone(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: 2, + service_id: "chunkdb-2".into(), + kind: "chunkdb".into(), + action: NodeServiceAction::Start, + configuration: format!("[server]\nhttp_listen_addr = \"{address}\"\n"), + environment: BTreeMap::new(), + }; + authority + .kv + .put( + 0, + 0, + b"/deployment/services/2/chunkdb-2", + &serde_json::to_vec(&intent).unwrap(), + None, + ) + .await + .unwrap(); + let mut worker = NodeServiceForTests::new_for_tests(profile.clone()).await.unwrap(); + worker.start_kv_child_for_tests().await.unwrap(); + worker.execute_for_tests(intent.clone()).await.unwrap(); + Self { + worker, + ready, + slow, + intent, + profile, + _authority: authority, + server, + _directory: directory, + } + } + + pub fn execution(&self) -> serde_json::Value { + serde_json::from_slice( + &std::fs::read( + self.profile + .paths + .data_root + .join("services/chunkdb-2/execution.json"), + ) + .unwrap(), + ) + .unwrap() + } + + pub async fn stop(&mut self) { + for id in ["chunkdb-2", "kv", "web"] { + self.worker.stop_child_for_tests(id).await.unwrap(); + } + } +} + +impl Drop for TestManagedHealth { + fn drop(&mut self) { + self.server.abort(); + } +} + +async fn health_server( + ready: Arc, + slow: Arc, +) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let observed = Arc::clone(&ready); + let health_slow = Arc::clone(&slow); + let ready_slow = Arc::clone(&slow); + let server = tokio::spawn(async move { + let app = axum::Router::new() + .route( + "/health", + axum::routing::get(move || { + let slow = Arc::clone(&health_slow); + async move { + if slow.load(Ordering::Acquire) { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + "healthy" + } + }), + ) + .route( + "/ready", + axum::routing::get(move || { + let ready = Arc::clone(&observed); + let slow = Arc::clone(&ready_slow); + async move { + if slow.load(Ordering::Acquire) { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + if ready.load(Ordering::Acquire) { + axum::http::StatusCode::OK + } else { + axum::http::StatusCode::SERVICE_UNAVAILABLE + } + } + }), + ); + axum::serve(listener, app).await.unwrap(); + }); + (address, server) +} diff --git a/container/crowdb-monitor/tests/node_managed_health_test.rs b/container/crowdb-monitor/tests/node_managed_health_test.rs new file mode 100644 index 000000000..fd90b7a26 --- /dev/null +++ b/container/crowdb-monitor/tests/node_managed_health_test.rs @@ -0,0 +1,73 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/managed_health.rs"] +mod fixture; +#[path = "common/node_setup.rs"] +mod node_setup; + +use fixture::TestManagedHealth; +use std::{ + sync::atomic::Ordering, + time::{Duration, Instant}, +}; + +#[tokio::test] +async fn background_recovery_yields_while_a_new_managed_process_is_not_ready() { + let mut test = TestManagedHealth::new().await; + test.worker.stop_child_for_tests("chunkdb-2").await.unwrap(); + test.ready.store(false, Ordering::Release); + let started = Instant::now(); + let result = test.worker.reconcile_for_tests().await; + let elapsed = started.elapsed(); + let pending = test.execution(); + test.worker.tick_for_tests().await.unwrap(); + let unavailable = crowdb_monitor::StatusStore::open(&test.profile.paths.run_root) + .unwrap() + .read(Duration::from_secs(3)) + .unwrap(); + test.ready.store(true, Ordering::Release); + test.worker.reconcile_for_tests().await.unwrap(); + let recovered = test.execution(); + test.stop().await; + result.unwrap(); + assert!( + elapsed < Duration::from_secs(3), + "background recovery held health publication for {elapsed:?}" + ); + assert!(!pending["complete"].as_bool().unwrap()); + assert_eq!(pending["attempts"], 1); + assert!(!unavailable.services["chunkdb-2"].healthy); + assert_eq!( + unavailable.services["chunkdb-2"].pid, + pending["pid"].as_u64().map(|pid| u32::try_from(pid).unwrap()) + ); + assert!(recovered["complete"].as_bool().unwrap()); + assert_eq!(recovered["pid"], pending["pid"]); + assert_eq!(recovered["intent"]["operation_id"], test.intent.operation_id); +} + +#[tokio::test] +async fn slow_probes_share_one_health_pass_and_do_not_hide_real_failures() { + let mut test = TestManagedHealth::new().await; + test.slow.store(true, Ordering::Release); + let started = Instant::now(); + test.worker.tick_for_tests().await.unwrap(); + let elapsed = started.elapsed(); + let store = crowdb_monitor::StatusStore::open(&test.profile.paths.run_root).unwrap(); + let unavailable = store.read(Duration::from_secs(3)).unwrap(); + test.slow.store(false, Ordering::Release); + test.worker.tick_for_tests().await.unwrap(); + let recovered = store.read(Duration::from_secs(3)).unwrap(); + test.stop().await; + assert!( + elapsed < Duration::from_secs(2), + "probe deadlines accumulated: {elapsed:?}" + ); + assert_eq!(unavailable.services.len(), 3); + assert!(unavailable.services.values().all(|service| !service.healthy)); + assert!(recovered.services.values().all(|service| service.healthy)); + for (id, service) in &unavailable.services { + assert_eq!(service.pid, recovered.services[id].pid); + } +} diff --git a/container/crowdb-monitor/tests/node_service_readiness_test.rs b/container/crowdb-monitor/tests/node_service_readiness_test.rs index d67e92ed2..44be04856 100644 --- a/container/crowdb-monitor/tests/node_service_readiness_test.rs +++ b/container/crowdb-monitor/tests/node_service_readiness_test.rs @@ -254,6 +254,7 @@ async fn completed_service_preserves_live_recovery_and_restarts_failed_health() drop(listener); let unhealthy = root.join("unhealthy"); let invocations = root.join("invocations"); + let listening = root.join("listening"); let program = profile.paths.bin_root.join("managed-service"); std::fs::write( &program, @@ -288,12 +289,18 @@ class Handler(BaseHTTPRequestHandler): def log_message(self, *_args): pass -HTTPServer(("127.0.0.1", port), Handler).serve_forever() +server = HTTPServer(("127.0.0.1", port), Handler) +with open("{invocations}") as marker: + invocation = marker.read().strip() +with open("{listening}", "w") as marker: + marker.write(invocation) +server.serve_forever() PY "#, invocations = invocations.display(), unhealthy = unhealthy.display(), port = port, + listening = listening.display(), ), ) .unwrap(); @@ -362,8 +369,22 @@ PY let state: serde_json::Value = serde_json::from_slice(&std::fs::read(root.join("services/chunkdb-2/execution.json")).unwrap()) .unwrap(); - assert_eq!(state["complete"], true); assert_ne!(state["pid"].as_u64().unwrap(), first_pid); + // Background recovery yields while the new listener starts. Wait for the + // fixture's actual bind event, then observe the same child on its next pass. + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while std::fs::read_to_string(&listening).unwrap_or_default() != "2" { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("replacement fixture did not bind its listener"); + worker.reconcile_for_tests().await.unwrap(); + let ready: serde_json::Value = + serde_json::from_slice(&std::fs::read(root.join("services/chunkdb-2/execution.json")).unwrap()) + .unwrap(); + assert_eq!(ready["complete"], true); + assert_eq!(ready["pid"], state["pid"]); assert_eq!( std::fs::read_to_string(&invocations).unwrap().trim(), "2", diff --git a/container/crowdb-monitor/tests/process_test.rs b/container/crowdb-monitor/tests/process_test.rs index e871a4e59..cfa45d317 100644 --- a/container/crowdb-monitor/tests/process_test.rs +++ b/container/crowdb-monitor/tests/process_test.rs @@ -204,3 +204,142 @@ async fn child_restarts_prune_old_pid_logs_without_crossing_log_channels() { assert!(directory.join("operator-notes.txt").exists()); assert_eq!(fs::read_to_string(active).unwrap(), "new"); } + +fn exit_evidence(logs: &TestLogs, pid: u32) -> serde_json::Value { + let body = fs::read_to_string(logs.0.join("monitor/monitor.log")).unwrap(); + let rows = body + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .filter(|row| row["kind"] == "child_exit_observed" && row["pid"] == pid) + .collect::>(); + assert_eq!(rows.len(), 1); + assert_eq!( + rows[0]["level"], + if rows[0]["process_exit"]["success"] == true { + "info" + } else { + "warn" + } + ); + assert_eq!(rows[0]["process_exit"]["original_pid"], pid); + rows[0]["process_exit"].clone() +} + +async fn wait_finished(manager: &mut ProcessManager) { + tokio::time::timeout(Duration::from_secs(2), async { + while manager.exit_status("fake").unwrap().is_none() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); +} + +async fn wait_marker(logs: &TestLogs) { + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if fs::read_to_string(logs.0.join("fake/service.log")).is_ok_and(|text| text.contains("ready")) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn completed_child_evidence_preserves_original_pid_and_real_exit_codes() { + for code in [0, 7] { + let logs = TestLogs::new(); + let mut manager = ProcessManager::new(logs.0.clone(), policy(2)).await.unwrap(); + let pid = manager + .start(&service(&format!("exit {code}")), &BTreeMap::new()) + .await + .unwrap(); + wait_finished(&mut manager).await; + assert_eq!(manager.pid("fake"), None); + manager.stop("fake", Duration::from_secs(2)).await.unwrap(); + let row = exit_evidence(&logs, pid); + assert_eq!(row["status_observed"], true); + assert_eq!(row["exit_code"], code); + assert_eq!(row["success"], code == 0); + assert!(row["exit_signal"].is_null()); + assert!(row["term_sent_at_unix_ms"].is_null()); + assert!(row["kill_requested_at_unix_ms"].is_null()); + assert_eq!(row["stop_path"], "already_exited_before_stop"); + } +} + +#[tokio::test] +async fn owned_term_stop_records_the_real_signal_without_success() { + let logs = TestLogs::new(); + let mut manager = ProcessManager::new(logs.0.clone(), policy(2)).await.unwrap(); + let pid = manager + .start(&service("printf ready; exec sleep 30"), &BTreeMap::new()) + .await + .unwrap(); + wait_marker(&logs).await; + manager.stop("fake", Duration::from_secs(2)).await.unwrap(); + let row = exit_evidence(&logs, pid); + assert_eq!(row["exit_signal"], 15); + assert_eq!(row["success"], false); + assert!(row["exit_code"].is_null()); + assert!(row["term_sent_at_unix_ms"].is_number()); + assert!(row["kill_requested_at_unix_ms"].is_null()); + assert_eq!(row["stop_path"], "owned_stop_requested"); + assert_eq!(row["grace_ms"], 2000); +} + +#[tokio::test] +async fn ignored_term_retains_original_grace_and_reports_kill_escalation() { + let logs = TestLogs::new(); + let mut manager = ProcessManager::new(logs.0.clone(), policy(2)).await.unwrap(); + let pid = manager + .start( + &service("trap '' TERM; printf ready; exec sleep 30"), + &BTreeMap::new(), + ) + .await + .unwrap(); + wait_marker(&logs).await; + manager.stop("fake", Duration::from_millis(30)).await.unwrap(); + let row = exit_evidence(&logs, pid); + assert_eq!(row["exit_signal"], 9); + assert_eq!(row["success"], false); + assert!(row["exit_code"].is_null()); + assert!(row["term_sent_at_unix_ms"].is_number()); + assert!(row["kill_requested_at_unix_ms"].is_number()); + assert_eq!(row["stop_path"], "owned_stop_grace_elapsed"); + assert_eq!(row["grace_ms"], 30); + assert!(row["elapsed_since_term_ms"].as_u64().unwrap() >= 30); +} + +#[tokio::test] +async fn self_signaled_exit_is_not_mislabeled_as_supervisor_kill() { + let logs = TestLogs::new(); + let mut manager = ProcessManager::new(logs.0.clone(), policy(2)).await.unwrap(); + let pid = manager + .start(&service("kill -KILL $$"), &BTreeMap::new()) + .await + .unwrap(); + wait_finished(&mut manager).await; + manager.stop("fake", Duration::from_secs(2)).await.unwrap(); + let row = exit_evidence(&logs, pid); + assert_eq!(row["exit_signal"], 9); + assert_eq!(row["success"], false); + assert!(row["exit_code"].is_null()); + assert!(row["term_sent_at_unix_ms"].is_null()); + assert!(row["kill_requested_at_unix_ms"].is_null()); + assert_eq!(row["stop_path"], "already_exited_before_stop"); +} + +#[tokio::test] +async fn unowned_stop_does_not_fabricate_exit_evidence() { + let logs = TestLogs::new(); + let mut manager = ProcessManager::new(logs.0.clone(), policy(2)).await.unwrap(); + assert!(manager.stop("missing", Duration::from_secs(2)).await.is_err()); + let body = fs::read_to_string(logs.0.join("monitor/monitor.log")).unwrap(); + assert!(!body.contains("child_exit_observed")); + assert!(!body.contains("\"success\":true")); +} diff --git a/doc/design/access-server/design-crowdb-access-server.md b/doc/design/access-server/design-crowdb-access-server.md index f49b30508..bfe282e31 100644 --- a/doc/design/access-server/design-crowdb-access-server.md +++ b/doc/design/access-server/design-crowdb-access-server.md @@ -214,9 +214,26 @@ address; a missing address requires deployment reconciliation before restart. One Access background sampler runs every second and publishes an immutable health snapshot through ArcSwap. It independently checks the routed Chunk-KV bucket namespace, ChunkDB and online disks through the process's storage clients. -Metadata and chunk checks have separate deadlines inside an 800 ms total bound; -only one sampling round can run at a time. Physical checks read existing disk +Metadata and chunk checks run concurrently with a two-second dependency budget; +hardware discovery, ChunkDB and disk headers share that chunk-check budget. A +2100 ms outer watchdog bounds the full round and cancels unfinished work. Only +one sampling round can run at a time; missed one-second ticks are skipped rather +than overlapping or replayed. A failed round publishes unavailable evidence and +logs its generation, configured budgets, elapsed time and dependency errors. +Sampling continues after every failure, and a later successful generation +recovers readiness without a restart. Failure never extends the probe budget, +adds a retry or changes the response to Ready. Physical checks read existing disk bytes without creating objects, allocating chunks or repairing storage. +ChunkDB point checks use the validated service route and actual owner/storage +reads. They do not initiate an ordinary periodic full registry refresh before +the query; cold discovery and failed-route refresh retain the existing rules +and shared deadline. A cached route never substitutes for a current response. +Each physical check reads a 4096-byte disk header through the storage client's +bounded priority connection group. It retains the same authoritative disk route +and shares the chunk check's total deadline. Bulk response bytes on a normal +connection cannot delay that header through TCP response ordering. A rejected, +short or late header still fails the observation; the priority lane supplies no +additional retry or grace period. Credential-cache freshness, listener state, native-pool pressure and cleanup backlog are included in the same snapshot. Application requests only record their own errors and metrics; their outcomes cannot overwrite health evidence. @@ -224,8 +241,11 @@ their own errors and metrics; their outcomes cannot overwrite health evidence. Health requests read the snapshot without dependency I/O or a new sampling task. Responses retain the check timestamp, generation, age, exact flags and diagnostic. New observations replace old failures or successes. Evidence older than three -seconds, or an absent first sample, reports Unknown and HTTP 503; readers never -extend its validity or clear it to Ready. Process stop is an immediate lifecycle +seconds from the sampling start, or an absent first sample, reports Unknown and +HTTP 503; readers never extend its validity or clear it to Ready. Completion +does not renew the start timestamp. Consecutive slow successful rounds can leave +an expired-evidence interval before the next round completes; retain that 503 +and recover only when a new current generation is published. Process stop is an immediate lifecycle fence even when the cached sample was healthy. The sampler is cancelled with the listener. Recovery needs neither application traffic nor an Access restart. diff --git a/doc/design/chunkdb/design-crowdb-chunkdb-range-binding.md b/doc/design/chunkdb/design-crowdb-chunkdb-range-binding.md index 132a1234a..dc32b404e 100644 --- a/doc/design/chunkdb/design-crowdb-chunkdb-range-binding.md +++ b/doc/design/chunkdb/design-crowdb-chunkdb-range-binding.md @@ -82,6 +82,12 @@ and interleaves slots among storage groups; this assignment is initial policy, not the definition of a slot. Readiness requires initialized maps and valid storage routing. An instance with zero service slots is ready but rejects work outside its empty bitmap; local disks are not a service-readiness requirement. +HTTP readiness reads the independently sampled ownership result without issuing +Group-0 requests. One worker refreshes the complete map every second, with a +three-second read deadline and no overlapping refreshes. Evidence ages from the +start of that authority read. Pending, failed or older-than-three-second evidence +returns unavailable; HTTP readers cannot renew it. This observation does not +replace per-slot execution admission or weaken map validation. The fixed-layout monitor audits ownership and does not reassign slots when a heartbeat expires. Legacy assignment writers reject updates. @@ -110,11 +116,25 @@ Explicit `NotMyRange` permits bounded refresh and retry. Unknown allocation or mutation outcomes return to the caller without blind resubmission; read-only queries can refresh and retry. No original-owner fallback is supported. +Readiness sends real point queries through the initialized service route without +first running the ordinary periodic Group0 registry/map refresh. Cold discovery +still requires a validated map; ownership rejection or transport failure retains +the existing read-only refresh/retry path and caller deadline. The receiving +owner validates admission and reads its actual KV storage. This keeps a bounded +probe independent of an unrelated full registry refresh without caching a +successful result or changing ordinary requests' freshness rules. + +The owner's KV transport isolates Get responses from background scans with one +additional connection per endpoint and the same RPC workers and completion +budget. The point request still reads current storage and retains its original +read mode, error handling and deadline. A partially received catalog page must +not hold the receiving owner's point-read response behind that page's bytes. + ## 5. Maintenance authority One process hosts separate system and user-data lifecycle/task runtimes. -Wal, BtreePage, and PageIndex are system purposes; Stream, S3, and IcebergTable -are user-data purposes. Each runtime has its own scan scope, claims, queue, +Wal, BtreePage, and PageIndex are system purposes; Stream, S3, IcebergTable, +and Dataset are user-data purposes. Each runtime has its own scan scope, claims, queue, execution capacity, and recovery. Shared transport, codecs, IO, and algorithms do not create shared task authority. @@ -126,6 +146,13 @@ domains at read, admission, claim, and publication boundaries. Execution verifie the durable claim before handler side effects; claim generations and writer epochs fence delayed completion after same-owner restart. +Chunk and reservation metadata retain their existing ChunkId-ordered keys. +Scoped scans pass the domain's half-open purpose-byte bounds to KV before +transferring values: system types 1 through 3 and user-data types 4 through 7. +Continuation stays within those bounds; ownership filtering and fixed-cutoff +pagination remain in effect. Growing an unrelated domain must not add pages +to the scoped read. Unscoped inventory retains the complete keyspace. + The reservation budget is apportioned by service slot share and split equally between system and user-data runtimes. Execution limits are independently reserved; shared reservation gauges aggregate domain contributions. Domains can diff --git a/doc/design/chunkds/design-crowdb-chunk-kv-server.md b/doc/design/chunkds/design-crowdb-chunk-kv-server.md index b1eb070ab..dc1152802 100644 --- a/doc/design/chunkds/design-crowdb-chunk-kv-server.md +++ b/doc/design/chunkds/design-crowdb-chunk-kv-server.md @@ -122,6 +122,12 @@ and atomically preserves its sequence and greatest expiry alongside the grant. Replacement, renewal and grant publication therefore serialize on one record; replacement cannot miss a concurrent extension. An observation for another process epoch cannot receive a grant, and a boot rejects grants for other epochs. +Grant publication attempts each independently ready instance once per pass in +instance-ID order. A failed instance publication does not skip other instances; +the pass reports all publication failures after processing them. Each publication +retains its process-identity, revision, expiry and atomic lease/grant checks. +Invalid shared catalog, readiness or exclusion evidence still rejects the pass +before instance publication. No conditional retry or authority extension is added. One bounded control-plane task submits a boot's renewal and discovery writes sequentially so their periodic ticks cannot exhaust each other's conditional admission budget. Recovery, data IO and load sampling remain outside that task. @@ -543,6 +549,15 @@ is required. A retained marker from a previous completed split does not block this publication when the source has no inherited overlay. A conflicting range, owner, epoch, artifact or reused operation identity still fails closed. +For a recovered target without an inherited overlay, normal placement cleanup +may remove its completed transition marker before its first root checkpoint. +The already prepared root handle may then claim publication authority from the +exact current `Serving` partition, range, owner, epoch, tree and stream, with +both assignments free of overlays. Marker removal alone cannot invalidate that +published ownership. A different nonempty transition, another assignment, or a +catching-up entry still cannot authorize publication through this exception; +root authority continues to fence lower owner epochs. + Recovery uses persisted evidence only. Before source release, a target failure leaves the source serving and target preparation is repeatable. After release, abort is forbidden: catalog reread decides whether to publish or resume @@ -610,7 +625,7 @@ a resident index separator or a bounded small key window; exact byte or key medians are unnecessary. Live-key witnesses establish nonempty children when the observation is made. Concurrent mutations can change that observation. The structural query never loads cold pages or flushes the tree. A service -allows only one background sampling job, and heartbeats use epoch-fenced cached +owns at most one background sampling job, and heartbeats use epoch-fenced cached samples without waiting for that job. Every serving partition is considered; an unsplittable largest partition does not exclude the remaining candidates. Retained pack estimates use the opened manifest, including shared packs, rather @@ -709,6 +724,16 @@ through the stream's current durable tail, reports readiness, and serves only after installing a matching grant. Shutdown atomically stops new admission and clears authority; already admitted R142 operations retain handles and finish before bounded checkpoint/drain work. + +`begin_drain` closes sampling admission before background workers stop. Before +draining hosted WAL writers, `drain_writers` waits for the sampling job's partition +payload and captured Tokio `Handle` to finish teardown while the driver remains +alive, including when queued work is discarded or the job unwinds. Closed sampling +cannot relaunch; completed drain clears its cache and permits no further publication. +This barrier owns sampling only. OS I/O remains uncancellable, and the existing +10-second monitor and 15-second console stop grace periods remain unchanged; +forced termination retains normal writer-lease exclusion. + Catalog refresh recovers every new or changed local assignment first, then replaces the catalog and hosted-partition snapshots; unchanged exact-epoch handles remain live and departed assignments are dropped. diff --git a/doc/design/console/design-crowdb-console-ui.md b/doc/design/console/design-crowdb-console-ui.md index 023698ce9..09aa7cc71 100644 --- a/doc/design/console/design-crowdb-console-ui.md +++ b/doc/design/console/design-crowdb-console-ui.md @@ -268,10 +268,13 @@ remain subject to the resource capabilities and lifecycle rules in the text. budget per service operation; successful readiness clears that budget. - Legacy standalone Add Node retains its one-dialog registration and saved six-service plan. Its session reconciler saves revision-checked progress before - deployment, resumes waiting steps while that Console session is active, and - requires explicit retry for interrupted deployment. It never duplicates a - registered Node or successful service. This session-owned plan is distinct - from the standard target monitor's accepted node setup. + deployment and resumes waiting steps while that Console session is active. + Unavailable process observations and failed steps continue read-only observation + every two seconds, as defined in §13. A fresh, identity-matched running + observation reconciles interrupted progress to deployed. Explicit Retry is + required to initiate another failed Start, Restart, or Deploy mutation. It never + duplicates a registered Node or successful service. This session-owned plan is + distinct from the standard target monitor's accepted node setup. - Internal service health probes the RPC endpoint independently for each service. Startup records lacking RPC endpoints use process observation. Before its first store exists, PKV reserves but does not bind its RPC port; @@ -591,9 +594,55 @@ resource menu. Data browsing is read-only; Query/Scan remain center query tools. Missing or conflicting recorded identity fails explicitly. - Service failures name the failing step and permit a safe retry. Existing instances are not silently duplicated or treated as healthy merely by presence. +- A registered service whose current process observation is unavailable remains + waiting while the session observes it every two seconds. Missing, expired, + rejected, or failed observations do not prove that the service stopped. The + plan retains its registration and successful siblings without sending another + Start, Restart, or Deploy request. A fresh, identity-matched running observation + reconciles the step to deployed. Endpoint health remains independently reported + and never supplies a missing PID or proves process absence. +- A fresh, identity-matched observation that confirms a registered service is + stopped marks the step failed when no accepted startup is pending. Failed steps + continue read-only observation every two seconds, including when no other + deployment is queued. A fresh, identity-matched running process automatically + reconciles that registered service's progress to deployed. This also reconciles + an interrupted deployment whose owned process is later confirmed running. +- Read-only recovery never sends Start, Restart, or Deploy again. Missing or + unverified process evidence does not clear a recorded failure. Explicit Retry + is required to initiate another failed lifecycle mutation; an accepted startup + retains its existing operation and launch budget. Error observation does not + use the missing-Capacity backoff. +- An unavailable service-inventory read preserves successful steps, recorded + failures, and fixed-slot warnings. Unattempted queued steps keep waiting for + current inventory instead of becoming failed lifecycle operations. - Operational type descriptions remain available even though instance labels use the compact convention in §4. +Session-owned Node service plan with unavailable process observation +(relevant progress rows and existing controls): + +```text ++--------------------------------------------------------------------+ +| Node 1 services | +| | +| crowdb-access-server deployed | +| crowdb-chunk-kv deployed | +| crowdb-chunk-db deployed | +| crowdb-disk-db deployed | +| crowdb-disk-io deployed | +| crowdb-paxos-kv queued · waiting | +| Waiting: current process observation is unavailable | +| | +| [Cancel] [Done] | ++--------------------------------------------------------------------+ +``` + +Existing dialog guidance remains unchanged. Done closes progress; observation +continues while this Console page is active. Confirmed stopped with no pending +startup becomes failed and continues observation. A later fresh running process +reconciles progress to deployed automatically without a lifecycle request. The +existing Retry failed services action initiates a needed lifecycle mutation. + ## 14. DiskGroup and disk management - **DISK-01:** The physical tree ends at Disk: Rack → Node → Storage → DiskGroup @@ -1493,7 +1542,12 @@ CRUD merely to match another domain. Dataset lifecycle support is defined in §2 at 10/30 s retain slow/very-slow reports. Compare per-test baselines; investigate more than 2× rather than hiding time with retries or larger deadlines. - **TEST-05:** One worker owns the mutable runtime; installed system browser, - no automatic browser installation. Default assertion deadline is 3 s and + no automatic browser installation. Auxiliary-service dependency convergence + may wait up to 10 s for Chunk-KV range activation and matching catalog/grant + authority. After 3 s it records the latest readiness response; it also records + the final response and elapsed time. This prerequisite does not change UI + response or process-observation assertion budgets. + Default assertion deadline is 3 s and election 10 s, polling every 100 ms. No fixed sleeps. Native large multipart and SF=1 loader acceptance run separately from ordinary UI iteration; normal smoke data remain small while preserving multi-Strip and pagination cases. diff --git a/doc/design/deploy/design-crowdb-deploy.md b/doc/design/deploy/design-crowdb-deploy.md index 879bc273b..18038b3c5 100644 --- a/doc/design/deploy/design-crowdb-deploy.md +++ b/doc/design/deploy/design-crowdb-deploy.md @@ -332,6 +332,14 @@ disk preparation with dependent service startup; DiskIO may start before Online because its ownership proof enables Online. Explicit later service stops or group retirement supersede historical setup and prevent automatic recreation. +Background service recovery observes readiness once per pass, retaining the +same live child and durable launch attempt until readiness succeeds. Explicit +service commands retain their bounded initial startup wait. Local read-only +health probes run concurrently with their individual deadlines, and publication +rechecks actual owned process liveness and PID before reporting health. Slow +startup or several unavailable listeners must remain visible as unhealthy; +publishing a fresh timestamp cannot substitute for those observations. + The versioned deployment policy is durable and independent of UI permissions. Local and Production inputs cannot replace one another through an environment change. Initial voter count, storage, listeners and scope are sealed before node @@ -370,6 +378,15 @@ SSH connection/authentication has a ten-second bound; bidirectional proof has a twenty-second bound. Passwords are discarded, sessions are reused during setup, and only public keys cross nodes. +Each Web process sampler owns one read-only SSH connection per bound remote +node. Process observations retain their one-second cadence and three-second +deadline; connection reuse does not extend observation freshness. The sampler +checks the returned node and complete bootstrap identity on every response. +Connections are discarded after a command error, cancelled read, binding or +node configuration change, and are reauthenticated at least once per minute. +They use the provisioned node key and existing strict host-key verification, +without password fallback or sharing a session with management mutations. + ### 6.4 Authenticated member address recovery Node identity survives changes to management and data addresses. Each accepted diff --git a/doc/design/diskio/design-crowdb-diskio.md b/doc/design/diskio/design-crowdb-diskio.md index 5abbf20fd..a69986ab2 100644 --- a/doc/design/diskio/design-crowdb-diskio.md +++ b/doc/design/diskio/design-crowdb-diskio.md @@ -754,6 +754,12 @@ client's references, while the final in-flight reference controls when its C++ connection is closed. Selection and route/status snapshots use immutable RCU state and atomics, so request routing does not acquire a caller lock. +Readiness uses the priority group for bounded 4096-byte reads of existing disk +headers. Normal bulk responses and readiness headers therefore have independent +TCP response ordering while sharing the same validated disk identity and route. +Priority selection does not change physical read semantics, error propagation, +route refresh rules or the caller's total deadline. + The internal wire transport retains the caller's owned `Bytes` allocation in the RPC buffer until completion. This avoids a client-side `Bytes` to `Vec` payload copy. On the server, block-size-one and already aligned payloads can diff --git a/doc/design/kv/design-crowdb-kv-group0.md b/doc/design/kv/design-crowdb-kv-group0.md index 0c75815f4..7d02c0e18 100644 --- a/doc/design/kv/design-crowdb-kv-group0.md +++ b/doc/design/kv/design-crowdb-kv-group0.md @@ -396,6 +396,16 @@ Vec`; kv-server carries `hosted_stores`, `hosted_groups`, (`register_diskdb`/`heartbeat_diskdb`/`read_all_diskdb_instances`) and kv-server wrappers delegate to the generic methods. +DiskIO physical identity and configured hardware scope are immutable once +accepted. Physical identity is pinned with instance-revision CAS before scope +publication. The first configured scope requires an existing Rack record and +is published atomically with an advance of that Rack's revision, fencing +concurrent disk membership changes. Revalidate the instance after reading the +Rack revision so another first-scope publication cannot be overwritten. If the +Rack is absent, retain the physical pin and reject scope publication; an +instance-only write cannot substitute for the Rack fence. Ordinary heartbeats +with the same accepted scope update liveness without advancing the Rack. + ### 4.2 Services registered - **diskdb** — `/srv/diskdb/`. Heartbeat includes diff --git a/doc/design/kv/design-crowdb-kv-rpc-client.md b/doc/design/kv/design-crowdb-kv-rpc-client.md index d00178c22..21b0ca707 100644 --- a/doc/design/kv/design-crowdb-kv-rpc-client.md +++ b/doc/design/kv/design-crowdb-kv-rpc-client.md @@ -179,6 +179,17 @@ source. `conn_for` derives the client-facing crowdb-rpc port from the base port via the `KV_CLIENT_RPC_BASE` offset. Pool selection returns its generation, and retryable errors invalidate only that generation. +Callers with concurrent bulk scans and latency-bounded point reads may opt +into `with_isolated_point_reads`, or construct `CrowdbKvClient` with +`new_with_isolated_point_reads`. Get replies then use one additional connection +per endpoint, sharing the existing RPC workers, request-ID source, completion +pool, tunables and deadlines. Scans and mutations retain their ordinary pool. +Errors invalidate the selected pool's exact generation; explicit reconnect +clears both pools. The default constructor retains its existing connection +behavior. ChunkDB enables this mode so actual metadata point reads do not wait +behind an incomplete background scan response. Isolation does not cache success +or convert rejection, transport failure or an expired caller deadline to Ready. + Unary methods (`send_put`, `send_get`, `send_delete`, `send_batch_write`, `send_scan`, `send_journal_scan`): build request flatbuffer → `rpc.call` → await → parse via `Ref` wrapper → map to the existing @@ -263,6 +274,15 @@ election wait and topology refresh rather than issuing an unbounded RPC loop. Scan and journal continuations reset this redirect history after a successful page. Conditional writes retain their original mutation identity on retries. +After a transport failure or leadership rejection without a usable hint, +discovery searches the target group across management seeds. A reachable seed +that repeats the rejected RPC endpoint cannot hide another member's replacement +route. Concurrent refreshes share the existing refresh gate and interval; a +newer cached route is reused without another fetch. If no replacement is known, +the operation retains its original bounded election wait/backoff and retry +budget. Discovery never establishes authority for a mutation; the destination +still applies its normal leadership and conditional-write checks. + `TopologyCache` publishes one immutable `TopologySnapshot`. Every group route contains its leader, replica endpoints, read cursor, endpoint statistics, and write-slot high-watermark. A refresh builds a complete replacement snapshot; diff --git a/doc/design/kv/design-crowdb-kv-wal.md b/doc/design/kv/design-crowdb-kv-wal.md index 3308aa8ad..b49b67819 100644 --- a/doc/design/kv/design-crowdb-kv-wal.md +++ b/doc/design/kv/design-crowdb-kv-wal.md @@ -177,7 +177,10 @@ Each individual `Accept` is small (tens of bytes overhead + payload). Per-record The durable operation is backend-specific: - **Filesystem file backend:** append with normal file writes, then `fdatasync` / equivalent durable flush. -- **Block-device backend:** issue aligned direct writes. If the device requires 4 KiB sectors, the WAL writer pads or read-modify-writes through the block backend's alignment planner; no filesystem `fdatasync` is involved. +- **Block-device backend:** issue aligned positional writes, using direct I/O + when selected. If the device requires 4 KiB sectors, the WAL writer pads or + read-modify-writes through the alignment planner. The real file-backed + implementation then calls `sync_data` before releasing durable acknowledgments. - **RAM / SCM / simulated backends:** use the backend's own durability contract. A volatile RAM backend is for tests only and may complete immediately; persistent-memory backends may require cache-line flush / fence semantics instead of filesystem sync. The acceptor code talks to `WalEngine::append_*` and does not branch on these backend details. @@ -249,6 +252,20 @@ enum with four variants: block device, or a tmpfs file makes no difference; in Unix, a block device *is* a file. +On the production multi-thread Tokio runtime, including a runtime with one +worker, synchronous block-device file and layout operations use Tokio's +[`block_in_place`](https://docs.rs/tokio/1.52.3/tokio/task/fn.block_in_place.html). +This allows independently spawned apply, RPC, and timer tasks +to run while the OS call blocks. The closure completes before returning, so +borrowed WAL buffers and the file remain live without an extra payload copy. +Write, durable flush, index publication, and acknowledgment keep their original +ordering and error behavior. This changes scheduling rather than durability, +timeouts, retries, or configured worker/resource limits. Current-thread and +non-runtime callers retain synchronous compatibility; they do not gain +concurrent task progress. The production path uses independently spawned WAL +workers on the ordinary multi-thread runtime, not a `LocalSet` or same-task +`join!` branches. + `BlockDevice` has two constructors selected by the `O_DIRECT` flag: - `BlockDevice::new()` — aligned (4K), buffered (no `O_DIRECT`). Default for diff --git a/doc/design/tree/design-crowdb-tree-chunk-storage.md b/doc/design/tree/design-crowdb-tree-chunk-storage.md index c81fb9c2c..7960713ff 100644 --- a/doc/design/tree/design-crowdb-tree-chunk-storage.md +++ b/doc/design/tree/design-crowdb-tree-chunk-storage.md @@ -167,6 +167,15 @@ generation and ownership epoch. Only that compare-and-publish operation makes the checkpoint visible. Stale, failed, or ambiguous work remains unreachable and is accounted as orphan data. +Once a tree has written an anchor, a failed final durability barrier leaves +that anchor pending under the existing snapshot serialization gate. Before +preparing another snapshot, synchronous and asynchronous callers retry the +pending barrier over the same staged image. Continued unavailability remains +a typed failure; no new snapshot data is written until the pending barrier +succeeds. Writes may continue into the live tree and are captured by the next +snapshot. Failed publication never advances the tree's durable snapshot or WAL +retention frontier. + Recovery normally opens the newest epoch-valid, fully verified manifest. It may fall back only to the preceding complete retained generation. A caller holding topology recovery evidence may instead request one exact immutable diff --git a/doc/dev/physical-node-test-plan.md b/doc/dev/physical-node-test-plan.md index 003ffd9c2..b217a7c43 100644 --- a/doc/dev/physical-node-test-plan.md +++ b/doc/dev/physical-node-test-plan.md @@ -3,352 +3,703 @@ # Physical Node Test Plan -这是可复用的物理测试用例目录。按“大类 → 用例 → 场景”组织，不记录本轮 -完成状态。每轮选择的场景、实际命令、源码/镜像身份和结果保存在独立 run -目录；历史 PASS 不能直接归给新镜像。 - -上游：[部署与恢复](../design/deploy/design-crowdb-deploy.md)、 -[OCI 构建与运行](../../container/oci-image/README.md)、 -[容器验收](../../e2e-test/README.md)、 -[KV 恢复](../design/kv/design-crowdb-kv-slot.md)。 - -## 1. 执行约定 - -### 环境与范围 - -- 默认两个物理故障域：A `192.168.31.129`，B `192.168.31.184`。 - 每轮重新核对主机身份、接口和地址；IP 不是资源归属证明。 -- A1/A2/A3、B1/B2 共五个固定 voter，分别使用独立持久卷和发现身份。 - A 的端口 offset 为 0/1/2，B 为 0/1；按当前启动接口核对实际接受值。 -- 每个容器固定 host network、1 CPU、2048 MiB、512 PIDs；每个服务一个 - Tokio worker。记录实际 cgroup 限制、存储容量、预留量和偏差。 -- 仅通过停止本轮指定主机上的容器模拟主机故障。不改变共享网络、路由、 - 防火墙、内核或 Docker daemon。三节点变体必须另立 profile，不能代替五节点。 -- 使用现有 SSH key 和非交互 BatchMode。凭据通过私有输入提供，不把密码、 - 私钥、token、登录身份或带认证参数的命令写入提交的文档/报告。 -- 所有构建、测试和可执行程序，包括远端命令，都通过 `pixi run`。 - -### 用例范围与结果 - -- P00–P19 保留为父用例编号；`.1`、`.2` 等是可分别核验的子场景。 - 默认主流程覆盖全部父用例及其普通场景。明确标为“扩展”的场景仅在所选 - profile 中要求执行；未选择不表示它已经通过。 -- 启动前冻结用例范围、并发、资源、数据规模、超时和验收条件。新增回归单独 - 记录；不能在失败后删去场景、放宽断言或改 profile 来取得 all-pass。 -- 每个子场景记录前置条件、注入对象/时刻、实际动作、断言、耗时、结果、证据 - 路径和输入身份。结果使用 `PASS / FAIL / NOT RUN`，未触发的注入计覆盖缺口。 -- 按依赖顺序运行，首错停止依赖它的后续工作。保留原失败和已通过结果；修复后 - 只重跑输入或行为受影响的场景，再继续未完成项。不要盲目重放部分执行的阶段。 -- mutation 在调用前落盘 operation/key/ID 和预期内容，成功后落盘 acknowledgment。 - 超时或断连记为 `OutcomeUnknown`，只读查证；测试控制器不得重放不确定写入。 -- 不增加 timeout，不削弱校验，不添加调用方重试。P14 仅观察生产 client 原有 - 明确安全重试策略，不为测试改变它。 -- 构建先于消费者；同一输出目录不并发构建。源码冻结后构建 OCI，保留 source - tree、commit/dirty diff、客户端摘要和镜像摘要。源码变更后重新判断受影响范围。 -- CI 与物理结果分别记录。用户要求不检测 CI 时，不查询、不推断新 run，也不把 - 等待 CI 作为继续物理测试的前置；本地受影响门禁仍须完成。 - -### 所有恢复用例共同遵守的断言 - -- 故障前保存容器 ID、卷名/CreatedAt、发现 UUID、node ID、bootstrap binding、 - 35 个服务的 container/PID/start-time 身份、catalog 和副本状态。 -- 连续观察覆盖“注入 → 恢复 → 完整数据审计”。完整采样间隔不超过五秒； - 单次缺少 `/proc` 的故障目标仍记录为 absent，不能丢掉整次兄弟服务样本。 -- 只允许该场景指定的进程变化。故障恢复期间，仍然存活的 monitor 应持续提供 - 新鲜的真实观察；故障目标离线可以不可观察，不能把旧 Ready 当作当前证据。 - 观察新鲜度和服务健康是两个断言，Unavailable 不能靠刷新旧健康时间戳伪装。 -- 恢复门禁必须取得所有节点当前、健康、晚于本容器启动且接收时不超过三秒的 - monitor 观察。此时固定全部 35 个进程身份，直到完整审计结束；不能再次设置 - 基线来容忍第二次重启。 -- 分别测量 catalog 接管 180 秒、服务恢复 180 秒、Groups 0/1/3 每组固定 - checkpoint catch-up 180 秒。每个门禁有自己的起点和耗时，不用整个阶段耗时 - 代替。P10 的 60 秒、P13 的 30 秒及生产 client 的更短限制另行保留。 -- 恢复后核对 P03 基线、P04/P05 数据、P07 已确认存活/删除记录和所有历史 - witness，并验证本场景的新写入。经全部五个恢复后的入口读取，完整校验内容。 -- EC 的实际 fragment 分布决定 payload 可用性；KV quorum 不能替代它。临时 - 超出丢失预算时的可用性结论，与恢复后的完整历史数据正确性分别报告。 - -## 2. 大类 A：主机、镜像与集群形成 - -### P00 环境、认证与镜像 - -- **P00.1 主机与资源**：验证两台真实主机、SSH key、时钟、接口、跨主机 - listener 可达性、Docker/cgroup、空闲磁盘和内存。清点已有资源的归属，记录 - 五节点限制与存储预算；准备不足即失败，不静默缩小数据集。 -- **P00.2 同一镜像**：两台主机导入同一已校验的镜像传输文件。校验传输摘要、 - manifest 引用的 config 内容哈希、平台、源码/runtime labels 和有序 rootfs - layers。Docker 可能分别显示 manifest ID 和 config ID，不能仅比较 ID 字符串。 -- **P00.3 拒绝错误输入**：离线回归覆盖损坏传输文件、源码/runtime label 不匹配、 - layer 顺序不一致和架构不匹配；失败时不能开始升级节点。 -- **证据**：脱敏主机清单、镜像/构建 receipts、两台主机实际 inspect、来源映射。 - -### P01 五节点发现、初始化与身份 - -- **P01.1 正常形成**：通过正常生产 policy、发现、key admission 和存储准备形成 - 五个 voter。验证五个不同 discovery UUID/node ID、实际 accepted initial_voters、 - 固定成员和真实 listener；使用当前支持的五 voter 路径，不杜撰 CLI 参数。 -- **P01.2 地址与端口**：逐节点核对 offset 只应用一次、同主机无冲突、跨主机 - 不使用对方不可达的 loopback，以及两个 rack/host 的归属。 -- **P01.3 保留身份**：已有节点重启只恢复原 binding、卷和身份，不重新 bootstrap。 - 过期绑定、外来卷和相同名称下被替换的容器必须拒绝采用。 -- **证据**：accepted policy、成员/监听地址、卷映射、bootstrap 和发现身份。 - -### P02 全集群 readiness 与持续观察 - -- **P02.1 基线**：五个 Web 入口均 Ready，每节点七个服务、合计 35 个真实存活 - 进程；同时验证健康、依赖和 DiskDB 当前 ownership lease 的持续续约。 -- **P02.2 新鲜度**：记录接收时刻，拒绝早于容器启动或年龄超过三秒的 monitor - 快照。区分进程存活、listener 存活和数据服务 Ready。 -- **P02.3 authority 退化**：Group0 切换/不可用或一个远端观察变慢时，本地 monitor - 仍能报告当前进程和真实健康。服务意图读取不能拖住本地健康发布或诊断控制。 - 不允许旧 healthy 值只更新时间；不因 readiness 暂退就重启仍存活的服务。 -- **P02.4 观察器负例**：回归旧快照、错误容器/PID/start-time、OOM、兄弟进程 - 丢失、超过五秒完整采样间隔，以及恢复后第二次身份变化；均必须失败。 -- **证据**：完整原始样本、各门禁 receipt、恢复身份 fence 和观察器结果。 - -## 3. 大类 B：原生数据、协议数据与规模 - -### P03 五入口基线数据和分配 - -- **P03.1 KV**：验证 Groups 0/1/3 的五副本映射和当前 leader/lease。执行单次 - single/batch 写入，从五入口读回完整 key/value；记录 acknowledgment。 -- **P03.2 ChunkDB**：生产 native client 分配并查询同一个 Chunk ID，核对位置、 - 大小和身份，不以一个新 ID 的成功查询替代原分配结果。 -- **P03.3 S3**：经 `crowdb-tpc-loader` 生成独立 S3 SF1 数据，并保存每个对象的 - URI、长度和摘要；经五入口完整读回八张表对应文件。另保留至少 96 KiB 的 - run-unique witness，核对完整字节和 hash。 -- **P03.4 Iceberg**：独立生成八表 SF1，核对 namespace/table UUID、snapshot、 - metadata location、manifest 文件集合、文件长度/摘要、所有行解码和独立扫描。 - schema、属性提交和数据文件分别验证；只读 footer 或十行 sample 不能代替全读。 -- **P03.5 Dataset**：生成含 inline fields、multipart payload 和不可变 sample - snapshot 的数据。逐份报告、逐字段、经五入口完整读取；核对实际报告数。 -- **P03.6 协议隔离**：原生压测 key 必须位于实际安全范围，避开 S3/Iceberg/ - Dataset 的保留元数据前缀；给协议 key 添加测试后缀仍可能破坏协议扫描。 -- **证据**：数据/ack ledger、loader 来源和命令、全部生成报告、五入口校验结果。 - -### P04 512 条短复现与五 owner 覆盖 - -- **P04.1 旧 key 形态**：独立运行历史 512 条复现，记录其真实 owner 覆盖； - 不把它自动计为五 owner 场景。 -- **P04.2 正式形态**：完整读取 catalog，在五个 Serving owner 的真实范围选择 - run-unique 安全 key。32 批 × 16 条 × 64 KiB，共 512 条/32 MiB；保留每批 - 默认五秒 deadline。统一首字节分布不能代替路由 owner 证明。 -- **P04.3 完整性**：逐批记录确认与未知结果，完整回读；保存 split/cutover、 - 各阶段延迟和错误。未知写入只读查证，不重放。 - -### P05 更大工作集与持久存储边界 - -- **P05.1 规模**：P04 通过后，新 namespace 至少 4096 条确定性 64 KiB 记录， - 共 256 MiB，覆盖五个实际 Serving owner。预算 mirrors、metadata、WAL、 - reclamation 和 artifacts，测试 media 保持至少 30% 空闲。 -- **P05.2 extent 边界**：根据实际 block 大小构造跨文件尾部的持久数据，冷打开后 - 触发异步读取跨一个及多个 extent。核对完整内容和一次 completion；记录实际 - offset/length/边界，不能只因工作集大就声称覆盖该路径。 -- **P05.3 错误传播**：组件回归覆盖缺失后续 extent、尾部截断和范围溢出，保留 - 真实 I/O 错误，拒绝短读成功、重复 completion 或自动重试掩盖错误。 -- **证据**：owner/key/字节数、全量摘要、实际 extent 路径和组件/物理覆盖分别列明。 - -## 4. 大类 C：30 分钟混合负载与最终一致性 - -### P06 持续混合负载 - -- **P06.1 参数**：开始前声明并发、比例、pacing、key/version 规则和有界数据量。 - 负载覆盖两个主机和五个 owner，复用/回收数据时仍持续产生成功操作。 -- **P06.2 原生与协议并发**：混合 native 插入、覆盖、删除、读回；同时执行 S3 - 读写删、Iceberg commit/load/cleanup 和 Dataset publication/readback。 -- **P06.3 有效时长**：至少 30 连续分钟成功混合工作重叠，不含构建、部署、空等。 - 固定 seed 后等待不算负载。修复或控制器重启打断负载后重新完成连续 30 分钟。 - -### P07 确认语义和数据审计 - -- **P07.1 ledger**：mutation 前后保存 operation、key/version、预期长度/hash、 - ack 状态和时间；同 key 覆盖串行化或有明确版本合同，避免 verifier 自身竞态。 -- **P07.2 在线抽查与收尾全查**：连续读取已确认数据，结束时核对所有 live keys、 - 覆盖后的值、完整协议 payload、witness，以及所有已确认删除的内容仍然不存在。 -- **P07.3 未知结果**：超时/断连保存 OutcomeUnknown；恢复后存在时验证预期内容， - 不能直接断言不存在，也不能为消除未知而重放 mutation。 -- 正常负载中的请求失败、内容不符、OOM 或未计划进程替换均使该负载阶段失败。 - -### P08 性能与资源观测 - -- **P08.1 完整采样**：至少每五秒记录吞吐、成功/失败/未知数、p50/p95/p99/max、 - CPU、RSS/cgroup memory、空闲空间、进程身份、catalog 和 reclamation。 -- **P08.2 分段统计**：分别列 build、启动、seed、持续负载、故障和审计耗时； - 保留成功操作/字节、有效重叠时长和不同阶段延迟，不混算或仅报告容器存活。 -- **P08.3 主机异常**：保存 diskstats、I/O/memory pressure、cgroup throttling/ - OOM 和有权限读取的内核错误。低 CPU/充足内存不能排除 fsync 或设备 I/O 停顿。 - -## 5. 大类 D：进程、容器与主机故障 - -### P09 原卷冷启动与镜像升级 - -- **P09.1 冷重建**：停止并重建全部五个容器，沿用原卷、policy、UUID/node ID - 和 bootstrap。验证 35 服务、所有固定 checkpoint 和五入口完整历史数据。 -- **P09.2 同卷升级**：生产修复后验证新镜像内容身份，再替换容器；所有卷的名称/ - CreatedAt 和数据身份不变。镜像传输 tar 是分发文件，不是数据卷备份。 -- **P09.3 操作竞态**：重建前再次核对旧容器 ID、stopped 状态/时间、挂载和归属； - 若准备期间曾被其他操作启动/替换，则停止，不盲目重复 remove/create。 - -### P10 单个无响应 ChunkKV - -- **P10.1 注入**：仅向一个已核对 container/PID/start-time 的 ChunkKV 发 SIGSTOP。 - 不暂停整容器或误伤同名服务。 -- **P10.2 恢复**：60 秒内旧进程退出并被新 PID 替换，其余 34 个服务身份不变。 - 验证旧数据和本场景新确认写入；恢复后全量审计期间不允许再次静默替换。 -- **P10.3 失败收尾**：保留错误后解除仍存活旧进程的 STOP，防止遗留注入污染后续。 - -### P11 跨主机 KV 生命周期 - -- **P11.1 Follower**：选择与当前 leader 不同物理主机的 follower，经它的 Web - stop 本地 KV；核对 completed Stop receipt 和实际退出。远端 authority 仍可 - 管理该节点，单次 restart 得到新 PID，其余 34 服务不变，完整历史审计通过。 -- **P11.2 Leader**：重新识别当前 leader，执行 Stop；即使它已退出，原 completed - receipt 仍可读取。等待存活节点的新 authority，再单次重启原节点并完整审计。 -- **P11.3 切换窗口**：全过程检查幸存节点的本地 monitor 新鲜度与真实进程身份。 - 不能只检查最终 Ready，从而漏掉服务意图读取阻塞、旧健康快照或兄弟服务重启。 -- **P11.4 退出证据**：出现 native abort、自 join、WAL drain 失败或其他异常退出， - 先保存对应 PID/容器代际的应用、RPC、tree、stderr 和 crash/core，再诊断。 - -### P12 B 主机两个节点同时离线 - -- **P12.1 真正接管**：故障前证明 B1/B2 都是当前 Serving owner，再停止它们。 - A 的三 voter 在 180 秒内接管 catalog；执行单次 KV 写/读，记录真实新 ack。 -- **P12.2 停机期间新 payload**：B 保持离线，经 A 新写 S3 witness 和 Iceberg - property commit；新 payload 的实际布局须能由 A 独立读取，三个 A 入口全读。 -- **P12.3 EC 边界**：记录旧对象每主机 fragment 数和丢失预算。例如 EC 8+4 在 - B 上有六片时，不能要求 B 离线期间该对象仍可重建，也不能宣称已有 rack - 保护。此时的新写可用性结论与旧 EC 暂不可用分别记录，不混为一个成功。 -- **P12.4 恢复与持久化**：恢复 B，再分别用原卷重建 B1/B2，验证原身份、五副本 - 固定 checkpoint 和五入口全部历史数据，包括停机时不可用的旧 EC 对象。 -- **扩展**：若要验证旧 EC 数据也能抗整机故障，单独选择满足丢片预算的真实布局， - 并在停机期间全读旧数据；不能由三 voter 存活推导这个保证。 - -### P13 A 主机三个节点同时离线 - -- **P13.1 失去 quorum**：停止 A1/A2/A3，B 在 30 秒内报告当前不可用；新 mutation - 不得确认成功。记录失败与未知结果，不把 connection loss 等同于没有写入。 -- **P13.2 租约安全退出**：因原租约到期而退出的幸存 ChunkKV 必须有同一进程的 - lease 失效、主动停止、RPC worker join 和退出证据。native abort 或未知原因 - 的替换不能归类为正常 lease fencing。 -- **P13.3 恢复与协议读取**：恢复 A，验证全部服务和固定五副本 checkpoint， - 再核对所有原生/协议历史。每个 Iceberg reader 独立保存 endpoint、PID、 - table/object/stage、时间、stdout 和退出码，再聚合结果。 -- **P13.4 错误归属**：区分真实 curl/GetObject 首错、其他 reader 的中止和后续未 - 执行项。S3 原文件通过不能替代 Iceberg 全读；不能按并发输出顺序猜测入口。 - -## 6. 大类 E：分配、积压恢复与空间正确性 - -### P14 分配级显式安全重试 - -- **P14.1 有界故障**：保留所有 KV 进程和 Group0 leader/term/lease，在 Group1 - 注入 leadership step-down，三秒内至少一次被当前 leader 接受。 -- **P14.2 实际触发**：单次启动生产分配 client，必须观察到 allocation-level - 明确 Unavailable safe retry。仅最终成功、transport timeout 或未触发故障均 - 不能代替该断言。 -- **P14.3 原策略完成**：释放注入，用同一 ID 在原五次重试/十秒限制内完成 - allocate/query。保存真实 retry/ack 证据；instrumentation 不改变分支或策略。 - -### P15 超过 1024 slots 的积压与全冷恢复 - -- **P15.1 实际 gap**：停止一个真实 KV follower，经存活副本确认至少 1100 次 - 单写；记录 offline checkpoint 上界、写前/写后目标，证明实际差值大于 1024。 -- **P15.2 catch-up 与冷启动**：恢复 follower，验证 FetchGap 有界并发和真实进度； - 然后全停五节点并沿原卷冷启动。故障前固定 Groups 0/1/3 的 ack target， - 每组五副本健康引擎都必须达到目标，再核对全部数据。不能随新写入移动目标。 -- **扩展 P15.3**：分别制造 reclaimed WAL、empty sealed WAL、high-promise/ - old-chosen 等物理边界，记录触发证据。未实际制造就保留覆盖缺口；组件回归 - 和普通大 gap 的成功不等于这些物理边界已经通过。 - -### P16 最终数据与 DiskDB 占用 - -- **P16.1 最终门禁**：catalog 全部 Serving，五入口 Ready，35 服务健康；下一样本 - 与最终审计结束时仍是相同进程身份。 -- **P16.2 新旧数据**：新 KV batch、新 ChunkDB allocation/query 与所有历史数据 - 分别校验，继续区分 acknowledgment、已确认删除和 OutcomeUnknown。 -- **P16.3 全量占用**：只读扫描所有本轮 DiskDB occupancy，覆盖完整 disk/zone/ - continuation，无截断；核对 live mirror segments 不跨 chunk 重叠，并保存 - 实际 live、allocated、free 和 reserve 数。 - -## 7. 大类 F：首错诊断、修复和清理 - -### P17 失败证据与分类 - -- **P17.1 首错**：打印失败用例、操作和完整错误，调试时启用 - `CROWDB_E2E_PRINT_SERVICE_ERRORS=1`。保留原始客户端输出，不仅保存包装层 panic。 -- **P17.2 同时间线**：保存 inspect/exit/OOM、当前 PID/start-time、健康时间戳、 - KV role/term/lease/checkpoint、catalog/transfer、DiskIO 和磁盘资源。 -- **P17.3 native 与主机**：进程异常先分析 crash/core；I/O 停顿补 diskstats/ - pressure 和可读的阻塞线程栈。观察事实、推断和未证实原因分开写；后续 PASS - 不会反向证明旧无栈 abort 或旧无入口标签超时的原因。 -- **P17.4 完整日志**：按实际 PID、容器启动时间和日志类型选择 application、RPC、 - metrics、tree 文件。历史 stderr 不能算作新进程错误，也不能因日志非空就丢弃 - 新错误；保存前后证据或能绑定本次执行的增量。 - -### P18 修复和续跑 - -- **P18.1 最小复现**：找第一次偏离，区分代码、时序/注入、观察器和环境原因； - 对代码缺陷补能够在旧代码失败的 focused regression，再运行受影响组和门禁。 -- **P18.2 实际输入**：生产输入变化后重新构建、校验、分发镜像及客户端；先修复 - 会使后续慢/挂/失败的依赖，不让队列继续反复撞同一问题。 -- **P18.3 续跑账本**：每次 attempt 使用独立身份和不可覆盖 receipt，保留原失败、 - 已确认写入和未知结果。观察器失败不能靠重设基线或移动 stop marker 自动消除； - 必须记录修复/环境核对和新审计范围，再继续未完成项。 -- **P18.4 覆盖边界**：组件、容器、物理和 soak 分开记；新镜像的局部 PASS 不 - 继承旧镜像的全部覆盖。负载被修复打断须新跑 30 分钟；后续恢复修复是否影响 - 旧 soak 输入须明确说明，不为了制造“同镜像 PASS”重复不受影响的全部测试。 - -### P19 停机留证与精确清理 - -- **P19.1 双主机预检**：先固定两台主机全部待清理对象；核对 run label、名称、 - container ID、原卷名称/CreatedAt、mount、最后 stopped 状态/时间。任一项改变 - 即停止，不能只因名称相同继续删除。 -- **P19.2 停机留证**：停止本轮节点后保存完整服务日志、crash、inspect 和数据 - ledger；两台主机全部留证成功后才开始第一次删除。 -- **P19.3 数据卷备份策略**：默认不新增整卷归档，依照用户本轮保留/清理要求执行； - 已存在的私有备份继续保留。只有用户明确要求时才做额外卷备份并验证完整性。 - 服务日志留证、镜像传输文件和数据卷备份是不同对象，不互相替代。 -- **P19.4 精确删除**：只删除已获授权且预检成功的归属对象，数量与清单完全 - 相符；不使用 daemon-wide prune，不清共享 runtime 或其他任务资源。 -- **P19.5 清理后验证**：两主机本轮目标资源均不存在、所属监听端口关闭、SSH key - 仍可用；保留 run artifacts、原始错误和约定保留文件。不得在报告中隐去之前 - 发生的越界删除或无法恢复的数据丢失。 - -## 8. 每轮交付与验收结论 - -统一目录：`.crowdb-runtime/artifacts/physical-nodes//`。保存 harness 的 -实际源码与命令；当前没有一个仓库 Pixi task 实现本目录的全部流程，不能依赖 -先前临时文件或把方案描述成已有的一键任务。 - -- `run.json`：所选父/子场景及扩展 profile、两主机、节点/端口/资源映射、归属 - 清单、源码/镜像/客户端身份、dirty diff、harness 来源和构建/传输 receipts。 -- `results.json`：各场景 PASS/FAIL/NOT RUN、独立门禁起点/限制/耗时、实际注入 - 是否触发、未执行原因、所属 attempt 和证据路径。旧结果不可覆盖。 -- `data-ledger.jsonl`、`metrics.jsonl`、`checkpoints.json`：数据及确认语义、分段 - 统计、固定目标和实际恢复进度；大文件分段保存。 -- `loader/` 与 `audits/`：各协议源版本、生成/注入/验证命令、完整报告和每个入口 - 的独立结果。未实现路径标 NOT RUN，不以其他协议或组件结果替代。 -- `observations/`、`first-failures/`：注入至完整审计结束的连续观察、身份 fence、 - 首错、完整日志、crash 和明确仍未证实的原因。 -- `cleanup.json` 与 `summary.md`：精确清理结果、实际源码代际覆盖、未完成/未触发 - 场景、数据保留/丢失事实和下次起点。明确区分“结束运行”与“所选必需场景全过”。 - -最终结论只覆盖本轮明确选择并实际通过的场景。人工共同 review 与测试执行分别 -记录；测试 PASS 不自动完成共同 review。 - -## 9. 实现和回归入口 - -- 启动/镜像：`container/oci-image/{run-node,runtime,artifact,transfer}.py`、 - `container/single-node-container/entrypoint.sh` 和 - `container/crowdb-monitor/src/node/startup/policy.rs`。 -- 形成/观察：`lib/crowdb-protocol/src/mgmt/node/`、`app/crowdb-web/src/node/`、 - `lib/crowdb-console-shared/src/ops/cluster/readiness/` 和 - `container/crowdb-monitor/src/node/runtime/`。 -- 数据/恢复：`lib/crowdb-{kv,kv-client,chunk-kv-client,chunkdb-client}/`、 - `app/crowdb-chunkdb/src/topology/`、`lib/crowdb-chunk-stream/src/production_chunk/` - 和 `lib/crowdb-tree/src/backend/local/`。 -- 组件回归：KV backlog/FetchGap durability、ChunkStream finalized-empty、Monitor - discovery/health/service receipts、ChunkDB allocation retry/topology availability、 - BlockAsyncPageStore 跨 extent 读取。按实际改动选择受影响组，通过 `pixi run` 执行。 -- 容器/协议回归：`e2e-test/tests/deployment/`、`e2e-test/tests/console/`、 - `e2e-test/tests/access_iceberg/` 和对应 S3/native SDK 入口；它们与真实跨主机 - 故障、连续 30 分钟负载分别计覆盖。 +This reusable catalog follows category, test case, and scenario. It does not +track the current run's progress. Keep selected scenarios, actual commands, +source/image identities, and results in a separate run directory. Historical +passes do not establish coverage for a new image. + +Upstream: [deployment and recovery](../design/deploy/design-crowdb-deploy.md), +[OCI builds and runtime](../../container/oci-image/README.md), +[container acceptance](../../e2e-test/README.md), and +[KV recovery](../design/kv/design-crowdb-kv-slot.md). + +## 1. Execution contract + +### Environment and scope + +- Use two physical failure domains by default: A `192.168.31.129` and + B `192.168.31.184`. Recheck host identities, interfaces, and addresses each + run; an IP address does not prove resource ownership. +- Use five fixed voters, A1/A2/A3 and B1/B2, with separate persistent volumes + and discovery identities. Port offsets are 0/1/2 on A and 0/1 on B; verify + the values actually accepted by the current launch interface. +- Fix each container at host networking, 1 CPU, 2048 MiB, and 512 PIDs, with + one Tokio worker per service. Record actual cgroup limits, storage capacity, + reserves, and deviations. +- Simulate a host fault only by stopping this run's designated containers on + that host. Do not change shared networking, routes, firewalls, the kernel, + or the Docker daemon. A three-node variant requires a separate profile and + cannot substitute for five-node coverage. +- Use the existing SSH key and noninteractive BatchMode. Supply credentials + privately; exclude passwords, private keys, tokens, login identities, and + commands containing authentication arguments from committed docs/reports. +- Run all builds, tests, and executables, including remote commands, through + `pixi run`. + +### Case selection and results + +- Retain P00-P19 as parent case IDs; `.1`, `.2`, etc. identify independently + verifiable scenarios. The default main sequence covers every parent and its + ordinary scenarios. Scenarios explicitly marked as extensions are required + only by the selected profile; an unselected extension has not passed. +- Freeze scope, concurrency, resources, data size, timeouts, and acceptance + conditions before starting. Record added regressions separately. Do not + remove scenarios, weaken assertions, or change profiles after failure to + obtain an all-pass result. +- For each scenario, record prerequisites, fault target/time, actual actions, + assertions, duration, result, evidence paths, and input identities. Use + `PASS / FAIL / NOT RUN`; an injection that never occurred is a coverage gap. +- Follow dependency order and stop dependent work at the first failure. + Preserve failures and valid passes. After repair, repeat only scenarios + whose inputs or behavior changed, then resume unfinished work. Do not + blindly replay partially executed stages. +- Persist the operation, key/ID, and expected content before each mutation; + persist its acknowledgment after success. Record timeouts/disconnections as + `OutcomeUnknown` and reconcile them read-only. Controllers must not replay + uncertain writes. +- Give fault injection and later read-only audits distinct command-log names. + Preflight those names before mutation. A corrected fresh attempt needs its + own artifact namespace and operation receipt; retain the failed attempt and + establish its mutation outcomes before submitting another operation. +- Do not increase timeouts merely to hide failures, weaken checks, or add caller + retries. An agreed production probe-budget change needs a separate attempt + and preserves earlier failures. P14 observes + the production client's existing explicit safe-retry policy unchanged. +- Complete builds before their consumers and serialize builders sharing an + output directory. Freeze sources before building OCI; retain the source + tree, commit/dirty diff, client digests, and image digest. Reassess affected + coverage after source changes. Separate compilation/linking from acceptance + workloads on shared physical hosts. A contention scenario must declare its + competing workload in advance and collect host pressure, CPU, and disk data. +- Record CI and physical results separately. When the user stops CI inspection, + do not query or infer new runs or make CI completion a physical prerequisite. + Required local gates still apply. + +### Assertions shared by recovery cases + +- Before injection, capture container IDs, volume names/CreatedAt, discovery + UUIDs, node IDs, bootstrap bindings, all 35 services' container/PID/start-time + identities, catalog state, and replica state. +- Observe continuously from injection through recovery and the full data audit. + Complete samples must be no more than five seconds apart. A fault target + missing from `/proc` remains an absent observation; retain sibling samples. +- Permit only the process changes specified by the case. Surviving monitors + must keep publishing fresh, truthful observations during recovery. An offline + target may be unobservable; old Ready evidence is not current evidence. + Freshness and health are separate assertions: updating an old healthy + timestamp cannot conceal Unavailable. +- Recovery gates require current, healthy monitor observations from every + node, newer than its container start and no more than three seconds old at + receipt. Fence all 35 process identities until the full audit ends; do not + reset the baseline to tolerate another restart. Establish the first strict + health check and identity fence within the original service-recovery budget. + An early Ready response cannot defer actual recovery verification beyond + that budget. Independently check strict health again when the full audit + starts. Later health failures or identity changes are audit failures, not + another recovery-wait interval. +- Measure catalog takeover, service recovery, and each of Groups 0/1/3's fixed + checkpoint catch-up separately, each with its original 180-second budget. + Record each gate's own start and duration. Preserve P10's 60-second limit, + P13's 30-second limit, and shorter production-client limits separately. +- After recovery, verify P03 baselines, P04/P05 data, P07 acknowledged live and + deleted records, every historical witness, and this case's new writes. + Read and validate complete content through all five recovered endpoints. +- Actual EC fragment placement determines payload availability; KV quorum + cannot substitute for it. Report temporary availability beyond the fragment + loss budget separately from complete historical correctness after recovery. + +## 2. Category A: hosts, images, and cluster formation + +### P00 Environment, authentication, and images + +- **P00.1 Hosts and resources**: verify both physical hosts, SSH keys, clocks, + interfaces, cross-host listener reachability, Docker/cgroups, free disk, and + memory. Inventory existing resource ownership and record five-node limits + and storage budgets. Fail inadequate preparation without shrinking data. + Record the actual link medium, selected route/interface, and each Wi-Fi + adapter's power-management state. Declare the network profile before loaded + acceptance; neither enabled nor disabled power saving is a universal pass + requirement. Any setting change requires user authorization, before/after + verification on both hosts, and a restoration or continuing-profile decision. + Where available, record kernel ACK policy, CPU idle/governor state, scheduler + pressure, and Wi-Fi power saving as possible timing variables. Their presence + does not prove causation; do not tune host settings automatically. + Do not disconnect an interface or alter shared routing as a test fault. +- **P00.2 Identical image content**: import the same verified image transfer + file on both hosts. Check its digest, manifest-referenced config content hash, + platform, source/runtime labels, and ordered rootfs layers. Docker may expose + a manifest ID on one host and a config ID on the other; comparing ID strings + alone is insufficient. +- **P00.3 Invalid inputs**: offline regressions reject damaged transfers, + source/runtime label mismatches, reordered layers, and architecture mismatch + before any node upgrade starts. +- **Evidence**: redacted host inventory, image/build receipts, actual inspect + results from both hosts, and provenance mapping. + +### P01 Five-node discovery, initialization, and identity + +- **P01.1 Formation**: form five voters through production policy, discovery, + key admission, and storage preparation. Verify five distinct discovery + UUIDs/node IDs, accepted initial_voters, fixed membership, and real listeners. + Use the supported five-voter path without inventing CLI arguments. +- **P01.2 Addresses and ports**: verify each offset is applied once, same-host + ports do not collide, cross-host addresses avoid unreachable loopback, and + both rack/host assignments are correct. +- **P01.3 Identity retention**: restart existing nodes with their original + bindings, volumes, and identities without bootstrapping again. Reject stale + bindings, foreign volumes, and replacement containers reusing an old name. +- **Evidence**: accepted policy, membership/listeners, volume mappings, + bootstrap bindings, and discovery identities. + +### P02 Cluster readiness and continuous observation + +- **P02.1 Baseline**: all five Web endpoints are Ready, with seven services + per node and 35 actual live processes. Verify health, dependencies, and + continued renewal of DiskDB's current ownership lease. +- **P02.2 Freshness**: record receipt times and reject monitor snapshots older + than container start or three seconds. Distinguish process liveness, listener + liveness, and data-service readiness. Cross-host sampling may reuse an + authenticated SSH connection, but each response requires a complete fresh + host sample. Record transport/process-start cost separately from node sampling + time. Reuse cannot relax the five-second complete-observation bound, replay + samples, renew old timestamps, or hide sampling failures. + For loaded cross-host latency, retain idle/load/idle observations with the + same native or protocol audit, source/image, data size, logical deadlines, + concurrency, resource limits, and pinned process identities. Record host I/O, + competing builds, cached-health responses and failed generations separately. + If authorized, compare declared Wi-Fi power-management profiles as distinct + attempts; retain both states and results without rewriting the first failure + or treating a faster comparison as complete recovery acceptance. Use matched + request/response packets and conservative cross-host clock bounds to locate + delay. Record capture scope, startup boundary, kernel drops, and gaps: dropped + or missing capture packets cannot prove network loss. Reconcile collector + failures offline when retained bytes suffice, preserving the original receipt + and qualifying derived evidence separately. + Positive matched packet timestamps include host stack and queue delays; + they are not pure Wi-Fi propagation or server-handler duration. State clock + assumptions and retransmission limits. With capture gaps, an observed request + or reply need not be the earliest emission or arrival. +- **P02.3 Authority degradation**: during Group0 failover/unavailability or a + slow remote observer, the local monitor still reports current processes and + truthful health. Service-intent reads must not block health publication or + diagnostic control. Readiness HTTP requests share independent background + evidence rather than rereading full authority on every poll. Component + regressions cover reads lasting over one second, concurrent readers, expired + or failed evidence, and recovery without page traffic. A warm ChunkDB point + probe must still read the real owner/storage when its client's ordinary + registry refresh becomes due; it must not expand into a full authority scan. + Cover cold discovery, stale-owner rejection, failed-route refresh, and actual + storage errors with the original deadline. When a bulk response + blocks a normal TCP connection, the same storage client's physical probe + reads the original disk's 4096-byte header through the bounded priority lane. + Rejection, short reads, and probe timeout still fail. Compare the same client, + target, and deadline; distinguish response counts from failed probe generations. + Start physical headers as soon as hardware discovery succeeds while the + independent ChunkDB read continues. Exercise the production orchestration + with hardware taking 50ms, ChunkDB 700ms, and a header 100ms: all checks must + complete within the original shared 750ms budget. Held or failed dependencies, + empty/offline hardware, short headers, and cancellation must still fail without + leaked work. With 17 disks, preserve the eight-read batch bound. + This 750ms case remains a component scheduling control. Physical Access + qualification uses the declared two-second production dependency budget and + 2100ms outer watchdog. Record every failed generation and continue sampling; + a successful new generation recovers automatically. Raise every probe failure + below three seconds with its configured timeout, actual duration, error and + recovery; decide jointly before increasing the production budget again. + During a sustained dependency outage, verify continuous negative generations + within the original fault window. Require at least three distinct advancing + generations per fixed endpoint, reject missing/error/Ready samples, and retain + the complete history. Negative evidence may truthfully expire between slow + rounds; do not require every response to be current or renew an old timestamp. + Validate its declared three-second age and require current Ready evidence only + after actual recovery. Preserve process identity and durable-data checks. + Separate deadline correctness from availability qualification: a real read + completing after its declared shared budget requires truthful Unavailable/503, + rather than renewed Ready evidence. That response still fails any applicable + recovered-health availability check; it does not by itself prove a software + defect. Distinguish measured OS/link timing from a source defect established + by a causal regression. Keep formal recovery, identity, data, and availability + assertions unchanged while the cause remains unresolved. + Hold a KV scan frame after its first control byte while the same ChunkDB KV + client issues a real point read. Verify point progress with one additional + connection per endpoint and the existing worker/completion budgets. Cover + rejection, deadline expiry, failed-generation reconnect, explicit pool clear, + and the unchanged ordinary transport as a control. + Under mixed system/user-data metadata, record background KV pages and bytes. + Grow only the other maintenance domain and verify that scoped chunk and + reservation scans do not transfer its pages. Cover all supported purposes, + zero-valued ID suffixes, continuation, foreign slots, and multiple destinations; + unscoped inventory must still return both domains. + Hold one instance's process-lease CAS while another instance remains ready. + A grant pass must attempt each independently ready instance once, atomically + publish the healthy instance's lease/grant pair, and report the failed instance + after the pass. Cover a valid exhausted sequence as another independent + publication failure. Preserve the failed instance's bytes and all process, + assignment, revision, expiry, and replacement fences without a conditional retry. + An error's phase label alone does not identify where a shared budget was + consumed. Never renew an old healthy value or restart a live service solely + for transient unready status. Cover a new process that remains unready and + several listeners exhausting their probe deadlines: background recovery must + yield between passes, and real health probes run concurrently so their waits + do not accumulate and stall the entire snapshot. +- **P02.4 Observer negatives**: reject old snapshots, wrong container/PID/start + identities, OOM, missing siblings, complete-sample gaps over five seconds, + and a second identity change after recovery. +- **P02.5 WAL I/O and executor progress**: use the production multi-thread + runtime with one worker and a real file-backed BlockDevice WAL. Hold an + actual write or durable-sync boundary and verify that independently spawned + apply/RPC-like work still progresses while the append remains unacknowledged. + Release the boundary, then verify write order, durable reopen, and original + error behavior. Keep a current-thread compatibility control; do not claim + that synchronous fallback provides task isolation. In physical slow windows, + correlate Group0 read barrier, apply-fence wait, actual engine apply, WAL + append/sync, and cgroup counters. Record that a shared readiness deadline can + expire before DiskIO starts; a phase label is not proof of slow physical I/O. +- **P02.6 Process observation validity**: follow the + [Console service lifecycle contract](../design/console/design-crowdb-console-ui.md#13-service-lifecycle). + Pair actual owned-process samples with focused backend/UI controls for failed + reads, malformed or incomplete inventories, expired evidence, and wrong Node, + bootstrap/operation epoch or process identity. Missing PID evidence is unknown, + not confirmed absence; retain registered services and successful siblings. + Only a complete, fresh, identity-matched observation can confirm that no process + exists, and an accepted startup must remain pending. Endpoint health cannot + supply a missing PID or prove process absence. +- **P02.7 Read-only plan recovery**: retain a failed-only saved plan and verify + observation continues every two seconds even without Capacity data or another + queued deployment. Unknown evidence keeps waiting or retains the recorded + failure; a current owned running process reconciles progress to deployed. + Include the same PID before and after an observation gap, interrupted-plan + reload, and stale/foreign/expired evidence that must not clear a failure. + Keep the original three-second UI reconciliation assertion. The plan must + issue no automatic Start, Restart or Deploy; explicit Retry alone initiates + another failed lifecycle mutation. Progress recovery does not establish health. +- **Evidence**: complete raw samples, gate receipts, recovery identity fences, + observer results, persisted plan revisions, and lifecycle request counts. + +## 3. Category B: native data, protocols, and scale + +### P03 Baseline data and allocation through five endpoints + +- **P03.1 KV**: verify five-replica mappings and current leaders/leases for + Groups 0/1/3. Issue single and batch writes once, record acknowledgments, and + read complete keys/values through all five endpoints. +- **P03.2 ChunkDB**: allocate and query the same Chunk ID with the production + native client. Check location, size, and identity; a successful new ID cannot + substitute for the original allocation result. +- **P03.3 S3**: independently generate S3 SF1 data with `crowdb-tpc-loader`. + Retain each object's URI, length, and digest, and fully read all eight tables' + files through five endpoints. Retain a run-unique witness of at least 96 KiB + and verify its complete bytes and hash. +- **P03.4 Iceberg**: independently generate eight-table SF1. Check namespace/ + table UUIDs, snapshots, metadata locations, manifest file sets, file lengths/ + digests, decoding of every row, and independent scans. Verify schema, property + commits, and data files separately; footer reads or ten-row samples cannot + replace full reads. For credential expiry, retain the same FileIO and open + stream across the service grant's actual lifetime, then seek/read the same + immutable object. Record original TTL, identity, and refresh scope; do not + substitute table reloads, clock changes, or longer TTLs. Limited scans must + release the native reader before the input file on success and failure. The + verifier must exit normally; a crash or hang after row validation is FAIL. +- **P03.5 Dataset**: generate inline fields, multipart payloads, and immutable + sample snapshots. Fully read each report and field through five endpoints; + verify the actual report count. +- **P03.6 Protocol isolation**: native stress keys must occupy proven safe + ranges outside S3/Iceberg/Dataset metadata prefixes. Adding a test suffix to + a protocol key can still corrupt protocol scans. +- **Evidence**: data/ack ledgers, loader provenance and commands, all generated + reports, and independent verification from five endpoints. + +### P04 A 512-record reproducer and five-owner coverage + +- **P04.1 Historical keys**: run the original 512-record reproducer separately + and record its actual owner coverage. Do not automatically count five owners. +- **P04.2 Formal workload**: read the full catalog and choose run-unique safe + keys within five actual Serving owners' ranges. Use 32 batches of 16 records, + each 64 KiB: 512 records/32 MiB total, retaining the default five-second batch + deadline. Uniform first-byte distribution does not prove routed ownership. +- **P04.3 Integrity**: record each batch's acknowledgments and unknown outcomes, + then fully read it back. Retain split/cutover state, phase latency, and errors. + Reconcile unknown writes read-only without replaying them. + +### P05 Larger working sets and persistent-storage boundaries + +- **P05.1 Scale**: after P04 passes, use a new namespace with at least 4096 + deterministic 64-KiB records, totaling 256 MiB across five actual Serving + owners. Budget mirrors, metadata, WAL, reclamation, and artifacts; keep test + media at least 30% free. +- **P05.2 Extent boundaries**: construct persistent data across file ends using + actual block sizes. After cold open, trigger asynchronous reads crossing one + and multiple extents. Verify full content and exactly one completion. Record + actual offsets, lengths, and boundaries; a large working set alone does not + prove this path ran. +- **P05.3 Error propagation**: component regressions cover missing subsequent + extents, truncated tails, and range overflow. Preserve real I/O errors; reject + short-read success, duplicate completion, or retries masking errors. +- **Evidence**: owners/keys/bytes, full digests, actual extent paths, and + separately stated component and physical coverage. + +## 4. Category C: 30-minute mixed workload and final consistency + +### P06 Sustained mixed workload + +- **P06.1 Parameters**: declare concurrency, operation mix, pacing, key/version + rules, and bounded data volume before starting. Cover both hosts and five + owners; reused or reclaimed data must still produce successful operations. +- **P06.2 Concurrent native and protocol work**: mix native insert, overwrite, + delete, and readback with S3 writes/reads/deletes, Iceberg commit/load/cleanup, + and Dataset publication/readback. +- **P06.3 Effective duration**: sustain at least 30 continuous minutes of + successful overlapping mixed work, excluding builds, deployment, and idle + waits. Waiting after a fixed seed is not load. A repair or controller restart + that interrupts the workload requires a fresh continuous 30-minute run. + +### P07 Acknowledgment semantics and data audit + +- **P07.1 Ledger**: retain operation, key/version, expected length/hash, ack + status, and time before/after mutations. Serialize same-key overwrites or use + an explicit version contract to avoid verifier races. +- **P07.2 Online checks and final full audit**: continuously read acknowledged + data. At completion verify every live key, overwritten value, full protocol + payload, and witness, and continued absence of all acknowledged deletions. +- **P07.3 Unknown outcomes**: retain OutcomeUnknown for timeouts/disconnections. + If present after recovery, verify expected content. Do not assume absence or + replay the mutation to eliminate uncertainty. +- Request failure during normal load, content mismatch, OOM, or an unplanned + process replacement fails that workload stage. + +### P08 Performance and resource observations + +- **P08.1 Complete sampling**: at least every five seconds record throughput, + success/failure/unknown counts, p50/p95/p99/max, CPU, RSS/cgroup memory, free + space, process identities, catalog state, and reclamation. +- **P08.2 Phase statistics**: separate build, startup, seed, sustained load, + fault, and audit durations. Retain successful operations/bytes, effective + overlap, and phase latencies; container liveness alone is insufficient. + Comparisons of sampling overhead or repairs must hold workload, leader host, + routed owner, client/image, and actual cross-host path fixed, recording the + observer's enablement and cost. Leader movement or early workload completion + makes samples incomparable; retain them without claiming improvement. +- **P08.3 Host anomalies**: retain diskstats, I/O/memory pressure, cgroup + throttling/OOM, and readable kernel errors. Low CPU or ample memory cannot + rule out fsync or device I/O stalls. + +## 5. Category D: process, container, and host faults + +### P09 Cold start and image upgrade on original volumes + +- **P09.1 Cold recreation**: stop and recreate all five containers with original + volumes, policy, UUIDs/node IDs, and bootstrap. Verify 35 services, every fixed + checkpoint, and complete historical data through five endpoints. If this also + accepts a new image's startup, explicitly share that actual cold recovery + and its complete observation. Do not inject the same passed scenario again; + retain interrupted startup attempts as failures. +- **P09.2 Same-volume upgrade**: after a production repair, verify the new + image's content identity before replacing containers. Preserve all volume + names/CreatedAt and data identities. Image transfer tar files distribute + software; they are not data-volume backups. +- **P09.3 Operation races**: before recreation recheck old container IDs, + stopped states/times, mounts, and ownership. Stop if another operation started + or replaced a container during preparation; do not blindly remove/create it. + +### P10 One unresponsive ChunkKV + +Combine full catalog and current placement observations at the recovery gate. +All entries may be Serving while a handoff source is preparing or quiescing; +that alone cannot establish recovery. Reject active handoffs and selected moves. +Catalog generation must match before and after health sampling. Distinguish +historical, committed, serviceable overlays from active owner cutover. Retain +the original 180-second recovery budget and the subsequent strict health, +PID/start-time, and complete-audit assertions. + +- **P10.1 Injection**: SIGSTOP only one ChunkKV whose container/PID/start-time + was verified. Do not pause the entire container or a same-named service. +- **P10.2 Recovery**: the old process exits and is replaced by a new PID within + 60 seconds; the other 34 identities remain fixed. Verify old data and this + case's new acknowledged writes. Reject another silent replacement during + the full audit. Track takeover-triggered source checkpoints and return + migration so a recovered process cannot conceal permanently stuck preparation. + Component injection covers synchronous/asynchronous root-publication failure, + sustained unavailability, and intervening writes: complete the pending root's + original barrier before the next checkpoint writes data. Failure remains + Unavailable, the old root stays recoverable, and the recovered new root + contains acknowledged writes; repeated Internal errors are unacceptable. + A terminal-journal component case must drive an actual ChunkKV metadata + mutation during the DiskIO outage. Include a metadata-only operation, such + as bucket creation: object payload writes can fail before metadata publication + and therefore never exercise the intended journal failure or replay. Keep + the original restart, retained-data, and healthy Access identity assertions. +- **P10.3 Failure cleanup**: after retaining errors, resume any surviving + stopped process so injection does not contaminate later work. + +### P11 Cross-host KV lifecycle + +- **P11.1 Follower**: choose a follower on a different physical host from the + current leader. Stop its local KV through its Web endpoint; verify the + completed Stop receipt and actual exit. Remote authority still manages the + node. Restart once, require a new PID and 34 unchanged siblings, and complete + the historical audit. +- **P11.2 Leader**: identify the current leader again and Stop it. Its original + completed receipt remains readable after exit. Wait for surviving nodes' new + authority, then restart the old node once and complete the audit. +- **P11.3 Failover window**: throughout the case, check surviving monitors' + freshness and actual process identities. Final Ready alone can miss blocking + intent reads, old health snapshots, or sibling restarts. Component regressions + cover a reachable first management seed still returning the old leader: + connection failure or rejection without a hint must continue discovery of + another member's new route within existing retry and lease-safety budgets. +- **P11.4 Exit evidence**: before diagnosing native abort, self-join, WAL drain + failure, or another abnormal exit, retain application, RPC, tree, stderr, + and crash/core evidence for that exact PID and container generation. + Save stderr before injection and compare its exact retained prefix afterward; + older panic bytes must not be attributed to the new exit. Successful native + RPC joins do not establish that Tokio tasks or blocking native jobs finished. + A focused component case holds a real service's background sampling read with + a registered timer, drains the service, then drops its runtime. Drain must + retain ownership until the read and its native payload finish; closed sampling + admission must not launch another job. Capture the isolated process's complete + output under an external watchdog. A matching component panic proves that + path is reachable, not the originating future of a physical exit without a + backtrace. Keep existing supervisor grace and I/O deadlines unchanged. +- **P11.5 Leader tenure notifications**: a component regression accepts a + fenced Admin step-down before the old leader loop consumes its notification, + then drives a real election into a new term. The retained notification must + not step down the new leader; a fresh matching request must still fence it + immediately. During physical failover, retain term, role, and step-down + reasons so an unexpected extra election is not dismissed as a slow network. + +### P12 Both B-host nodes offline + +- **P12.1 Actual takeover**: prove B1/B2 are current Serving owners before + stopping them. A's three voters take over the catalog within 180 seconds; + issue one KV write/read and record its actual new acknowledgment. If earlier + faults changed placement, let normal balance restore both owner prerequisites. + Completed independently recoverable transfers must not lose balance eligibility + forever because of historical transition markers. Fail missing prerequisites; + stopping empty owners or changing balance policy cannot manufacture coverage. + Regress the ordering where a lease-excluded target's completed marker was + released before its first checkpoint: the exact current Serving assignment + may still claim root publication for its new epoch. Different owner, epoch, + tree, stream, or active transition must not bypass checks through marker + absence; the old root writer remains fenced. +- **P12.2 New payload while offline**: keep B offline and write a new S3 witness + and Iceberg property commit through A. The new payload's actual placement must + allow A to read it independently; fully read it through all three A endpoints. +- **P12.3 EC boundary**: record old objects' fragments per host and loss budget. + For EC 8+4 with six fragments on B, reconstruction while B is offline cannot + be required or claimed as existing rack protection. Report new-write + availability separately from temporary unavailability of old EC objects. +- **P12.4 Recovery and persistence**: restore B, then recreate B1 and B2 + separately on original volumes. Verify original identities, fixed checkpoints + on five replicas, and all historical data through five endpoints, including + old EC objects unavailable during the outage. +- **Extension**: to test whole-host tolerance for old EC data, separately select + a real placement within its fragment loss budget and fully read old data + during the outage. Three surviving voters cannot establish that guarantee. + +### P13 All three A-host nodes offline + +- **P13.1 Quorum loss**: stop A1/A2/A3. B reports current unavailability within + 30 seconds, and new mutations must not acknowledge success. Retain failed and + unknown outcomes; connection loss does not establish that no write occurred. +- **P13.2 Lease-safety exit**: a surviving ChunkKV exiting after its original + lease expires requires same-process evidence of lease invalidation, deliberate + stop, RPC worker join, and exit. Native abort or unexplained replacement + cannot be classified as normal lease fencing. + New runtime-shutdown panics remain failures even when recovery, data reads, + and native RPC joins pass. Retain actual exit status and backtrace when + available; a later replacement PID cannot supply either retrospectively. +- **P13.3 Recovery and protocol reads**: restore A, verify every service and + fixed checkpoint on five replicas, then audit all native/protocol history. + Independently retain each Iceberg reader's endpoint, PID, table/object/stage, + times, stdout, and exit code before aggregating results. +- **P13.4 Error attribution**: separate the first curl/GetObject error from + interrupted readers and undispatched remainders. Passing raw S3 files cannot + substitute for full Iceberg reads; concurrent output order cannot identify + an endpoint. + +## 6. Category E: allocation, backlog recovery, and space correctness + +### P14 Explicit safe retry at allocation level + +- **P14.1 Bounded fault**: preserve all KV processes and Group0 leader/term/ + lease. Inject Group1 leadership step-down; a current leader must accept at + least one injection within three seconds. +- **P14.2 Actual trigger**: start the production allocation client once and + observe explicit allocation-level Unavailable safe retry. Final success, + transport timeout, or an injection that never occurred is insufficient. +- **P14.3 Existing policy**: release injection and complete allocate/query for + the same ID within the original five-retry/ten-second limit. Retain actual + retry/ack evidence; instrumentation must not change branches or policy. + +### P15 Backlog beyond 1024 slots and full cold recovery + +- **P15.1 Actual gap**: stop a real KV follower and acknowledge at least 1100 + single writes through surviving replicas. Record its offline checkpoint upper + bound and targets before/after writing to prove an actual gap over 1024. +- **P15.2 Catch-up and cold start**: restore the follower and verify bounded + FetchGap concurrency and actual progress. Then stop all five nodes and cold + start on original volumes. Freeze Groups 0/1/3's acknowledgment targets before + the fault; all five healthy replica engines per group must reach them before + full data checks. Do not move targets with subsequent writes. +- **Extension P15.3**: separately create physical reclaimed-WAL, empty-sealed-WAL, + and high-promise/old-chosen boundaries and retain trigger evidence. An absent + trigger remains a coverage gap; component regressions or a normal large-gap + pass do not establish these physical boundaries. + +### P16 Final data and DiskDB occupancy + +- **P16.1 Final gate**: all catalog entries are Serving, five endpoints are + Ready, and 35 services are healthy. Require the same identities in the next + sample and at the end of the final audit. +- **P16.2 New and old data**: verify a new KV batch, new ChunkDB allocate/query, + and all historical data separately. Preserve distinctions among + acknowledgments, acknowledged deletions, and OutcomeUnknown. +- **P16.3 Full occupancy**: scan all this run's DiskDB occupancy read-only, + covering every disk, zone, and continuation without truncation. Verify live + mirror segments do not overlap across chunks; retain actual live, allocated, + free, and reserved counts. +- **P16.4 Cutoff and metadata-change evidence**: use one recorded scan cutoff + for busy allocations and live Chunk metadata, then retain the completion + barrier and journal fence. A bounded current-version scan is not an MVCC + snapshot: matching busy-allocation or `/chunk/` mutations during that interval + must fail the unchanged stability guard. Capture the exact failed prefix and + slot interval immediately, before journal GC or expensive failure collection, + with one bounded 20-second read-only diagnostic. Retain raw and decoded keys, + values, operations and source identities to classify the changed fields. + Diagnostic failure must preserve the original audit exception. GC gaps or + truncation leave classification unknown; they cannot establish a consistent + scan or waive fixed-cutoff, scope, completeness or non-overlap assertions. + A later unchanged scan is a separate attempt, not proof about the failed one. + +## 7. Category F: first-failure diagnosis, repair, and cleanup + +### P17 Failure evidence and classification + +- **P17.1 First error**: print the failing case, operation, and complete error; + enable `CROWDB_E2E_PRINT_SERVICE_ERRORS=1` during diagnosis. Retain original + client output rather than only a wrapper panic. +- **P17.2 Shared timeline**: retain inspect/exit/OOM, current PID/start-time, + health timestamps, KV role/term/lease/checkpoint, catalog/transfer, DiskIO, + and disk resources. For a cross-host timeout, distinguish connection setup, + request transmission, and response headers/body. Match the connection to the + current service's socket ownership. A generic client timeout alone does not + identify its phase. A scoped two-host packet capture must retain kernel + timestamps, clock bounds, snapshot/size limits, and capture drop counts; + capture drops alone do not establish network loss. Keep diagnostic helpers + outside the application's constrained cgroups, record their interference, + and retire only their exact owned identities. Reusing an observer connection + must still fetch a fresh response for every scheduled sample. Report a failed + exchange before a later scheduled reconnect; do not replay a cached body, + retry inside the sample, or extend the request timeout. +- **P17.3 Native and host evidence**: analyze crash/core before changing code + for an abnormal exit. For I/O stalls, add diskstats/pressure and accessible + blocked-thread stacks. Separate observations, inferences, and unproven causes. + A later pass does not prove an old abort without a stack or a timeout without + an endpoint label had the same cause. +- **P17.4 Complete logs**: select application, RPC, metrics, and tree logs by + actual PID, container start, and log type. Historical stderr is not a new + process error, but a nonempty old log must not hide new errors. Retain before/ + after evidence or a delta bound to this execution. + +### P18 Repair and continuation + +- **P18.1 Minimal reproduction**: find the first divergence and classify code, + ordering/injection, observer, or environment causes. Add a focused regression + that fails on old code, then run the affected group and relevant gates. +- **P18.2 Actual inputs**: rebuild, verify, and distribute images/clients after + production inputs change. Repair dependencies that make later cases slow, + hang, or fail before dispatching more work against them. +- **P18.3 Continuation ledger**: give each attempt a separate identity and + non-overwritable receipt. Retain original failures, acknowledged writes, and + unknown outcomes. Resetting baselines or moving stop markers cannot erase + observer failures. Record repairs/environment checks and new audit scope + before resuming unfinished work. Before startup or other mutations, verify + each stop/investigation marker's origin, time, and failure. Retain an old + marker and continue only after proving its failure repaired and current + identities unchanged. Unknown, new, or unrepaired markers keep blocking work. +- **P18.4 Coverage boundaries**: distinguish component, container, physical, + and soak coverage. A focused pass on a new image does not inherit all old + coverage. An interrupted workload needs a new 30-minute run. Explain whether + later recovery changes affect old soak inputs without repeating unaffected + work merely to produce a same-image pass. Sharing a completed scenario + requires matching source/epoch/image, actual fault receipts, complete + observations, and recovered identity fences. Diagnostic sampling gaps are + not continuous observation; a complete fault/audit interval requires a fresh + complete scenario. Keep an unchanged-image revalidation separate from a + causal repair: preserve the first failed attempt and carry an unexplained + failure into the final report. A passing revalidation does not explain that + failure or establish a long-term stability rate. + +### P19 Stop, retain evidence, and retire exact resources + +- **P19.1 Both-host preflight**: freeze the complete deletion inventory for + both hosts. Verify run labels, names, container IDs, original volume names/ + CreatedAt, mounts, and final stopped states/times. Stop on any change; a + matching name alone does not authorize deletion. +- **P19.2 Evidence after stop**: after stopping this run's nodes, retain complete + service logs, crashes, inspect results, and data ledgers. Finish evidence + capture on both hosts before the first deletion. +- **P19.3 Volume backup policy**: default to no additional full-volume archives + and follow this run's user retention/cleanup instructions. Retain existing + private backups. Create and verify extra volume backups only when explicitly + requested. Service logs, image transfers, and volume backups serve different + purposes and cannot substitute for each other. +- **P19.4 Exact deletion**: delete only authorized owned resources that passed + preflight, with counts exactly matching the inventory. Do not use daemon-wide + prune or clean shared runtime/other tasks' resources. +- **P19.5 Post-cleanup verification**: confirm this run's target resources are + absent on both hosts, their listeners are closed, and SSH key access still + works. Retain run artifacts, original errors, and agreed files. Disclose any + earlier out-of-scope deletion or unrecoverable data loss in the report. + +## 8. Run deliverables and acceptance conclusion + +Use `.crowdb-runtime/artifacts/physical-nodes//`. Retain actual harness +sources and commands. No single repository Pixi task currently implements this +entire catalog; do not depend on old temporary files or describe an existing +one-command runner that does not exist. + +- `run.json`: selected parent/child scenarios and extension profiles, hosts, + node/port/resource mappings, ownership inventory, source/image/client + identities, dirty diff, harness provenance, and build/transfer receipts. +- `results.json`: each scenario's PASS/FAIL/NOT RUN, independent gate start/ + limit/duration, actual injection trigger, unrun reason, attempt, and evidence + paths. Never overwrite old results. +- `data-ledger.jsonl`, `metrics.jsonl`, `checkpoints.json`: data and acknowledgment + semantics, phase statistics, fixed targets, and actual recovery progress. + Segment large files. +- `loader/` and `audits/`: protocol source versions, generation/injection/ + verification commands, complete reports, and independent endpoint results. + Mark unimplemented paths NOT RUN; other protocols or components cannot + substitute for them. +- `observations/` and `first-failures/`: continuous injection-through-audit + observations, identity fences, first errors, complete logs, crashes, and + explicitly unproven causes. +- `cleanup.json` and `summary.md`: exact retirement, actual source-generation + coverage, unfinished/untriggered scenarios, retention/loss facts, and next + steps. Distinguish ending a run from passing every selected required scenario. + +Conclusions cover only explicitly selected scenarios actually passed in this +run. Track human joint review separately; a test pass cannot complete it. + +## 9. Implementation and regression entry points + +- Startup/images: `container/oci-image/{run-node,runtime,artifact,transfer}.py`, + `container/single-node-container/entrypoint.sh`, and + `container/crowdb-monitor/src/node/startup/policy.rs`. +- Formation/observation: `lib/crowdb-protocol/src/mgmt/node/`, + `app/crowdb-web/src/node/`, + `lib/crowdb-console-shared/src/ops/cluster/readiness/`, and + `container/crowdb-monitor/src/node/runtime/`. +- Data/recovery: `lib/crowdb-{kv,kv-client,chunk-kv-client,chunkdb-client}/`, + `app/crowdb-chunkdb/src/topology/`, + `lib/crowdb-chunk-stream/src/production_chunk/`, and + `lib/crowdb-tree/src/backend/local/`. +- Components: KV backlog/FetchGap durability, ChunkStream finalized-empty, + Monitor discovery/health/service receipts, ChunkDB allocation retry/topology + availability, and BlockAsyncPageStore reads across extents. Select groups + affected by actual changes and run them through `pixi run`. +- Containers/protocols: `e2e-test/tests/deployment/`, `e2e-test/tests/console/`, + `e2e-test/tests/access_iceberg/`, and corresponding S3/native SDK clients. + Count them separately from real cross-host faults and the continuous + 30-minute workload. diff --git a/doc/doc_index.md b/doc/doc_index.md index 3b93da310..8497de03a 100644 --- a/doc/doc_index.md +++ b/doc/doc_index.md @@ -52,7 +52,7 @@ Temporary plans live under `doc/working/`; flow analyses live under | `doc/dev/crash_debugging.md` | Core collection, host setup, GDB, and exact-build symbols. | | `doc/dev/env_setup.md` | Benchmark commands, sentinels, prerequisites, and perf-counter setup. | | `doc/dev/hyper_fork.md` | Hyper fork branches, submodule, build, sync, validation, and recovery. | -| `doc/dev/physical-node-test-plan.md` | Physical-node test cases, scenarios, recovery gates and evidence. | +| `doc/dev/physical-node-test-plan.md` | Physical cases, Wi-Fi profiles, loaded latency, recovery gates and evidence. | | `tools/README.md` | Tool directories, Pixi task entry points, CI checks, and suite timing. | ## Component Acceptance diff --git a/doc/working/plan-ci-stability-review.md b/doc/working/plan-ci-stability-review.md index 768308d60..88d1c3d1f 100644 --- a/doc/working/plan-ci-stability-review.md +++ b/doc/working/plan-ci-stability-review.md @@ -3,666 +3,84 @@ # CI Stability Review Plan -Upstream: [first failed run](https://github.com/buzzcrow/crowdb/actions/runs/41003611925), -[second failed run](https://github.com/buzzcrow/crowdb/actions/runs/41128190276), -[newer failed run](https://github.com/buzzcrow/crowdb/actions/runs/41159636766), +Upstream: [Physical Node Test Plan](../dev/physical-node-test-plan.md), [container acceptance](../../e2e-test/README.md), and [CI workflow](../../.github/workflows/ci.yml). -Goal: review all 16 failed jobs individually, verify their fixes and regression -coverage, and identify the remaining work needed to improve CI stability. - -用户要求保留这份 TODO，待本轮测试、提交、推送和新 CI 完成后逐项共同 review。 -测试通过只更新验证证据，不自动勾选 review。全部 review 完成后，归档结论， -将仍需实现的工作保留到对应任务，再删除这份临时清单。 - -2026-11-01 用户调整顺序：收尾当前 RPC 生命周期修复后，优先完成 CI 稳定性 -改动与验证，然后 commit/push 并检查新 CI；之后恢复 P13 和其余物理测试。 -提交时如实保留未完成的物理验证，不将其提前勾选。 - -2026-11-01 用户恢复执行。此前暂停于 GC 派发前；恢复后加强版 GC 三例、 -辅助服务/Journal 三轮组合、Go SDK 三轮和 Dataset 分阶段验证均已通过。 -本地补充验证控制器正常退出；首次推送后的三个新失败已修复并完成本地复验， -修复已推送为 `70ae86b1`，完整 CI 为 26/27 PASS；新发现的 terminal writer -恢复故障已定位为 split 物化与父分区租约转移的冲突，修复和本地复验已完成， -已推送为 `5af2737f`，run `41304660599` attempt 1 已结束，25/27 PASS。 -terminal writer recovery 已远端通过；auxiliary 与六节点部署修复已推送为 -`18015caf`，run `41311084753` attempt 1 全部 27 个 job 通过。 -后续只读复核发现早期 placement settle 不能排除自动 split，正在补强 -Access Restart 前的实际依赖就绪观察；该测试修正仍需独立发布和精确 SHA CI。 -物理容器在恢复前已停止，原因未知；七个容器、镜像和卷身份均保持不变。 - -## 当前执行队列 - -- [x] **完成 CI 稳定性修复**：保留 auxiliary lifecycle 原有 combined 三秒 - predicate，将 `/api/servers` 的采集移出 HTTP 读路径；authority 与各节点 - 独立采样，新鲜度从采样开始计，绑定/端点/operation 与 mutation fence - 阻止旧 PID 复活。保留 Group 0 不可用时 authenticated monitor 的实际 - PID 观察，以及真实 Access readiness。补慢 peer、过期/错身份、乱序、 - 失败恢复及 sampler 退出的确定性回归。同时修复 browser batch 未派发 - peer 的错误归因，明确未执行，不能计为 PASS。 - 文件：`app/crowdb-web/src/services/observation/`、`services/remote.rs`、 - monitor 私有 process reply、`e2e-test/browser_batch.py`、`fixture.py` - 及对应测试。 -- [x] **恢复后完成最终本地验证**：受影响组件、fmt/clippy/C++ gate、共享 clients - 与新 OCI 镜像、原失败精确场景、396 个 canonical case 及并发隔离； - 每次首错停止后续派发，保留已通过项与完整错误。 - 当前 core 1165、access/monitor 2039、console 426、完整 fixture 301 项通过； - UI、C++/FFI、storage、fmt、full clippy 与 CI task mapping 通过。 - 新代码的 helper 拆分及 Rust 1.75 兼容修正后，受影响 focused 组再次通过； - 没有增加 lint allow、延长 timeout 或改变 UI/schema。新镜像已构建并核对 - production source；原 auxiliary 三秒断言通过，加强后的真实 PID/namespace、 - 周期前后 operation、注册唯一性和三节点 sibling 身份断言也通过， - Playwright 从原断言基线 9.7 秒到加强版 12.4 秒。native-operations 全组通过。 - 新镜像全量已收尾：396 个 canonical case 的最新最终结果全部 PASS， - 必需的四例并发隔离通过，七个保留物理容器及卷身份不变。加强版 CLI - 五例随后全部通过，用时 102.757 秒。恢复后正常刷新 clients：GC 三例 - 通过；浏览器组合曾在第二轮发现 Access 依赖恢复边界缺失，修正后三个 - 全新组合轮次全部通过，约 45–46 秒。Go 三轮普通和注入后 cleanup 均通过。 - Dataset browse client 278.32 秒，其中 seed 2.276 秒、默认 balance - 260.002 秒、512 条完整回读 0.273 秒；12 partitions、三个 owners。 - 最终 fmt/full clippy、受影响 C++ format/tree-lint、TypeScript 和 scope - guard 通过。新增断言均有独立实际运行证据，不能与旧 bundle 的全量混算。 -- [x] **提交并推送修复**：CROWDB `task-fix-build` 一个一致提交； - 记录精确 SHA、run/attempt 和全部展开 job，修复真实失败后推进。 - 已推送 `76fc6508189dd31018e953674a8f1c693011498d`，新 run `41295426521` - attempt 1 已结束：27 项中 24 成功、3 失败。CI merge checkout 与 head - 的完整 tree 一致。三个新失败的本地修复与受影响组复验均已通过，已推送 - `70ae86b1c62e095be2754ed178890736e49be5a4`；新 run `41299662435` - attempt 1 已结束，26/27 PASS，只有 console-api 的 terminal writer recovery - 失败；上一轮三个修复场景已远端通过。全部日志和校验过的失败 artifact - 保留于 `ci/published-70ae86b1/`。此前完整日志与三个 artifact 保留在 - `ci/published-76fc6508/`，不并入原 16 项人工 review。 - 最新修复已推送为 `1814405e`；本地新镜像 19-case 操作组、UI 369 和 - fmt/clippy/TypeScript 全部通过，提交钩子通过。 -- **CI 检测已按用户要求停止**：不再查询 `1814405e` 的 run `41320020131`， - 不推断最新结果，保留已收集的证据；不作为物理工作的前置。 -- [~] **恢复物理验证**：用户取消新增卷归档。B 重启后已连通并核对主机身份， - 七个原始停止容器及卷身份均通过检查。镜像传输及 SHA-256 校验通过； - A 使用 manifest ID、B 使用 config ID，已通过内容哈希及完整层列表证明一致。 - 五个当前节点已用原卷升级，完整历史审计通过，包括五端点 S3/Iceberg SF1、 - 187 份 Dataset 报告及全部见证/删除记录。625 次观察通过，35 个身份不变。 - 新 P09 在 A2 Group 0 复现真实的跨块异步短读，已停止五个节点保留原卷， - P09–P19 尚未完成。最小回归重现完全相同的 958464/1212416 字节错误； - 跨块读取修复、19 项 focused 检查及完整 C++/FFI、core、storage、fmt、 - tree-lint、clippy 门禁均通过，已推送 `c063a169`；新镜像同卷升级通过。 - 新启动全部 35 服务在 37.439 秒恢复，原生历史与五端点 S3 SF1 通过。 - Iceberg 校验时 A 主机 NVMe 约十秒 I/O 停顿，随后 monitor 过期、Group0 - 超时和五个 ChunkKV 租约退出；OOM 为零，设备/文件系统根因尚未证实。 - 单对象原限制全读/校验通过；新的完整只读历史审计随后通过，1230.088 秒， - 597 次观察、最大间隔 3.843 秒、35 身份不变；187 份 Dataset 报告及全部 - 五端点协议/见证/删除记录通过。原失败不改写；c063 的新 P09 完整通过（1164.469 秒），P10 也通过 - （1023.819 秒）。P11 follower 完整审计通过；leader 恢复门禁通过但 B2 - monitor 观察过期 3519 ms，后续 P12–P19 未派发。 - 根因是本地健康发布前同步等待 Group0 desired-service scan。旧代码的慢 - authority 回归失败；改为一个有界后台只读任务并保留操作前权威 fence 后， - focused 八项通过；完整 fmt/clippy、Access/Monitor 2041、Console 426 及正常 - task 附属检查通过。未改动的 C++/core/storage 门禁保留 c063 输入证明。 - 发布、新镜像和受影响物理复验待完成；五节点已停止、原七卷保留，失败和 - 修复证据分别留存。可复用物理方案已按测试用例重组。 - 未查询新 CI。 - 这不是新增 CI job 失败，不并入原 16 项共同 review。 - A 已完成的三个备份和全部原始失败证据继续保留，不再新增备份。 - -## Run 41295426521：首次发布后的三个新失败 - -- **Chunk allocation 丢回复注入**：旧代理丢弃首段 TCP 数据，其中可能只有 - `0xfffe` capability handshake，allocation 尚未执行，busy bytes 仅为 1 MiB。 - 复用共享 frame-aware 代理，要求恰好转发一次握手、丢弃一次 application - response，保留 Unreachable、精确 2 MiB 和不重放 allocation 的断言。 - 原精确/默认 scope/串行基线均通过；该 scope 只有一个 case，不能据此声称 - 复现了多 case 竞争。修复精确场景通过；完整失败 job 的 47 个剩余场景 - 本地通过，保留三个未受影响的远端 PASS。 -- **Shell lazy inventory**：原 trace 在 N-3 选中后、DOM 断言返回前发生正常 - topology refresh；四次 inventory 均只请求 N-3，均返回 200。现按 topology - 观察周期精确断言 `[3]`，保留初始零发现、全部 DOM 断言和三秒期限。 - 原精确基线 4.2 秒，修复后 3.9 秒；原 shell/capacity 并发组合分别 - 4.8/10.8 秒通过。生产 UI 不变。 -- **CLI final readiness**：自动 split 发布 generation 2 后，旧 grant 短暂 - StaleCatalog，最终即时 check 返回 degraded；真实进程仍存活。最终 check - 现在仅使用启动时原 180 秒绝对期限的剩余量，调用一次官方 `--wait`，保留 - Ready、cluster identity 和整个等待前后的 lifecycle equality。过期不派发、 - 失败不重试、错 cluster 拒绝均有回归；不新增 mutation replay 或等待预算。 - 原精确和五例并发基线通过；修复后六例通过，包含被独立 fixture 首错中断的 - terminal writer recovery。该 peer 不是另一个已证实的产品失败。本地修复轮 - 没有复现原 StaleCatalog 窗口，不能以快速 PASS 声称复现了该时序。 -- **门禁与保留**：fixture 304 tests、fmt、full clippy、TypeScript、diff check - 全部通过。生产 source tree 仍为 `376fdc26a0e96d7810313ea670bf043c4f0dfcae`； - 正常 clients builder 已刷新对应输入。七个物理容器及卷身份未变；尚未进行 - 物理故障、升级或退休。完整执行证据见 `local-repair-closure.json`。 - -## Run 41299662435：split 物化与租约恢复 - -- 完整 CI 为 26/27 PASS；console-api 的 92 例中 91 例通过，只有 - `terminal_chunk_writer_recovery` 在原 180 秒恢复期限内未恢复。 - 故障解除后的首次持续分歧是 Group 0 monitor 的 exact writer pair 错误， - 80 次错误阻止后续 serving grant 更新，最终两个 ChunkKV 范围 LeaseExpired。 -- 已提交 split 的父分区因租约失效转移后，其 marker 正常更新为 transfer ID； - 子分区独立可恢复时，旧物化逻辑仍要求两个 split marker。确定性回归在 - 旧代码重现同一错误，也发现仅计数会接受复用 marker 的错误身份。 -- 修复从持久化 split proof 核对每个剩余 marker 的完整 assignment；只有 - CatalogCommitted 才允许另一分区已被后继转移取代，并保持其新状态不变。 - ChildPrepared 仍需两个精确 successor，旧 epoch、错误 artifact 和复用 - marker 均拒绝。原超时、错误处理、故障时长及写入次数保持不变。 -- 精确容器基线通过，但不据此声称复现远端时序。旧代码确定性回归两例失败， - 修复后的完整 KV Server 86 tests 通过，包含四个新回归。全新生产镜像、 - ChunkKV Server 112 tests（另有一个 opt-in RDMA ignore）、fmt 和 full clippy - 均通过。新镜像精确 terminal 151.1 秒通过，恢复 check 21.2 秒；默认并发 - CLI 九例、ChunkKV 两例、balance 一例和 console-api 剩余 83 例全部通过， - 共 95 个不同 canonical case。七个保留物理容器及卷身份不变。 - 证据：`ci/split-materialization-fixed/`；已推送 `5af2737f`，其精确 SHA - CI run `41304660599` attempt 1 已结束；原 terminal recovery 已通过。 - 此前 396-case PASS 属于旧生产镜像，不能将两轮输入混为最终镜像全量验证。 - -## Run 41304660599：依赖等待与生命周期前置条件 - -- 全部 27 个 job 日志和两个失败 artifact 已保留并校验；25 PASS / 2 FAIL。 - `console-native-operations` 为 18 PASS / 1 FAIL，`console-native-data` 为 - 13 PASS / 1 FAIL。六节点 trace 为 27.4 MB，超过原 16 MiB 导出上限， - 远端未保留；该部分只能依据完整失败日志与本地重现诊断。 -- Auxiliary 外部 Stop、OS 进程退出和页面 Failed 均通过，随后 Restart - 在 ChunkKV 初始放置的 TransferFencing 窗口返回 409 Waiting。原精确 - 场景 12.4 秒 PASS；原并发组合 12.1 秒复现；串行对照 13.7 秒 PASS。 - 首次修复沿用 placement settle，后续复核发现其 transfer observation - 不覆盖同一或下一 tick 的自动 split。现改为每次 Access Restart 前直接 - 观察同节点 ChunkKV /ready 与 instance identity，保留三秒观察期限， - 不调整生产 balance 策略、response/PID/OS/sibling/readiness 断言。 -- 六节点 Access Waiting 缺少分类，本地另复现带换行的 ChunkKV Waiting - 不匹配原正则。monitor 已恢复实际进程，计划仍为 failed。新增两例确定性 - 回归先红；修正分类后又捕获同 waiting 状态保留旧 detail，导致已登记启动 - 误用十秒缺盘退避。修复按已有 CLU-03/SVC-03 契约观察目标 monitor， - 回归要求最终 deployed，且 deploy 一次、restart 零次。 -- 原有序并发组在更早的 KV lifecycle 失败；精确串行同样失败。停止 Group 0 - leader 后过早提交 restart，权威校验三秒到期。增加同 cluster/operation - 的权威恢复观察，使用已有选举预算，再提交一次 mutation，保留响应预算。 -- 新镜像的 48 个独立 case 全部通过：native operations 19、browser 27、 - 六节点和固定端口/失败重试两例。先通过三个精确场景，再在原有序生命周期 - 组复验。六节点两次 Playwright 为 51.2/49.2 秒；固定端口为 5.4 秒。 - 数据组虽配置 concurrency 2，资源预算实际使两例顺序执行，不能声称并发。 -- UI 366、plan-hook 28（含两例新回归）、fixture 304、完整 TypeScript、 - fmt/full clippy 均通过。七个物理容器及卷身份未变。证据保存在 - `ci/lifecycle-dependencies-fixed/`；已推送 `18015caf`，对应 run - `41311084753` attempt 1 全部 27 个 job 通过，完整日志保留。 - 原 16 项人工 review 保持未勾选。 - -## Run 41311084753 后：Access Restart 的实际依赖边界 - -- 27/27 PASS，上一轮 auxiliary 和六节点部署失败组均通过；CI merge - checkout 与 HEAD 完整 tree 相同。源码一致不表示本地与 CI 二进制相同。 -- 复核 balance monitor 发现 transfer observation 可报告 within tolerance， - 同一 tick 仍可规划 split。早期 settle 通过不证明后续 Restart 依赖就绪。 - 新辅助函数在两处 Access Restart 前 GET 本机 ChunkKV /ready，核对 - instance_id 与真实 ready，再保留原单次 mutation 和响应断言。 -- 首次新单例因使用不存在的 url 字段报 Invalid URL；已改为 API 实际 - mgmt_url，保留失败证据。修正后精确单例 35.6 秒通过，原并发组合中的 - auxiliary Playwright 14.5 秒通过。完整操作组为 17 PASS / 1 FAIL / 1 未执行。 - 生产镜像输入未变，继续使用 lifecycle-fixed 镜像并核对 source projection。 -- 新失败为 S3 multipart 删除后旧行超过原三秒期限。完整 trace 显示 - DELETE 204 用时 1.279 秒，后续 list 200/KeyCount 0 用时 2.036 秒； - 页面成功删除后仍等待 list 才清除缓存行。精确单例和并发组合对照通过， - 串行对照再次复现同一断言。建议收到成功后立即移除精确 Bucket/key 行， - 再执行原刷新；失败/未知结果保留行，原三秒 E2E 断言不变。 - 用户已同意该行为，先更新了 S3-04 和 ASCII 布局。可控延迟列表回归在 - 旧 UI 复现原行未移除，单行生产修正后三项通过，覆盖精确 key、其他行、 - 刷新失败以及删除拒绝/未知结果。初始测试 provider/字段标签错误单独保留， - 不作为产品 red 证据。新镜像已通过精确 multipart（Playwright 4.6 秒）、 - 原 auxiliary 并发组合（auxiliary 12.7 秒）和全部 19-case 操作组 - （417.230 秒；multipart 再次 4.1 秒通过）。UI 369、TypeScript、 - fmt 和 full clippy 通过；新测试不支持的 exact 选项所致类型错误与 - 修正后的门禁分别保留。新的提交、推送和 27-job CI 尚待完成。 - 证据：`ci/access-readiness-fixed/s3-delete/`、`ci/s3-ui-fixed/`。 -- 物理 observer 的离线回归补充了恢复后固定 35 个进程身份的 fence， - 数据审计期间不再沿用故障目标的宽松重启许可。两项 fence 回归及七项 - closure 接受性检查通过；未派发物理启动、故障、归档或退休。 - -## 通过场景的稳定性改善 - -用户要求同时处理 PASS 但不稳定的场景。本节记录额外改善与待验证边界， -不增加历史失败 job 数，也不代替后面的 16 项人工 review。 - -- **GC 与真实 SDK 的重叠窗口**：前一镜像 matrix 虽通过，实际删除增长与 - 前台工作同时活跃的七个窗口合计仅 1.763 秒；冷启动 SDK 发生在已删除 - 一个文件后。现先在可执行 purge marker 发布前启动四个真实 SDK worker， - 等 namespace 请求成功并 parked，再执行原三次重启，直到实际删除边界 - 才放行请求与小写入。warmup、seed、run 共用原 720/3300 秒绝对期限。 - 保留九文件、三次重启、64 次小写入、六类各至少 48 次成功 SDK 操作和 - 成功请求完成时刻落在真实删除增长区间的断言。独占进程组正常清理失败 - 必须使 case 失败；取消与 Drop 保留首错及完整 stdout/stderr。 - 六项纯协议回归、full clippy 和独立只读复核通过；三个受影响真实 GC - case 在新 bundle 均已通过，队列用时 480.220 秒。convergence/matrix - 分别有 240/81 个 SDK 完成时刻落在 5/7 个真实删除区间，均保留 64 次 - 写入；matrix 三次重启先于放行。证据：`resume-gc-proof.json`。 -- **Journal 刷新响应归属**：原 pathname-only waiter 可被选择 partition - 自动产生的 GET，或同路径的旧 cursor/树页请求满足。现先完成选择时的 - 自动观察，再关联本次 Refresh 的具体 Request 和完整资源/cursor， - 等响应 body 完成并核对 Journal record。全部新增等待共用原三秒预算， - owner-loss、restart、stale-cursor 断言保留。四项事件回归和完整 E2E - TypeScript 检查通过；最终与 auxiliary 配对三个全新轮次均通过。 -- **辅助服务故障的依赖恢复边界**：恢复后的浏览器组合第二轮在 ChunkKV - 已返回新 PID、Access 尚未恢复时进入下一个周期，触发原即时 `up` 断言。 - GET inventory 为 0.3–12 毫秒，Access PID 未变化。精确单例和原组合串行 - 基线通过；原并发组合的只读采样确认依赖故障期间真实 readiness 503， - 后续恢复 200。现把同节点 Access 恢复加入已有三秒 replacement predicate， - 保留即时检查、真实 PID/namespace/operation、兄弟进程和注册唯一性。 - 没有额外等待窗口、sleep、超时放宽或 mutation 重试。随后三个全新配对 - 轮次均通过；首错和诊断保留在 `ACCESS-DEPENDENCY-CLOSURE.md` 及 - `first-resumed-browser-failure/`。这不是生产 false-down 根因修复的声明。 -- **CLI 生命周期快照**：本轮全量首错为 - `plain_docker_target_attachment` 的 `KeyError: 'pid'`；cluster Ready - 不承诺独立 `/api/servers` cache 瞬时填齐可选 PID。现核对 owned - 容器/镜像/binding，读取 21 个服务的 monitor PID/generation/operation， - 并在对应容器 `/proc` 拒绝已经退出的旧正 PID；显式要求 Web 加六类服务。 - 同时保留 18 个唯一注册、所有端点字段与已有 cached PID/namespace - 一致性检查。四项回归通过；最终加强版五例已在新 bundle 全部实际执行 - 并通过，用时 102.757 秒。没有对可选 PID 加等待或调用方重试。 -- **Dataset 准备阶段余量**：前一镜像五个相关 client 为 280.68–288.44 秒， - 其中 browser 仅 4.72–11.20 秒，主要耗时为正常 production balance - 收敛，外层 client 为 360 秒。不能声称并行 seed 能解决约 270 秒的主耗时。 - 保留 512 条记录、至少 12 partitions、三个 owners、默认 policy 和原期限， - 新增 seed/balance/逐条验证的独立阶段时间；本轮及新 SHA CI 继续核对余量。 - 当前新镜像的全量验证中，browse 为 281.284 秒，import 为 321.901 秒， - recovery 为 331.468 秒；后两者离 360 秒外层期限仅余 38.099/28.532 秒。 - 两者到 272 秒仍只有九个 partitions，302 秒才观察到十二个；尚无证据 - 表明条件已满足后存在可去掉的空等。30 秒为测试日志节流，生产 planner - 实际为两秒 heartbeat，不能把打印间隔当成决策周期。只读审计及下次复现 - 所需的有界 planner/transition/freshness 证据见 - `ci/stability-final/DATASET-HEADROOM-AUDIT.md`。 - 本轮新增计时的 browse 已通过，client 278.32 秒，默认 balance 占 - 260.002 秒；该项尚未证明有行为级性能改善，不能仅因一次通过关闭风险。 -- **有针对性的重复验证**：新的辅助服务/Journal 组合与 Go SDK cleanup - 各三个全新 fixture 轮次均通过；此前首错停止记录完整保留。 - Go 原始 cleanup 原因仍未知；重复通过只提供当前覆盖，不能声称已找到 - 历史根因或证明长期失败率。执行记录保留在 `ci/stability-final/`。 - -## Review 规则与证据 - -- 两次原始 run 分别有 27 个 job，失败数为 8 和 5；没有 cancelled job。 -- 用户追加的最新 run `41159636766` 同样有 27 个展开后的 job，24 成功、 - 3 失败、无 cancelled；attempt 为 1，head 为 - `8885d2adc351d9c99f0219e18ecf0c4782bc3079`，分支为 `task-fix-build`。 - 原始 13 项保留，新一轮逐项追加为 14–16；相同 job 名不代表同一个首错。 -- 原始测试版本分别为 `39440d023d7fb162dd61c377e8eb17eefa8c4c9a` 和 - `25059cf7d9a5f08d61479f4749659aa6f3fe119f`。 -- 已获取前两轮全部 13 个失败 job 的完整日志、7 份原始诊断 artifact，并逐项映射 - 32 条容器失败记录及 2 个非容器失败 case。 -- 第三轮另有 3 份完整失败 job 日志和 3 份诊断 artifact；下载档案的 SHA256 - 均与 GitHub metadata digest 相符。累计 16 个失败 job、35 条容器失败记录， - case 名称有跨 run 重复，不能说成 35 个不同 case。 -- 区分首个真实错误、同一故障的后续症状、同批启动失败导致的未执行项、 - 故意注入的失败和包装层 panic。不能把它们都算作独立根因。 -- 保持原有超时、断言和调用方重试策略；修复真实前置条件、竞态和资源隔离问题。 -- 单次 PASS 是覆盖证据，不是长期失败率结论。发布后的精确 SHA、所有子 job、 - attempt 和结果也必须记录。 -- 已有组件与前一镜像的完整 396-case 账本均通过。上一版物理 P12 修复镜像 - 全量 396 个 canonical case 及并发隔离均通过；共 400 个完整 fixture，0 失败。 -- 本地证据保存在 - `.crowdb-runtime/artifacts/physical-nodes/physical-20261031T194149Z/ci/prior-ci-audit/`： - `REPORT.md`、`job-matrix.json`、`case-coverage.json`、完整 job 日志及原始 artifact。 - 第三轮另见 `run-41159636766/REPORT.md`、`job-matrix.json`、 - `artifact-sha256.json` 及 `current-exact-case-validation.json`。 - 该版记录位于同一 run 的 `ci/repair-target-fixed/`；当前新验证队列位于 - `ci/stability-final/`，不能将旧镜像的 PASS 直接归给新增 RPC/观察路径修复。 -- 上一版 P12 修复镜像输入 tree 为 `76f794e03757249f37efa96cd0bfcad064be4388`； - 32 条历史容器失败记录所对应的 case 已全部在此镜像实际执行并通过， - 精确映射见 `new-image-case-progress.json`。全量结束后的 scope guard 确认 - 原有 7 个停止容器及卷身份完全不变。物理恢复仍须完成；用户随后取消 - 继续检测 CI，不再要求新提交的远端 CI 作为物理验收前置。 - -## Run 41003611925：8 个失败 job - -- [ ] **01 / AccessTests**：review accepted policy 的测试前置条件。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123893403740)。 - - 首错：`retained_quorum_startup_does_not_block_discovery_or_confuse_liveness_with_health` - 在 retained quorum 启动时返回 `accepted node policy is absent`。 - - 根因已确认：fixture 未先接受启动所要求的 node policy。 - - 当前处理：启动 trust/profile 前先 `accept_node_policy`，明确 3 voters。 - 文件：`container/crowdb-monitor/tests/kv_resume_test.rs`。 - - 验证：后一次原始 run 的该 job 成功；本地 access 完整组及 exact case 通过。 - - Review：policy 接受顺序是否符合真实启动流程；是否仍分别验证 discovery、 - liveness 与 health；新 SHA 的 AccessTests 结果待补。 - -- [ ] **02 / ContainerDeployment (deployment-reference)**：review diskless - policy 对 Group 0 的要求。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123899615882)。 - - 首错：`direct_docker_three_nodes` 初始化 HTTP 500，明确报告 - `accepted node policy has no enabled Paxos KV listeners`。 - - 根因已确认：diskless/manual fixture 把初始化 Group 0 所需的 Paxos KV 也禁用了。 - - 当前处理：共享 setup choices 保留已接受的 KV listener 10000/10100。 - 文件：`e2e-test/profiles/setup_choices.py`、`e2e-test/console_native_policy.py`。 - - 验证：前一镜像 exact case 与 fresh reference 重跑通过；P12 修复镜像 - Docker 三节点完整通过，约 215 秒。 - - Review：diskless 与“无 Group 0”是否正确区分；单/三节点、保留卷、显式非 voter - 恢复及 Compose 覆盖是否完整；清理必须只作用于自己记录的资源。 - -- [ ] **03 / S3E2E**：review Go SDK 注入后的 cleanup 证据缺口。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123899615935)。 - - 首错：`s3.sdk::go` 普通流程先通过；`after-mpu` 故意失败本应返回 42， - 后续 `cleanup FAILED` 把退出码改为 1。 - - **旧根因未确定**：日志没有具体操作/API code，原始 artifact 也无法区分 - ListMultipartUploads、AbortMultipartUpload、ListObjectsV2、DeleteBucket 等失败点。 - 不能声称 cleanup 算法已被修复。 - - 当前处理：记录脱敏的 Smithy operation/API error code；保留原 cleanup 流程与期限。 - 文件：`e2e-test/clients/s3-go/main.go`、`multipart.go`、`e2e-test/clients/s3-sdk.sh`。 - - 验证：第二次原始 run、前一镜像完整验收及P12 修复镜像 exact case 均通过； - 本轮日志明确确认注入后的 bucket/MPU cleanup 完成。 - - Review：确认此项保留为历史原因未知；核对新 SHA 的 Go 场景。 - 若复现，用独立 artifact root 跑 `s3.sdk::go`，保留增强错误、Access 请求日志及 - bucket/MPU 只读状态后再修复；不增加重试、不放宽清理成功条件。 - -- [ ] **04 / ContainerConsole (console-native-data)**：review 9 个数据/拓扑场景。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123899615992)。 - - 首错：Dataset browse 初始化的 disabled-KV policy 错误；其余表现为 - No route、SSH 断开和 socket 错误。 - - 根因已确认：9 个实际失败 fixture 的节点原始日志均有 policy 错误和 exit 1， - 发生在各自网络症状之前，不能归因为跨 job 的共享 Docker 清理。 - - 当前处理：复用保留 Group 0 listener 的 diskless policy。 - 文件：`e2e-test/profiles/setup_choices.py`、`e2e-test/console_native_policy.py`。 - - 验证：前一镜像 9 个 exact case 全通过；第二次原始 run 的该 job 成功。 - P12 修复镜像 9 个 exact case 也已全部通过。 - - Review：逐一核对 Dataset browse/Split/capabilities、六节点往返、三种多 rack/ - 子集拓扑、mixed units 和 maintenance/status badges；启动失败与实际 UI 断言分开。 - 同时检查预算边界：`e2e-test/scenarios.py` 中 Dataset browser client 总期限为 - 360 秒，而 `e2e-test/tests/console/native_balance.rs` 的 - `wait_balanced` 内部上限为 600 秒；本轮 browse/import 的 - 总 client 时间约 287 秒。明确各阶段应达到的准备条件和计时，不直接延长期限。 - - 对应 case（均以 `console::` 开头）：`browser_dataset::browse`、 - `browser_dataset::split_revisions`、`browser_dataset::capabilities`、 - `browser_data::six_node_round_trips`、`browser_topology::multi_rack_leaders`、 - `browser_topology::independent_subsets`、`browser_topology::simple_and_complex`、 - `native_geometry::mixed_units_reject_consumers_without_starting_services`、 - `native_browser::disk_maintenance_and_status_badges`。 - -- [ ] **05 / ContainerConsole (console-browser)**：review 两个 Group 0 bootstrap 场景。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123899616023)。 - - 首错：replica navigation 和 logical create 的 public bootstrap 返回 No route。 - - 根因已确认：两个 fixture 的三个节点均先因 disabled-KV policy exit 1， - inspect 中 IP 已被清空。 - - 当前处理：共享 accepted diskless policy 保留 Paxos KV。 - 文件：`e2e-test/profiles/setup_choices.py`、`e2e-test/console_native_policy.py`。 - - 验证：两个 exact case 的前一镜像完整验收通过；第二次原始 run 该 job 成功。 - P12 修复镜像两个场景均已实际执行并通过。 - - Review：不把已退出进程的连接错误当成独立网络缺陷，核对 bootstrap 后的实际断言。 - -- [ ] **06 / ContainerConsole (console-inspection)**：review inspection 启动与 UI 边界。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123899616075)。 - - 首错：`page_layout_and_iceberg_inspection` 在 bootstrap 阶段 No route。 - - 根因已确认：三个节点原始日志有 disabled-KV policy 错误、exit 1、OOM false 和空 IP。 - 失败发生在 inspection 断言之前。 - - 当前处理：统一 accepted policy；文件同上。 - - 验证：前一镜像 exact case 通过；第二次原始 run 该 job 成功。 - P12 修复镜像进入全部 13 个 Playwright 断言并通过，总 client 约 326 秒。 - - Review：核对布局和 Iceberg inspection 的实际断言，区分 fixture 启动错误。 - -- [ ] **07 / ContainerConsole (console-transition)**：review journal transition 的真实前置条件。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123899616076)。 - - 首错：`production_split_and_transfer_journals` bootstrap 超时。 - - 根因已确认：node0 先因 disabled-KV policy exit 1，随后 bootstrap 等待超时； - 不能把这次失败说成 journal 状态转换断言失败。 - - 当前处理：统一 accepted policy；文件同上。 - - 验证：前一镜像 exact case 通过；第二次原始 run 该 job 成功。 - P12 修复镜像实际执行 split/transfer journal 断言并通过。 - - Review：检查 split/transfer journal 的状态转换，保留原有等待预算。 - -- [ ] **08 / ContainerConsole (console-native-operations)**：review 真实启动失败与同批级联。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41003611925/job/123899616153)。 - - 首错：`diskdb_and_kv_independence` 初始化中的 SSH authentication failed； - 该 job 共打印 11 个失败 case。 - - 根因已确认：10 个实际失败 fixture 均有本地 disabled-KV policy exit 1。 - `binding_recommendations` 已准备但未派发浏览器：后一个 fixture 启动失败， - `ExitStack` 将同一异常传给已进入的 peer，后者按自己的 case 名打印失败。 - - 当前处理：修复实际 policy 前置条件；没有跨 fixture 凭据或 SSH target 共享的证据。 - 后续新增 `CaseNotExecuted`，同批未派发 peer 保留完整清理，并在 fixture、 - report 和 measurement 单独记为 `not_executed`，不能计作 PASS 或独立 case - 失败。真正的启动错误和 peer 自身的诊断/清理错误分别保留，batch 仍失败。 - 正常 client 后的清理失败打印自己的 case；已有首错时，后续清理错误单独 - 打印且保留原异常优先，不增加重复的 `FAILED CASE` 记录。 - 文件：`e2e-test/browser_batch.py`、`e2e-test/fixture.py`、 - `e2e-test/fixture_result.py`、`e2e-test/report.py`、 - `tools/test-metrics/measure.py` 及对应 Python 测试，以及共享 policy。 - - 验证：前一镜像 11 个 exact case 均通过；partial-startup 回归确认不派发 clients - 且清理已启动 peer。P12 修复镜像 11 个场景也全部通过。新增归因回归在旧实现 - 确定性失败，修复后完整 fixture suite 290 tests 通过；没有操作真实容器。 - - Review：逐项核对这 11 个场景及其实际派发状态，并核对新的未执行统计、 - startup/client/cleanup 各自归属及 fail-fast 清理；不将它扩大解释为凭据共享 bug。 - - 对应 case（均以 `console::` 开头）：`browser_lifecycle::diskdb_and_kv_independence`、 - `browser_capacity::binding_recommendations`（原 run 未执行）、 - `browser_capacity::mounted_crud_and_owner_binding`、`native_browser::auxiliary_process_menus`、 - `native_browser::interrupted_plan_reconciliation`、 - `native_browser::storage_windows_and_access_recovery`、 - `native_browser::large_journal_records_and_restart`、 - `native_load::production_load_keeps_heartbeats_and_durable_bytes_across_owner_restart`、 - `native_browser::chunk_ownership_scope`、 - `native_browser::multipart_upload_and_physical_navigation`、 - `native_binding::prebound_groups_acquire_live_owners_and_require_explicit_binding_repair`。 - -## Run 41128190276：5 个失败 job - -- [ ] **09 / UnitTests**：review leader role 与可提交状态的区别。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41128190276/job/124312015280)。 - - 首错：`g3_leader_change::leader_change_simulation` 的第二笔单次写入未通过新 leader 提交。 - - 已支持的原因：旧等待只看 leader role；focused regression 证明 role 可能先于 - 当前 proposing term 和有效 quorum lease。旧日志没有保留该次 RPC reply code， - 不应拿后续故意注入的 WAL 错误替代首错。 - - 当前处理：等待既有 proposing-term/quorum-lease readiness，并保护健康 leader - 在当前 quorum lease 内不投票离任。 - 文件：`lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs`、 - `lib/crowdb-kv/src/cluster/local_replica_election.rs`、 - `lib/crowdb-kv/tests/election_test/lease_test.rs`、 - `lib/crowdb-kv/tests/election_test/vote_test.rs`。 - - 验证：exact case、role-before-lease 和 quorum-lease vote 回归均通过； - 本地 core 1165 tests / 91 binaries 通过。 - - Review：准备条件不能替代提交断言；检查选举/恢复交互、无新增热路径锁， - 并核对新 SHA 的完整 UnitTests。 - -- [ ] **10 / ContainerDeployment (deployment-services)**：review KV-only readiness 与实际端口。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41128190276/job/124321492635)。 - - 首错：`monitor_creates_real_kv_groups_then_validates_after_restart` 与 - `selected_kv_services_recover_acknowledged_data` 一直不能 Ready。 - - 已确认问题：diskless KV-only policy 被要求提供普通 storage metadata；member - 收集还假定 Web 使用 9090，而未使用已注册端口。 - - 当前处理：按 `requires_default_preparation` 收集确实需要的依赖，使用 `node.web_port`。 - 文件：`lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs`、 - `lib/crowdb-console-shared/src/ops/cluster/readiness/collect/members.rs`、 - `lib/crowdb-console-shared/tests/cluster_readiness_test.rs`、 - `lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs`。 - - 验证：前一镜像 deployment-services 完整组、两个 exact case，以及 - KV-only offset-port/no-storage-preparation 回归通过。 - - Review：检查依赖来源符合 accepted policy，普通 storage readiness 仍严格验证； - P12 修复镜像两个 exact case 已通过；新 SHA 完整 job 仍须核对。 - -- [ ] **11 / ContainerConsole (console-api)**：review listener 来源和 terminal fault 的作用域。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41128190276/job/124321492786)。 - - 首错：plain Docker attachment 向旧 S3 端口 9091 请求；terminal writer - 检查旧 health 端口 9094，连接失败。 - - 当前处理：plain Docker public URI 改为此 fixture 使用的 S3/Iceberg/Dataset - 端口 16000/17000/18000；terminal health 从 accepted listener policy 读取。 - terminal 用例在 owned network namespace 注入 DiskIO outage。之后本地真实复现的 - ChunkKV 同步 native I/O 阻塞 executor/lease 问题另由 worker offload 修复。 - 文件：`e2e-test/console_cli_fixture.py`、`e2e-test/console_cli_recovery.py`、 - `e2e-test/console_cli_target.py`、`e2e-test/console_cli_io_fault.py`、 - `lib/crowdb-chunk-kv/src/partition/native.rs`、 - `app/crowdb-chunk-kv-server/src/storage.rs` 及相关回归。 - - 验证：accepted-listener 回归、console-api 完整组和两个 exact case 通过； - P12 修复镜像这两个 case 已完整通过。 - - Review：区分旧端口错误与后来复现的 liveness bug；故障确实作用于 I/O， - 原 lease/terminal replay/Stop fencing 断言和资源限制保留。 - -- [ ] **12 / IcebergE2E (iceberg-gc-cleanup)**：review 前台负载与实际删除窗口的重叠。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41128190276/job/124321492803)。 - - 首错：`restarted_cleaners_finish_large_and_sparse_backlogs_beside_an_invalid_catalog` - 未观察到成功 SDK request 的完成时刻落在观测到的 active-deletion 区间。 - - 根因证据：GC 在约 117 秒完成并执行了 3 次 restart；有限、同步步调的 - 48 cycles 没有保证请求与删除窗口相交，负载还先于 cleanup 结束。 - 这不证明 GC 自身未完成。 - - 当前处理：压力持续至 GC 和小写入均结束，保留每类最少 48 次操作， - 错开 4 个 workers，继续要求真实成功请求重叠；FileReader leaf cache 的 - 性能修复单独记录。 - 文件：`e2e-test/tests/access_iceberg/gc/sdk_pressure.py`、 - `e2e-test/tests/access_iceberg/gc/convergence/pressure.rs`、 - `e2e-test/tests/access_iceberg/gc/notifications/matrix.rs`、 - `e2e-test/tests/access_iceberg/gc/notifications/matrix/records.rs`、 - `lib/crowdb-access-iceberg/src/file/reader.rs`。 - - 验证：focused 重跑、前一镜像完整验收通过；新镜像 exact matrix 约 123 秒通过， - inline GC 前台并发及完整退休宽限期也通过。 - - Review：确认 overlap 是从实际事件/单次请求时间计算，原断言未放宽； - 不从该场景推断单独的生产 latency SLO。 - -- [ ] **13 / ContainerConsole (console-native-operations)**：review journal UI 的 coherent observation。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41128190276/job/124321492894)。 - - 首错：`journal_identity_and_owner_recovery` 导航后，current-owner 条件 - 在原 3000 ms 窗口内未选出 coherent journal observation。 - - 当前处理：Rust fixture 先创建 owned witness、等待已有分区的 ownership 稳定； - 浏览器验证 witness 并保留同一 identity/lifecycle 断言和 step timing。 - 之后本地 native executor/lease 复现由 offload 修复。 - 文件：`e2e-test/tests/console/native_browser.rs`、 - `e2e-test/tests/console/native_balance.rs`、 - `e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts`、 - `lib/crowdb-chunk-kv/tests/native_runtime_test.rs`。 - - 验证：focused native-journal 约 28 秒通过，前一镜像 exact case 通过； - P12 修复镜像对应场景约 16 秒通过，原 3 秒 UI 断言不变。 - - Review：检查 fixture 的真实前置条件、UI 的 current-owner/epoch/cursor 一致性， - 保留 3 秒断言以及 owner-loss/restart/stale-cursor 场景。 - -## Run 41159636766：3 个失败 job - -这轮来自更新的远端 head；三个失败都不能套用同名旧 job 的首错。完整日志、 -浏览器 trace、API 时间线和当前 exact-case 映射保存在证据目录的 -`run-41159636766/` 下。 - -- [ ] **14 / ContainerConsole (console-native-operations)**：review 目标进程的 - 生命周期与聚合观察延迟。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41159636766/job/124418859705)。 - - 唯一真实失败：`console::native_browser::auxiliary_process_menus`。 - DiskIO Start 响应成功，但 `diskio-1 new process` 的 3000 ms predicate 超时。 - - 已确认的首个分歧：Start 响应已有新 PID `1389`；该次 predicate 内的 - `GET /api/servers` 约 3012 ms 才返回，晚于 predicate 的失败时刻。 - 返回内容也已包含目标 PID 和 health up，不能解释为 DiskIO 没有启动。 - 同响应中另一节点的整组 PID 缺失，支持等待无关节点 SSH 的解释； - 但原记录没有各内部阶段的慢 span，不能确定 intent refresh、remote process、 - health probe 各耗时多少，**该节点观察为什么不可用仍未确定**。 - - 当前修复：authority、每个节点的实际进程和每个服务的 endpoint health - 独立采样发布；GET 读取带身份/新鲜度 fence 的 remote facts，不等待 KV、 - SSH 或 endpoint 探测。仍读取本地 binding 以立即拒绝旧身份。 - 三秒从采样开始计，读取不刷新 age；mutation 期间及失败返回后的迟到结果 - 也必须失效。monitor 私有 Processes 将 live PID 与实际执行 operation/ - bootstrap 绑定，保留 Group 0 故障时的 actual PID 观察。 - 新校验同时发现 KV 自动恢复后旧 receipt PID 未更新的边界，补只更新 - 已验证当前执行回执的恢复路径，不再次 restart，不接纳 foreign bootstrap。 - 不将上述新发现说成原 CI 的已证实现场根因。 - 文件：`e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts`、 - `e2e-test/clients/console-ui/e2e/fixtures/nativeProcess.ts`，以及 - `app/crowdb-web/src/services/observation.rs`、 - `app/crowdb-web/src/services/remote.rs`、`services/observation/`、 - `container/crowdb-monitor/src/node/runtime/services/` 及其 process observation。 - - 验证：前一镜像 exact case 实际通过，Playwright 单 test 约 13.3 秒。 - 新生产观察路径的九个 focused 回归和两个真实 monitor 进程回归通过：慢 peer/同节点慢 sibling、 - concurrent read、expiry/identity、same-op late sample、unknown mutation、 - Group 0 故障、pending/exhaustion 和删除不复活；完整组件与 lint 已通过， - 新镜像和 E2E 门禁仍待完成。原 combined 三秒 predicate 保留，不通过移动 deadline 修复。 - - Review：分别计时 Start response、目标 typed process、聚合 inventory 和 DOM。 - 通过 owned monitor PID 与同 namespace `/proc` 核对 typed target，继续保留 - inventory 的唯一 identity、18 条记录、DOM 和健康检查及 combined 三秒。 - 完成原 test 的当前精确计时后，再补 Start body identity/data-preserved 与 - siblings PID/generation/operation 快照；不能把一次 PASS 说成长期稳定率。 - -- [ ] **15 / ContainerDeployment (deployment-services)**：review retained KV - restart 后读取 readiness 的顺序。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41159636766/job/124418859709)。 - - 唯一真实失败： - `deployment::kv_bootstrap::monitor_creates_real_kv_groups_then_validates_after_restart`。 - verify 的首个 `/stores/0/groups/1/ready` 请求返回 503，原因是 - `no leader elected`；预期为 200。不是上一轮 KV-only storage dependency 错误。 - - 已确认问题：verify 在重启后直接断言已有 Group 1 ready，没有先等待选举完成。 - - 当前处理：`cdf023d6` 让 write/verify 两个阶段都先调用现有 - `wait_for_witness_group`，再执行单次 200 断言和 acknowledged-data 验证； - 保留原有四秒准备期限，不重试写入。 - 文件：`e2e-test/tests/deployment/kv_bootstrap.rs`。 - - 验证：P12 修复镜像 exact case 通过；新 SHA 的完整 deployment-services 待核对。 - - Review：ready 必须包含已选 leader 和可提交条件，不能只看进程存活；核对 - 重启后的 retained group/data 验证确实执行，且 readiness 期限没有延长。 - -- [ ] **16 / ContainerDeployment (deployment-reference)**：review 自动加入与 - 显式 admit 的排序。 - [原始 job](https://github.com/buzzcrow/crowdb/actions/runs/41159636766/job/124418859712)。 - - 唯一真实失败：`deployment::reference::direct_docker_three_nodes`。 - Docker 三节点主体验证已经 PASS，后续手工 admit 一个 nonvoter 得到 409， - 原始响应说明 `service operation already running`。 - - 已确认问题：手工 admit 与已有 service operation 冲突；不是第一轮的 - disabled-KV policy 启动错误。旧 fixture 未先停 AutoJoin，源码与针对性修复 - 支持 AutoJoin/manual admission 竞争的解释；原 artifact 没有记录持有 operation - 的 actor/id，确切竞争者身份仍属有依据的推断。 - - 当前处理：`cdf023d6` 先停 AutoJoin 并等待 disabled 状态，再执行一次显式 admit。 - 文件：`e2e-test/oci_reference_recovery.py`。 - - 验证：P12 修复镜像 exact case 完整通过；原先的主体验证、nonvoter 行为及 - scoped cleanup 都保留。 - - Review：确认自动操作已结束后再手工操作；保留单次 admit 及 nonvoter 断言， - 不用重试 409 掩盖竞争；核对新 SHA 的完整 deployment-reference。 - -## 最后逐项 review 的记录 - -- [ ] **补齐最终验收证据**：整理 396-case 历史账本、后续修复的受影响 - 组件/E2E 与物理恢复，保留各自源码/镜像身份和覆盖边界，不将早期 PASS - 归给当前镜像。远端仅保留已经取得的结果；用户已取消继续检测 CI， - 不再要求本次新 SHA 的 27 个远端 job。 -- [ ] **逐项共同 review 01–16**：每项明确“原因确认程度、修复是否充分、 - 回归是否能捕获原问题、还需做什么”。保留 Go 旧原因未知的事实， - 核对 browser batch 新增的未执行/级联错误归因及独立统计。 -- [ ] **归档结论与后续工作**：将确定需要实现的稳定性工作拆成具体任务； - 不以反复重跑得到一次绿色、延长时间或删除断言代替修复。 -- [ ] **Review 物理 P12 的 EC 可用性边界**：此项不是新增失败 CI job。 - 新镜像的 catalog 接管在 40.184 秒通过；新 S3 witness 两份镜像均在 A， - B 停止时三个 A 入口都完整读回。旧 baseline witness 已转为 EC 8+4， - B 上有 6 个分片，超出 4 个分片的恢复预算；旧 harness 在 fault witness - 验证中额外遍历所有历史 witness，因旧对象暂时不可读而失败。 - 只读复现已区分新/旧对象，B 返回后两者五入口均通过，未重放 mutation。 - Review 既有 P12 合同的两阶段验证：停机时严格验证本次新写入；恢复后 - 保留所有历史数据的完整审计。若需要旧 EC 数据也抗整机故障，须另行确定 - 拓扑、放置和可用性合同；KV quorum 不能替代该承诺。 -- [ ] **复核 P12 幸存者的 native 崩溃修复及物理验证**：此项也不是新增失败 CI job。 - 完整区间核对发现 A3 ChunkKV 从 PID 451/generation 2 变为 - PID 66135/generation 3，其余 20 个 A 服务身份稳定。此前 stderr 为空， - 新增 `std::system_error: Resource deadlock avoided`；monitor 记录了停止与重启， - 尚无原进程 core。已通过真实 Rust FFI 最后 owner eviction 复现同样异常， - gdb 证实 `RpcServer` 析构在 response completion 内触发 `Worker::join` 错误 35。 - 共享路由 owner 已改为外部生命周期线程释放，覆盖 response/close/timeout、 - 递归退休、阻塞析构的 backlog 和进程退出顺序；修复后的完整 FFI、storage、 - fmt/clippy 已通过。旧物理进程的精确栈仍未知，不能声称已直接取得它的栈。 - Review 另发现 C ABI send/chain/slab 在同步 completion 释放 wrapper 后仍会 - 读它；四个确定性原生用例在修复前失败，保活 native client/connection/transport - 至返回后均通过。完整 C++、owned tree transport、storage、fmt/tree-lint 已通过。 - 生命周期修复的本地验证与发布已经完成；CI 按用户要求停止检测。 - 当前 c063 新镜像的 P12/P13 物理复验尚待完成，P14 门禁继续保留。 - 文件：`lib/crowdb-tree/ffi/src/chunk/transport.rs`、 - `lib/crowdb-rpc/ffi/src/owned_route.rs`、 - `lib/crowdb-rpc/src/client/client.cpp`、 - `lib/crowdb-rpc/src/transport/socket_transport.cpp`。 - 不以跳过 join、捕获后忽略异常、扩大健康预算或容忍静默重启代替生命周期修复。 -- [ ] **定位 P13 恢复后的 Iceberg 完整读取超时**：此项不是新增失败 CI job。 - 五个入口的 S3 SF1 原始文件均完整读回；并发 Iceberg 审计先后打印 - `NETWORK_CONNECTION during GetObject`、curl 28、三秒低速超时。 - 三个 verifier 真实失败，另两个在保留首错后收到定向 SIGTERM；后两项计为 - 中止，Dataset/history 尚未执行，不能把包装层的 exit -15 当成最初根因。 - 并发 stdout 没有入口标签，三次错误不能仅按打印顺序归属到 A1。 - 不能因原始 S3 校验通过就忽略 Iceberg scan 的失败，也不延长客户端期限掩盖原因。 - 保留本次并发、镜像、服务日志和原始错误，先确认进程身份与恢复状态， - 再复现首错并检查其与 native 崩溃的关系。 - B1/B2 此次 ChunkKV 重启已有 lease 到期、主动退出、正常 join 的日志闭环， - stderr 为空；不是已证实的同一 C++ 异常。下次每个入口单独保存 stdout、 - endpoint、起止时间与退出码，再聚合状态，避免并发错误归属缺失。 - c063 启动审计另捕获 A 主机约十秒 NVMe I/O 停顿，随后 Group0 超时、 - 五个 ChunkKV 租约退出和五个具名 Iceberg checksum reader 的 curl28； - 无 OOM，设备/文件系统原因未证实，不能据此倒推旧 P13 的根因。 - 单对象原限制验证及新的完整五端点只读历史审计随后通过；新 P09 冷启动 - 及完整历史审计也已通过，后续仍须完成 P12/P13 的独立故障验证。 - -## 文件与验证入口 - -- 文件按上面 01–16 各项归属列出；跨项共同检查 policy、fixture 生命周期与 - `e2e-test/oci_shortcuts.py` 的资源归属清理，不将独立问题混为一个根因。 -- 单元/组件：KV election/readiness、console cluster readiness、accepted-listener、 - partial browser startup、OCI shortcut cleanup、native worker 和 FileReader 回归。 -- 集成：完整 core/access/console/storage 组，以及精确的原始非容器失败 case。 -- E2E：396 个 canonical scope/case、并发隔离、原失败 case 的实际执行、Go - 故意注入后的 cleanup、GC 真实 overlap，以及新镜像上的物理恢复。 -- 远端：保留已记录的精确 SHA、attempt 和展开 job。用户已要求停止检测 CI， - 不查询或推断当前提交的新 run，也不将它作为后续物理验收条件。 +Goal: track the unresolved physical qualification and probe-observation work. + +## Current results + +On 2026-11-04, the current-image physical P09 and independent P14 passed. +P13 failed its survivor lease-evidence assertion; its data and recovery +observations passed, but its final exit, stderr, and health qualification did +not run. P16 failed its fixed-cutoff occupancy audit after the data and capacity +checks passed. Earlier physical results remain scoped to their original image; +these targeted results do not constitute full-matrix acceptance. + +The three CI jobs associated with runs +[41329198767](https://github.com/buzzcrow/crowdb/actions/runs/41329198767) and +[41361029738](https://github.com/buzzcrow/crowdb/actions/runs/41361029738) +were rerun locally: UnitTests passed 1,181 tests and Console API passed 92 +cases. Console Native Operations first failed one case; its other 18 cases +passed, the failed case passed separately, and a second complete run passed all +19. The user subsequently approved a ten-second limit for this backend +dependency-convergence prerequisite, with diagnostics after three seconds; +UI and process-observation assertion budgets stay unchanged. The updated +focused case and complete Native job passed (19/19, 420.334 seconds). The two +dependency checks took 6 ms and 5 ms, so the three-second diagnostic did not +fire. The original failed migration window remains retained; this pass does +not establish its eventual recovery or the cause of its control-plane delays. +Evidence: `ci/requested-full-rerun-20261104/approved-convergence/`. + +The artifact root below is +`.crowdb-runtime/artifacts/physical-nodes/physical-20261031T194149Z/`. +The five current test containers and seven scoped volumes remain retained. + +## Unfinished tasks + +For each issue, stop after two unsuccessful repair attempts, record the result, +and continue independent work. Complex issues are recorded for later analysis. + +- [ ] **P16 occupancy audit**: Determine why `/chunk/` metadata changed during + the fixed-cutoff audit over slots 2415597..=2415605. The immediate read-only + journal capture found two chunk updates and no busy updates, but did not + establish predecessor values, changed fields, or whether allocations overlap. + Obtain sufficient evidence for a consistent audit, preserve the original + completeness and nonoverlap checks, and requalify P16. Files: + `repair/readiness-probe-budget-2s/host-readiness-contract-continuation/independent-p16-attempt-1/run/`, + especially `P16-immediate-journal/`. +- [ ] **P13 survivor qualification**: Integrate and verify the staged test + correction for the valid expired-lease fencing path, then rerun the complete + P13 action with its original data, observer, exit-status, stderr, and health + checks. The staged focused controls passed, but do not replace the original + failed physical qualification. Files: + `repair/readiness-probe-budget-2s/p13-lease-evidence-staging/` and + `repair/readiness-probe-budget-2s/host-readiness-contract-continuation/targeted-physical-driver/run/`. +- [ ] **P13 WAL drain**: Investigate the two survivor processes' incomplete WAL + writer drain during lease fencing, including stalled partition writes and + expired-lease withdrawal. The current logs do not prove a successful drain or + identify its cause. Files: the P13 survivor application and RPC logs under + `faults/readiness-probe-budget-2s-P13-survivor-exit-logs/`. +- [ ] **Review failures with a two-second probe budget**: Inspect the saved failures, + particularly those without per-endpoint recovery evidence, and decide whether + any timeout should change. Keep sampling after failures and report the + configured budget, actual elapsed time, error, and recovery; do not silently + increase the two-second budget. P14 recorded five 2000 ms ChunkDB deadlines + during its fault, with 2000/2000/2001/2007/2000 ms elapsed and recovery in + the next generation. The CI reruns recorded further fault/restart warnings, + some with exact recovery unknown. Files: + `repair/readiness-probe-budget-2s/host-readiness-contract-continuation/p13-probe-inventory/`, + `repair/readiness-probe-budget-2s/host-readiness-contract-continuation/independent-p14-attempt-1/run/`, + and `ci/requested-full-rerun-20261104/attempt-{1,2}/access-warning-audit.json`. + The approved ten-second convergence runs add eight warnings with the same + 2000 ms dependency and 2100 ms outer budgets; actual elapsed times are 0, + 2000, or 2001 ms. Their exact per-endpoint recovery remains unconfirmed. + File: `ci/requested-full-rerun-20261104/approved-convergence/access-warning-audit.json`. +- [ ] **Finish physical qualification and resource closure**: After addressing or + explicitly disposing of the remaining failures, reconcile all results by + source and image, preserve their logs, and decide the exact cleanup of the + owned test resources. Physical work is stopped at the user's request after + CI verification and publication. Do not represent mixed-image results as a + fresh full-matrix pass. The previous detailed plan and its deferred review + descriptions remain at `ci/requested-full-rerun-20261104/plan-before-open-items.md`. diff --git a/doc/working/plan-physical-node-ci-handoff.md b/doc/working/plan-physical-node-ci-handoff.md deleted file mode 100644 index 8fbc222b4..000000000 --- a/doc/working/plan-physical-node-ci-handoff.md +++ /dev/null @@ -1,710 +0,0 @@ - - - -# Physical Node and CI Handoff Plan - -Upstream: [Physical Node Test Plan](../dev/physical-node-test-plan.md), -[CI Stability Review Plan](plan-ci-stability-review.md), -[container acceptance](../../e2e-test/README.md), and -[OCI image operations](../../container/oci-image/README.md). - -Goal: finish CI stability verification and publish CROWDB, then resolve P13 and -complete the two-host physical recovery matrix without losing data or evidence. - -## 1. Current checkpoint: P11 monitor freshness repair under validation - -Updated 2026-11-02, 07:05 Asia/Shanghai. On the published c063 image, P09 and P10 both passed -complete historical audits. P11 follower lifecycle and its complete audit passed; -P11 leader lifecycle recovered every original gate, but the continuous observer -rejected the unaffected B2 monitor at 3519 ms old. P12–P19 were not dispatched. -No I/O/memory stall accompanied this event, and no service identity changed -between the failure capture and the later repair preparation. - -The production health loop awaited a Group0 desired-service scan with a -three-second bound before publishing current local health. A slow-authority -regression fails on the old code. The repair moves that read into one bounded -background task, rejects expired/rebound observations, and retains current -binding/intent checks before any disruptive recovery action. Actual local probe -changes continue to publish with unchanged PIDs. Eight focused tests passed, -including rejection of an obsolete recovery intent before stopping a process. -Full fmt/clippy/access-monitor/console validation passed: 2041 Access/Monitor -Rust cases and 426 Console Rust cases, plus the normal task companion checks. -The first clippy semicolon failure and corrected continuation remain separate. -Unchanged C++/core/storage gates retain their c063 inputs and source proof. - -All five current nodes are stopped for repair with all seven original volumes -retained. Both hosts' complete stopped service/crash logs are frozen. Evidence: -first-failures/P11-monitor-freshness/ and repair/monitor-health-nonblocking/. -The repair still needs publication, a source-verified image, affected container -checks and a new physical epoch. Do not relabel c063 passes as new-image passes. - -The reusable physical plan now has six case categories, 20 parent cases and -71 named scenarios, with no checkboxes. It separates execution results from the -case catalog and cancels mandatory new volume archives. The original 16 human -joint-review boxes remain unchanged. CI inspection remains stopped. - -### Previous cross-extent repair checkpoint - -Updated 2026-11-02, 02:18 Asia/Shanghai. The cross-extent read repair is published -as `c063a169f5e400a44c48a5a66aa42fc48609b396`. The normal client/native/image -builds and same-volume deployment passed on both hosts. Image/runtime identity -is recorded in image/cross-extent-read-fixed/. Fresh startup recovered groups -0/1/3 within 1.887 seconds, catalog within 4.961 seconds and all 35 services -within 37.439 seconds, each under its original independent 180-second gate. -All native historical sets and five-endpoint S3 SF1 passed on this image. - -The subsequent five-endpoint Iceberg audit failed with GetObject curl 28 during -checksum reads. At 17:25:33 UTC the A host's nvme1n1 accumulated 209 outstanding -requests; reads/flushes nearly stopped for about ten seconds. All three A monitor -snapshots became stale, Group0 RPCs timed out, and all five ChunkKV processes -expired their leases and drained gracefully. OOM counters remained zero and -host memory/CPU were available. No kernel device error was recorded. The exact -device/filesystem cause remains unproven; do not attribute this event or the old -P13 timeout to the cross-extent repair without further evidence. Preserve the -failed startup receipt and observations/cross-extent-image-upgrade/ unchanged. - -The same 216-MB lineitem object passed isolated full decoding, checksum and -Iceberg sampling in 61.236 seconds with the original client limits. A new strict -complete read-only audit passed in 1230.088 seconds using -cross-extent-image-upgrade-audit and resume-start-audited.json. It includes all -five endpoints, 187 Dataset reports (935 endpoint/report results), all witnesses -and acknowledged deletions. All 35 identities remained fixed across 597 complete -samples; the maximum gap was 3.843 seconds. Additional 250-ms host I/O sampling -captured no new stall. No startup or mutation was replayed; no client timeout, -concurrency, assertion or lease deadline changed. Fresh P09 then passed in 1164.469 seconds, including all five original-volume -recreations, all recovery gates and full retained history. Its 35 expected service -replacements are the only identity changes; the recovered audit remained fixed. -P10 subsequently passed in 1023.819 seconds; P11 follower passed but its leader -observer failed as described above. The preserved dependent queue is -final-queue-cross-extent-read-fixed-after-io/. Both earlier P09 failures and the -new startup failure remain retained. See first-failures/cross-extent-upgrade-protocol/. - -Earlier resume history: the previous repair is published as -1814405e4dd1fe0fcd64f9f18ac7c71e98d64fd2. The user's latest instructions are -"不要检测CI" and "归啥档，测试能过就好了". CI inspection is stopped; do not -query its latest state or require it before physical work. No additional volume -archives are required or authorized by this plan. Preserve existing evidence -and A's three completed archives (2.835 GiB); these are not a cleanup prerequisite. - -B rebooted and is reachable as cj7960 at 192.168.31.184. All seven original -stopped container/volume identities passed the fresh guard. Image transfer and -archive SHA-256 verification passed. The first ID check failed because A exposes -the manifest digest as its Docker ID while B exposes the config digest. The -continuation verifies the manifest-to-config hash, identical platform, labels -and all rootfs layers. See image/s3-ui-fixed/resume-image-verified.json. Upgrade -the five current containers with their original volumes and recorded per-host -image IDs; do not create additional backups. The five-node same-volume upgrade -passed. Groups 0/1/3 recovered in at most 2.995 seconds, catalog in 5.727 seconds -and all services in 25.547 seconds, each within its original 180-second bound. -Complete retained history passed: three 512-record sets, the 4096-record set, -five Group-3 endpoints, baseline allocation, five-endpoint S3/Iceberg SF1, -187 Dataset reports, witnesses and acknowledged deletions. The audit observer -passed 625 complete samples (maximum gap 3.823 seconds), with all 35 identities -fixed. Fresh P09 found a production cross-extent read defect; P09–P19 remain -unfinished at that checkpoint. Those five nodes were stopped for repair; the -current c063 image is now running on the same volumes, with all seven original -volumes retained and no new archives or cleanup. - -- The first startup observer rejected monitor timestamps while managed services - were still launching. P02 requires fresh observations at recovered readiness; - it does not require unavailable recovery health to be Ready. The corrected - observer records transitional timestamps only for explicitly faulted nodes, - keeps all unaffected process identities fixed, and requires all five fresh - snapshots when recovery seals and for the complete audit. Four transition - checks, two immutable-identity-fence tests and seven acceptance-gate checks - passed. The original failure and stop marker remain preserved under - observations/final-image-upgrade/; start-audited reuses successful recovery - gates without restarting nodes or replaying mutations. -- The first fresh P09 recovered all groups, catalog and services within their - original bounds, but the sampler discarded complete node observations when - a deliberately stopping process disappeared from `/proc`. The fixed collector - records absent/unreadable process IDs alongside live siblings and cgroup data; - loss of any unaffected or audit-fenced process still fails. Four focused - checks passed, including an actual-container collection. Original evidence - is retained under observations/final-P09/. The new fault/witness identity is - final-P09-sampling-fixed; the ordered queue is final-queue-sampling-fixed/. -- That second P09 attempt exposed A2 Group 0's latched tree I/O fault after - cold startup. At global address 133259264, asynchronous reading requested - 1212416 bytes but returned 958464, exactly reaching the 128-MiB global - boundary between 64-MiB block files. `BlockAsyncPageStore::submit_read` - mapped only the first fd, while synchronous reads and asynchronous writes - already split extents. A deterministic reopened-store regression reproduces - the identical short read. The fix chains reads across the mapped extents, - completing once after all parts or the first real error; no retry or lock was - added. Cross-extent, missing/truncated-tail and existing async tests pass - (19 focused tests). Complete C++/FFI, core and storage verification, fmt, - clean changed-file tree-lint and full clippy all passed. The repair was published - as c063a169; its verified new image is undergoing retained-volume verification. - Evidence: first-failures/P09-retained-engine-unhealthy/ and - repair/cross-extent-read-fixed/. Do not dispatch P10–P19 on the old image. -- The new deployment controller is harness/physical_storage_repair.py with - explicit scope/image/upgrade/start stages and immutable per-stage receipts. - It checks the published source, quality source hashes and image identity; - no CI result or new volume archive is a prerequisite. Its frozen runtime - inputs belong under image/cross-extent-read-fixed/. The new ordered queue - is final-queue-cross-extent-read-fixed/, using cross-extent-P09–P13 labels. - Keep both earlier failed P09 controllers and their original logs. - -- The first resume scope check failed because all five current physical nodes - are stopped. All seven container/image/volume identities remain unchanged. - A stopped at 14:44 with exit 0; B stopped at 15:14 with exit 137 and no Docker - OOM flag. Both hosts subsequently rebooted. The stop cause is not established. - Preserve `ci/stability-final/resume-host-reboot-scope.json` and - `resume-host-states.json`; do not count this interruption as a physical PASS. -- The original `scope-guard-before.json` remains immutable. The resume guard - uses `scope-guard-resumed-before.json`, preserving exact identities and the - observed stopped state throughout CI. Physical restart remains deferred. - -- The previous-image container acceptance completed: all 396 canonical cases - have a finalized latest PASS; required concurrent isolation also passed. -- The strengthened CLI five cases, real GC three cases, three fresh paired - auxiliary/Journal browser rounds, three fresh Go SDK rounds and instrumented - Dataset browse all passed. The resumed controller exited 0; its closing - scope guard preserved all seven physical containers and volumes exactly. -- The resumed browser queue first failed an immediate Access-health assertion - after ChunkKV replacement. The test now closes dependency readiness within - its existing three-second predicate. Three fresh paired rounds passed; the - earlier failure, traces and read-only diagnosis remain preserved. -- Session 44834, the all-domain controller, also exited 0. Its resumed command - took 5574.963 seconds; the earlier actual CLI failure remains in its attempts. -- The CI pause marker was removed after the normal client refresh and resumed - scope guard passed. The separate physical investigation marker remains. -- The previous full-suite controller completed. Current lifecycle repair tests - are tracked under ci/lifecycle-dependencies-fixed/; original failures and - baseline comparisons remain under ci/published-5af2737f/. Physical work remains deferred; the - seven retained physical containers and their volumes remain owned and preserved. - Do not restart old session IDs. -- Commit 76fc6508189dd31018e953674a8f1c693011498d was pushed successfully. - [CI run 41295426521](https://github.com/buzzcrow/crowdb/actions/runs/41295426521), - attempt 1, finished with 24 successful and three failed jobs out of 27. - All job logs and the three failed-job artifacts are preserved under - ci/published-76fc6508/. Artifact sizes, SHA-256 and ZIP CRCs were verified. -- All three test-boundary repairs are locally verified: a frame-aware lost-reply - proxy, exact selected-node requests per topology cycle, and final CLI readiness - within the remaining original startup deadline. No production source changed. - The chunk job's 47 remaining cases passed, retaining three unaffected remote - passes. All six CLI cases, including the interrupted terminal-recovery peer, - and the original shell/capacity browser pair passed. Fixture 304, full clippy, - fmt and TypeScript passed; the closing scope guard preserved all seven nodes. - The coherent test repair commit 70ae86b1c62e095be2754ed178890736e49be5a4 - was pushed successfully. Its new run 41299662435, attempt 1, finished with - 26 successful jobs and one failure: console-api's terminal chunk writer - recovery. All 27 job logs and its verified failure artifact are preserved - in ci/published-70ae86b1/. The three previous failed cases passed remotely. -- After a lease-excluded transfer superseded a committed split's retained - parent, child materialization still required two original split markers. - The monitor repeatedly failed before serving-grant renewal; the original - 180-second recovery deadline expired. An exact local run passed, but the - durable-transition regression reproduced the same error deterministically. - The production fix validates remaining assignments against the durable split - proof and preserves the transferred parent. All 86 KV Server and 112 ChunkKV - Server tests passed, with one existing opt-in RDMA ignore. Full clippy and fmt - passed. A new source-verified image passed the exact terminal recovery case - (151.1 seconds; recovery check 21.2 seconds), all nine CLI cases at default - concurrency, two ChunkKV cases, balance and 83 remaining console-api cases. - These are 95 distinct current-image cases. The coherent fix was committed and - fast-forward pushed as 5af2737fd5797a1afd1395eac63427e8fdb892d2; commit hooks - passed fmt/clippy. Run 41304660599 attempt 1 ended 25 PASS / 2 FAIL. Its merge checkout - 3c87db4bcf71f2d532762bbadb3a4416732514f1 has the same complete tree as HEAD: - 4b93303bfaf8cc2e0b797b14815d8e884771e473. Previous 396-case coverage belongs - to the previous image. All 27 current jobs must pass before physical dispatch. -- The previous terminal writer recovery now passes remotely. The two new failures - are auxiliary external Access restart during an initial ChunkKV transfer and - six-node Access deployment Waiting classified as terminal failure. All 27 logs - and both verified artifacts are retained. The six-node trace exceeded the - existing 16-MiB export limit and was not uploaded; do not claim it was inspected. -- Local exact auxiliary passed (12.4 seconds), its original concurrent pair - reproduced the same restart 409 (12.1 seconds), and the serial auxiliary - passed (13.7 seconds). The first repair used placement settle; subsequent - source review found that its transfer observation cannot exclude a split - planned in the same or next tick. Use the actual local ChunkKV readiness - immediately before each Access restart. Preserve every original response, - process, sibling and readiness assertion. -- The original ordered local partition stopped earlier at KV restart after - stopping Group 0's leader. The isolated serial case also failed. A new leader - was elected after submission; the authority guard returned 502. Observe the - same cluster's surviving authority within the existing election budget before - submitting one restart. The mutation response budget remains unchanged. -- Two deterministic plan-hook regressions reproduced Access Waiting and multiline - ChunkKV Waiting becoming failed. A further regression exposed stale waiting - detail selecting the ten-second missing-disk backoff for a registered startup. - The repair restores the existing dependency-wait contract and observes the - accepted operation without a duplicate deploy/restart. The new source-verified - image passed all 48 distinct affected cases: native operations 19, browser 27 - and native deployment plans two. Exact KV, auxiliary pair and six-node cases - passed first; lifecycle cases passed again at original default concurrency. - The two data-plan cases used concurrency two but resource-aware batching ran - them sequentially (12,800 + 6,656 MiB exceed the 16,384-MiB budget). - UI 366, plan-hook 28 including two new regressions, fixture 304, TypeScript, - fmt and full clippy passed. Commit 18015caf0cc99978eec99517a3f52b468128c0e2 - was fast-forward pushed; run 41311084753 attempt 1 completed all 27 jobs - successfully. All logs, exact-head/merge source equality and the actual CI - image receipt are retained in ci/published-18015caf/. -- A test-only refinement replaces the insufficient early placement predicate - with a direct, identity-checked ChunkKV /ready probe before the single Access - restart. It retains the three-second observation budget and all mutation - assertions. Its first local attempt failed on an incorrect inventory URL - field; the corrected mgmt_url exact case and original concurrent pair passed. - The full operation group stopped at a separate S3 delete failure: 17 PASS, - one FAIL and one unrun. Preserve every attempt in ci/access-readiness-fixed/. - The current refinement and S3 UI repair need publication and all 27 exact-head - CI jobs before physical dispatch. -- S3 multipart delete's original three-second DOM assertion failed both in the - full group and a fresh serial baseline. The isolated and concurrent-pair - baselines passed. In the full-group trace, DELETE returned 204 in 1.279 seconds; - its subsequent list returned 200 with zero keys in 2.036 seconds. S3View clears - selection on deletion success but retains cached rows until listing finishes. - Correction: remove only the successfully deleted exact object's - current-bucket row before the existing refresh; retain failed/unknown rows. - The user approved that behavior. S3-04 and its ASCII layout were updated - first, followed by a deterministic deferred-list regression. After correcting - its provider/field-label setup, the old UI fails the exact stale-row assertion; - the one-line removal fix passes all three cases, including rejected and - unknown deletes and a failed post-delete list refresh. The original three-second - E2E assertion remains unchanged. Diagnosis, trace timeline and baselines: - ci/access-readiness-fixed/s3-delete/. New image and verification: - ci/s3-ui-fixed/ and image/s3-ui-fixed/. The new image passed the exact multipart - case (4.6-second Playwright), original auxiliary pair (12.7-second auxiliary) - and all 19 native operation cases at default concurrency (417.230 seconds). - Multipart passed again in the group at 4.1 seconds. UI 369, full TypeScript, - fmt and full clippy passed. Initial unsupported Testing Library options failed - type checking after all UI cases passed; that attempt and the corrected gate - are preserved separately. Publication and new exact-head CI remain pending. -- Offline physical observer regressions now require an immutable 35-process - identity fence after recovery and throughout the subsequent full data audit. - A second fence cannot accept a later restart. Both fence regressions and all - seven closure acceptance checks passed; physical resources remain untouched. -- Run 41299662435 checked out merge commit e9f241349146046d52c1198cf3f1b0e0dbdbccff. - Its complete source tree equals the pushed head's tree - 9a7a6a85ba27c2722610d4185912edf67b3c86bf. Local and CI image runtime digests - differ; source equality does not claim identical binaries. -- Two physical hosts have not completed final acceptance: P13 is still failed, - P14–P19 are unrun, and the latest native lifecycle fix needs physical validation. - -This is a temporary execution handoff. Delete it only after physical completion -and retention of the final report. Keep the separate 16-item CI checklist until -the user finishes joint review; a PASS must not check a human review box. - -## 2. Repository and publication boundary - -- CROWDB root: /cjdata/cpp/crowdb2. -- Branch: task-fix-build; latest published HEAD: - 1814405e4dd1fe0fcd64f9f18ac7c71e98d64fd2. -- The four previously local commits were included in the fast-forward push: - cdf023d6 (recovery/liveness), 81d07f03 (nearby mirrors), - d396a220 (healthy leader quorum lease), and 7a93c4e5 (cleanup/handoff). -- The production changes, subsequent test fixes and new split materialization - repair, lifecycle/dependency repair, Access test precondition and S3 UI repair - are committed and pushed. Preserve - any new user edits; do not reset or restore. -- Remote task-fix-build was fetched at 18015caf immediately before the last push; - GitHub then confirmed the new exact head 1814405e. - [PR 55](https://github.com/buzzcrow/crowdb/pull/55) is open. - Fetch and verify remote state again immediately before the eventual push. -- Earlier stability gates and their authorized push are complete. The latest user - instruction stops CI inspection and resumes the physical matrix. CI evidence - remains separate from physical PASS; preserve actual source epochs and failures. -- Loader and website publication are outside scope. Last recorded loader HEAD - was local 447a1e619550bbdc0150139976e0c442675f92b0, with origin/main at - 4cafdc9227871f55942a2b215ccb5a3078ade70d; website had local c9306c51. - These are historical identities, not freshly checked remote status. - Do not read or stage any .aws directory in any workspace. - -## 3. Evidence and frozen inputs - -All abbreviated paths below are relative to: - - /cjdata/cpp/crowdb2/.crowdb-runtime/artifacts/physical-nodes/physical-20261031T194149Z - -- CURRENT-STATE.json: current pause and next actions. -- ci/stability-final/: previous-image local gates, queues, complete logs, - coverage, independent reviews and pending refinement evidence. -- ci/stability-final/final-current-image-evidence.json: - finalized 396-case coverage and current-image mappings for all original - 32 container failure occurrences plus the newer run's three cases. - Occurrences include repeated case names; they are not 35 distinct cases. -- ci/stability-final/container-coverage.json: latest finalized case per scope/name. -- ci/stability-final/all-queue.json: final all-domain PASS plus preserved failed - attempt. container-all/run-72c5f00c04874d38ae2b97bb70822cd6 is the completed - resumed run. Do not label every historical attempt successful. -- ci/stability-final/refined-e2e-queue.json: all ten queue rows PASS, with - earlier browser attempts preserved. The per-round finalized fixture/client - identities and Dataset timings are in resume-refinement-closure.json. -- ci/stability-final/resume-publication-gates.json: Rust fmt/full clippy, - affected C++ format/tree-lint and diff check PASS. Full E2E TypeScript also - passed after the final test-only dependency-recovery predicate change. -- ci/stability-final/scope-guard-before.json and scope-guard-latest.json: - exact retained A3/B4 container, image, running-state and volume identities. -- ci/published-76fc6508/: all 27 job logs, failed artifacts, repair-diagnosis.json, - repair-queue.json, local-repair-closure.json and exact/original-pair results. - These three newly published failures are separate from the historical 16 jobs. -- ci/published-70ae86b1/: all 27 job logs, the checked terminal-recovery artifact, - exact baseline, deterministic red/green and both affected component suites. -- ci/split-materialization-fixed/: new image build, exact and grouped container - queues, 95-case source-specific local closure, quality gates and diff review. -- ci/published-5af2737f/: third publication, all 27 job logs, both checked failure - artifacts, concurrent/serial baselines and deterministic dependency regressions. -- ci/lifecycle-dependencies-fixed/: current prepare/exact/operations/browser/data - queues, 48-case source-specific closure, quality gates and diff review. -- ci/published-18015caf/: all 27 successful job logs, source equivalence, - CI image receipt and offline physical audit/retirement checks. -- ci/access-readiness-fixed/: the subsequent test-only readiness refinement, - first failed endpoint-field attempt, corrected verification and S3 diagnosis. -- ci/s3-ui-fixed/: UI red/green evidence, corrected gates, exact cases and full - 19-case operation group on the new source-verified image. -- ci/published-1814405e/: publication, source equality, background CI logs, - ci-wait-deferred.json and the B-host connection failure. Independent A backup - progress is recorded in independent-A-archives.json. -- The earlier long handoff is preserved at - handoff-history/plan-physical-node-ci-handoff-before-rewrite-20261101T062936Z.md. - It contains superseded historical instructions, not current authorization. - -Current local server image: - -- Tag: crowdb-node:s3-ui-fixed. -- Source tree: 4b061ce8ab3d74493e975659c79505e5b620b054. -- Docker image ID: - sha256:05bbc6cb53d6aafbc18004f8a2271ddc4708c88ac69d32a6a41fd30e96468c2c. -- OCI config digest: - sha256:381d352c595863b44b7deaa55c75cf1779ec038b40822d58b39466aeca0bf7b0. -- Runtime SHA-256: - 84cdaba42a73f087a5b278834cf768bc210b08af2774839166a74ea7e4dd778f. -- OCI archive: image/s3-ui-fixed/crowdb.oci.tar; SHA-256: - 4ab605d4b0f060cc040e5d217d94850bfa88e9f54437b980ce846867f7dac979. -- Revision label 18015caf predates the new fix; the source-tree label - records the actual source. Physical hosts have not been upgraded. Setup and - retirement require this image, current local verification and the eventual - new all-27-job CI PASS. -- The preceding lifecycle-fixed image and its 48-case validation remain in - image/lifecycle-dependencies-fixed/ and ci/lifecycle-dependencies-fixed/. - Its Docker ID begins 359897a7, source tree is - 1516797b67ad493c9bbf9c25048e574e7d994f31 and runtime SHA-256 is - c03d3b78cf126c42923cfd91b3e6d03160ae082e77039708afafda465d9a5e26. - Those 48 cases are not relabeled as new S3-image coverage. -- The preceding split-fixed image and its 95-case validation remain in - image/split-materialization-fixed/ and ci/split-materialization-fixed/. - Its Docker ID begins cec0d2c1 and source tree is - 861fc9d45ae32f72a5199585a8bb1b231048b448; those passes are not relabeled as - current-image coverage. - -Previous local server image and retained 396-case evidence: - -- Tag: crowdb-node:ci-stable. -- Production source tree: 376fdc26a0e96d7810313ea670bf043c4f0dfcae. -- Docker image ID: - sha256:c2bc8bfd6228cf61fa5a73e8ab296766cd880f8eb4fb100efe247fa23c0a55e9. -- OCI config digest: - sha256:c208f00b8c04b085053010227a755dfe114d1e5a756a13f30019422eb91d1bc0. - This is a different object from the Docker image ID. -- Runtime SHA-256: - 7e17ab9c83a2d4c0411114c39377cd4cabb28d2330791dd5d09d3dce3ac98cfd. -- OCI archive: image/stability-final/crowdb.oci.tar; SHA-256: - 565ffcde6e39f3286d3a671dea8310e646c467cac8d0a9e6af4e598a662100d8. -- Image revision label is the old HEAD 7a93c4e5; the source-tree label captures - the uncommitted production fixes. The image is built and locally verified. - It has not replaced the older image on the physical hosts. -- Latest compiled client tree used by all three fresh paired browser rounds, - all three Go rounds and Dataset browse: - 9f2a4a339b6b02852e39cffa16b5cae0f6f9d8a5. - The CLI used 65819b6aa8cdacf284941faf932681219d01cd0a; resumed GC used - d862cc4d3238afd6e9ccb39e4996ab415d10c5bc. Later changes affect only the - auxiliary browser helper and documentation, so those prior passes remain valid. -- ci/stability-final/refined-source-verification.json confirmed the previous - server sources at that checkpoint. The full all-domain run used an earlier - copied client bundle. - Its PASS cannot validate the newer GC/Journal assertions; their separate - strengthened runs passed. The new production split fix requires the separate - split-fixed image and its current source-verification receipt above. -- This handoff update changes the repository source identity again. Before - another E2E run, refresh compiled clients through the normal builder. - Do not bypass manifest validation or blindly rerun all-queue, which is complete. - -## 4. Work already verified - -- [x] **Core production fixes and component gates**: core 1165, - access/monitor 2039, console 426 and full fixture 301 tests passed, with the - existing opt-in/environment ignores retained. UI, full TypeScript, - Rust fmt/full clippy, affected C++ format/tree-lint, full C++/FFI, storage, - ChunkDB and CI task mapping passed. Exact queues/logs are retained. -- [x] **RPC/native lifecycle**: final shared route ownership is retired outside - RPC callbacks, including recursive retirement and process-exit draining. - C ABI send/chain/slab retain native owners before synchronous completion. - Deterministic Rust FFI and four native red/green cases, full gates and - independent review passed. See ci/route-retirement-fixed/ and - first-failures/P12-A3-survivor-restart/crash/FINAL-READONLY-REVIEW.md. - The physical P12 abort has no stack; do not call it the same proven stack. -- [x] **Service observation**: HTTP reads no longer perform authority/SSH/health - network collection. Independent samplers use sample-start freshness and - binding/endpoint/operation/mutation fences; real monitor identity survives - Group 0 unavailability. Actual readiness remains independently checked. - Nine focused Web tests, two real monitor tests, ten replay tests and full - component gates passed. Final source review has no blocking finding. -- [x] **Browser batch failure provenance and scoped cleanup**: undispatched peers - are explicitly not_executed, never PASS or an independent root failure. - Original errors and case-specific cleanup errors remain distinct. Shortcut - cleanup is scoped to recorded owned resources. Full fixture tests passed; - actual all-domain and scope-guard results passed. -- [x] **Auxiliary lifecycle coverage**: original combined three-second predicate - passed on the new image (Playwright 9.7 seconds). Strengthened actual PID, - namespace, old-process exit, generation/operation, sibling identity and - registration uniqueness coverage passed (12.4 seconds). All 19 native - operations cases passed. Monitor operation change is checked; the HTTP API - does not expose an exact accepted operation ID, so that is not claimed. -- [x] **CLI cold observation and actual process identity**: the real first - all-domain failure was KeyError: 'pid' in plain_docker_target_attachment. - A ready cluster need not have every optional cached PID immediately. - The fix checks owned containers/bindings, 21 monitor services and real /proc - liveness while preserving 18 unique full endpoint registrations. - Four regressions and all five strengthened actual CLI cases now pass. - This does not add waits or caller retries. -- [x] **Previous image container acceptance**: 396/396 latest canonical PASS and - required four-case concurrent isolation PASS. The progress helper shows eight - isolation passes because two all-domain attempts each ran four; do not count - them as eight distinct isolation cases. All seven physical resources retained - their exact identities. -- [x] **CI failure audit**: all 16 failed jobs from the three user-specified runs - are in the linked stability TODO. Complete job logs, ten original artifact - ZIPs with matching hashes, and case mappings are retained in ci/prior-ci-audit/. - The runs are 41003611925 (8/27 failed), 41128190276 (5/27), and - 41159636766 (3/27 at head 8885d2a). All 16 joint review boxes remain unchecked. - -Main source ownership in the published change: - -- RPC/C ABI: lib/crowdb-rpc/ffi/src/owned_route* and logging.rs, - lib/crowdb-rpc/src/c_api.cpp, lib/crowdb-tree/ffi/src/tree.rs, and tests. -- Repair/recovery/read path: app/crowdb-chunkdb/src/repair*, - app/crowdb-chunk-kv-server/src/storage.rs, - lib/crowdb-chunk-kv/src/partition* and lib/crowdb-access-iceberg/src/file/reader.rs. -- Observation: app/crowdb-web/src/services/observation* and remote/deployment/state - integration; container/crowdb-monitor/src/node/runtime/{observation,services}* - and corresponding crate tests. -- Test stability: e2e-test/browser_batch.py, fixture*.py, report.py, - oci_shortcuts.py, console_cli_target.py, GC convergence/pressure/sdk_pressure, - console/native_balance.rs, console-ui lifecycle/Journal flows and helpers, - their regressions, and tools/test-metrics/measure.py. -- Split materialization: app/crowdb-kv-server/src/background/domain_monitor/ - chunk_kv/catalog.rs and tests/common/split_materialization.rs. -- Documentation: this handoff, plan-ci-stability-review.md and - doc/dev/physical-node-test-plan.md. Inspect git status before staging. - -## 5. Remaining CI stability and publication tasks - -The user has resumed; proceed in the order below. - -- [x] **Validate the GC ready barrier with the new bundle**: run the three exact - convergence, notification matrix and foreground SDK cases in the saved queue. - Four real SDK workers now warm up and park before executable purge markers; - actual deletion releases work after the original faults. Preserve nine files, - three restarts, six operation classes, 64 writes, true completion/deletion - overlap and original 720/3300-second absolute deadlines. Six pure regressions, - compiled clients, clippy and independent review passed. All three strengthened - real cases passed on resume; queue duration 480.220 seconds. Inline convergence - and matrix retain 240/81 SDK completions inside 5/7 deletion intervals, with - 64 writes each and all three matrix restarts before release. Evidence: - `ci/stability-final/resume-gc-proof.json`; see GC-READY-BARRIER.md and - GC-READY-FINAL-REVIEW.md. -- [x] **Validate Journal request ownership and repeat the browser pair**: exact - request/cursor identity plus Response.request() association and body completion - replace pathname-only response matching. All waits share the original three - seconds; owner-loss/restart/stale-cursor assertions remain. Four event - regressions and full TypeScript pass. Run auxiliary plus - journal_identity_and_owner_recovery together, three fresh fixture rounds, - all required to pass. See JOURNAL-REQUEST-STABILITY.md. - Resume round one passed; round two failed the auxiliary Access initial `up` - assertion after Chunk-KV replacement, while Journal passed both rounds. - Exact isolated and paired serial baselines passed. Read-only sampling of the - unchanged concurrent pair confirms real Access 503 during dependency faults, - followed by recovery. The existing three-second replacement predicate now - additionally requires same-node Access recovery before advancing. No assertion, - timeout, or mutation retry was removed or relaxed. Three fresh paired rounds - on this strengthened source all passed in 45.303, 44.938 and 46.144 seconds. - Earlier attempts are preserved under - `ci/stability-final/first-resumed-browser-failure/`; diagnosis is in - ACCESS-DEPENDENCY-CLOSURE.md. -- [x] **Repeat real Go SDK cleanup three times**: normal and injected cleanup - assertions stay unchanged. Old CI cleanup's operation/root cause is unknown; - enhanced Smithy operation/API-code errors are retained. Repeats provide - current coverage, not a claim that an unknown historical algorithm was fixed. - Three fresh rounds passed in 11.143, 10.611 and 10.139 seconds. Each log - confirms both ordinary and injected-failure cleanup completed. -- [x] **Run instrumented Dataset browse and retain headroom risk**: phase timing - now separates seed, default production balance and exact 512-record readback. - In the completed all-domain run browse was 281.284 seconds, import 321.901, - recovery 331.468; the latter two leave only 38.099/28.532 seconds under 360. - Late partition formation is observed; no safe behavior fix is proven. - Keep policy, cooldown, three owners, at least 12 partitions, data and deadlines. - DATASET-HEADROOM-AUDIT.md records causal limits and bounded follow-up evidence. - Instrumented browse passed: seed 2.276 seconds, balance 260.002, exact readback - 0.273; 12 partitions and three owners. Client 278.32 seconds, full queue case - 291.230 seconds. This risk remains open even after a PASS. -- [x] **Close local evidence**: all refinement rows and finalized fixture - timings checked; fresh scope guard and production source verification pass. - Final-current-image evidence confirms 396 latest PASS, mapping all 32 older - failure occurrences and three newer cases. Client binary hashes match the - frozen build manifest; browser/Go/Dataset source digests agree. Stability TODO - updated; all human review boxes remain unchecked. -- [x] **Commit and push CROWDB only**: final relevant gates and staged diff - check passed; the pre-commit hook independently passed fmt/clippy. Created - 76fc6508 and fast-forward pushed after fetching and verifying the old remote. - The publication receipt is ci/stability-final/publication.json. Required rules: - inspect staged contents, and make one coherent commit with a single-line - subject, no body/trailers/doc references/requirement numbers. Fetch and verify - the remote before the authorized fast-forward push; do not force-push. - No Sirius-specific pre-push rules apply to this repository. -- **CI inspection stopped by user**: do not inspect run 41320020131 or infer - its latest result. Retain only already collected evidence. This supersedes - the historical inspection instructions in this section. - Run 41311084753 passed all 27 on 18015caf. - Earlier run 41295426521 attempt 1 ended 24 PASS / 3 FAIL. - The three fixes and local affected regressions passed and were pushed as - 70ae86b1. Run 41299662435 attempt 1 ended 26 PASS / 1 FAIL. Its production - split materialization repair now has 198 affected component PASS and 95 - exact-image container PASS. The fix is pushed as 5af2737f; inspect all jobs in - run 41304660599 attempt 1 ended 25 PASS / 2 FAIL. Repair the two new lifecycle - failures and the local Group 0 restart boundary, then publish and inspect the - next exact-head run. Current diagnosis: ci/published-5af2737f/repair-diagnosis.json. - Match the actual new head SHA/run/attempt, inspect every expanded job - (the three historical runs had 27), preserve every first failure and artifact, - fix real errors and rerun the appropriate new-head job/run. Do not rerun an old - failing SHA as validation of the new code. -- [ ] **Joint review**: retain all 16 job-level entries for the user. Passing - tests cannot close the historical Go root-cause unknown, Dataset headroom, - or unfinished physical validation. - -The refinement queue is complete. Do not replay it merely because this document -changed. If a later code fix needs E2E, refresh clients normally and select the -affected group while retaining valid completed evidence. Reuse the final image -while production sources still match. Every command and original error is durable. - -## 6. Physical checkpoint and remaining work - -The user explicitly stopped CI inspection and additional volume archives. -Earlier stability gates and publication are complete. The additional storage -repair passed local gates and is ready for publication. Physical work continues -using the original volumes, with exact ownership checks and complete evidence. - -- [x] **Image upgrade**: B connectivity and identical image content are verified. - All five current containers were recreated with their original volumes, - mounts, environment and limits. Seven volume identities remain unchanged. - A and B expose different Docker IDs for the same manifest/config content; - image/s3-ui-fixed/resume-image-verified.json records both. -- [~] **Cross-extent storage repair**: deterministic red/green regression and - all local quality gates passed. Publish the fix, build/verify the new image, - use the original volumes, complete its read-only history audit, then execute - fresh P09–P16 and the scoped P17–P19 closure. Earlier-image passes retain - their original source identities. - -- Hosts: A 192.168.31.129 (A1/A2/A3) and B 192.168.31.184 (B1/B2). - Existing SSH uses BatchMode/IdentitiesOnly. Verify host identity/clocks on resume. -- Remote project commands use /home/cj/.pixi/bin/pixi run with explicit manifest - /home/cj/crowdb-runs/pixi.toml. Remote artifact/harness root: - /home/cj/.crowdb-runtime/artifacts/physical-nodes/physical-20261031T194149Z. -- Current owned nodes: a1/a2/a3/b1/b2 labelled with this run. Two older stopped - B containers are additionally retained: seven physical containers total, A3/B4. -- Five voters, two physical racks, Groups 0/1/3, 35 managed services; - each node has hard 1 CPU, 2 GiB, 512 PIDs, one Tokio worker, - 16 GiB media with 1 GiB reserve. Preserve port offsets A=0/1/2 and B=0/1. - Port bases: Web9090, KV10000/10100, DiskDB11100/11200, DiskIO13000, - ChunkDB12100/12200, ChunkKV15100/15200, S316000, Iceberg17000, - Dataset18000, health19000, container SSH2222. Preserve host SSH22. -- P00–P08 passed on the initial image, including complete eight-table Iceberg - SF1 file/checksum/row and bidirectional SQL checks, independent zero-retry S3 - SF1, Dataset full fields/multipart, 512/4096 native records across five safe - owners, acknowledged-deletion audits, and the first complete 30-minute soak. -- Actual mixed-workload overlap was 1818.511 seconds: 1826 native cycles, - 1829 S3 cycles, 183 Iceberg and 183 Dataset cycles. All 35 identities unchanged; - 909 complete observations, max gap 2.617 seconds, no OOM. - Do not repeat/resume that timed workload merely to manufacture a new pass. - Production changes during a new soak would require restarting that new soak. -- P09–P11 passed on the older repair image with full data/history and sibling - identity closure. P12 original repair defect was fixed: expired unpublished - Allocated targets are replaced under the claim, copied/published targets and - normal owner-grace reclamation remain intact. Original takeover then passed - in 40.184 seconds. Old physical image input tree is - 76f794e03757249f37efa96cd0bfcad064be4388. -- A P12 harness witness exceeded its EC 8+4 outage budget when B removed six - fragments; the corrected fresh mirrored outage witness passes, and complete - history returns after B recovery. Keep this harness boundary distinct from - production repair and retain the complete post-recovery historical audit. - -- **Additional archives canceled by user**: preserve existing A archives and - all logs. Upgrade and retire using exact resource ownership/identity checks, - without creating more volume archives. Preserve the original failure receipts; - do not blindly replay resume_repaired_image.py or old witness mutation IDs. -- [ ] **Close unexpected P12 A3 identity change**: A3 ChunkKV changed PID451/gen2 - to PID66135/gen3, stderr Resource deadlock avoided, no usable core. - Local Rust-FFI gdb proves callback-final-owner destruction can join its own - RPC worker; the physical stack is unknown. Validate the new lifecycle fix on - physical hosts and close all collateral process identities through full audits. - Preserve first-failures/P12-A3-survivor-restart/ and - faults/P12-full-identity-closure.json. -- [ ] **Resolve P13 Iceberg read timeout**: after A quorum loss, recovery/native - and five complete raw S3 SF1 audits passed. Three Iceberg readers returned - curl 28/GetObject low-speed timeout; two remaining owned read-only readers - were stopped after preserving errors and are interrupted, not PASS. - Dataset/history checks were not dispatched. Exact endpoint/object/range cause - is unknown; unlabeled old stdout cannot assign failures by output order. - B1/B2 had controlled process-lease-expiry restarts with empty stderr, distinct - from the P12 A3 abort. Read first-failures/P13-protocol-read/, - owned-reader-stop.json, READONLY-REVIEW.md and ci/physical-after-witness-scope.log. - Add per-endpoint/object-stage read-only diagnostics before retrying verification; - keep original I/O semantics, limits and credential redaction. - The minority write returned OutcomeUnknown; later value presence is not an - acknowledgment and must not cause automatic mutation replay. -- [ ] **P14 allocation admission fault**: preserve bounded Group1 step-down, - unchanged Group0 lease/term observations, explicit allocation Unavailable, - same-ID safe retry under the original 10-second/five-retry client policy, - all 35 process identities, and complete subsequent data audit. -- [ ] **P15 backlog and cold restart**: stop a real Group3 follower, acknowledge - 1100 single writes and prove gap greater than 1024; restore within the original - independent 180-second checkpoint gates, audit all writes, then cold-restart - all five and audit complete retained history. Do not claim reclaimed-WAL or - other untriggered physical sub-boundaries independently exercised. -- [ ] **P16 final occupancy/live overlap**: validate 17 total KVs including the - fresh batch, allocation, full data/history, capacity and process identity - fences. Follow the actual contracts in harness/recovery_final.py and the - reusable physical plan; do not replace them with a weak readiness-only pass. -- [ ] **P17–P19 close only after all prerequisites pass**: freeze complete - evidence/results/summary, inspect identities across the entire fault and audit - intervals, privately preserve remaining data, then exact owned-only retirement. - Use only the reviewed harness/close_physical_final.py and - cleanup_host_final.py stages. Never replay obsolete close_run.py or - cleanup_host.py; no global cleanup. - -The separate investigate-unexpected-P12-restart marker prevents recovery_final.py -from dispatching P14. It is not the CI pause marker. Do not remove it merely to -make the controller advance; P12/P13 evidence and gates must actually be closed. -Old controllers already submitted mutations: never run all_physical.py or an -earlier resume script blindly. - -## 7. Non-negotiable evidence and data boundaries - -- Run all builds/tests/lints/project executables through pixi run, including - remote commands with the verified manifest. Serialize shared-output builders. -- Print the real failing case/error; use CROWDB_E2E_PRINT_SERVICE_ERRORS=1 while - diagnosing and retain complete logs. Only finalized timing records containing - total and stop count as results; provisional diagnostic status is not failure. -- Keep existing assertions, operation budgets and caller retry policies. - Add no hot-path lock or unsafe exception. Unknown mutations are reconciled - read-only; acknowledgment cannot be inferred from a later successful read. -- Native keys exclude reserved S/I/D prefixes. Keep the original 64-KiB values, - batches of 16, five-second operation deadline and actual safe-owner coverage. -- Observe all 35 service identities and OOM state with at most five seconds - between complete samples. Readiness must postdate the current process start - and be at most three seconds old; retain at least 30% free test media. -- Recovery service/catalog/fixed-checkpoint gates remain independently bounded - at 180 seconds; PID checks include the entire subsequent data-audit interval. -- Preserve the earlier destructive cleanup incident honestly in the final report: - the old unscoped shortcut cleanup removed seven containers, 89 volumes and one - network on A, including six physical A data volumes before archival. Those six - contents cannot be recovered from retained logs. Incident evidence is under - the older physical-20261031T145356Z run's - ci/native-worker-fixed/unscoped-cleanup-incident.json. -- Cleanup has since been scoped and verified, but it does not undo that loss. - All still-existing failed-run volumes were privately archived before previous - retirement. Never use Docker-wide prune, generic --all cleanup, prefix-wide - protocol deletion, or remove unrelated data. -- Current authoritative state is this checkpoint plus current runtime receipts. - Historical 024347 runtime was deleted; its old paths and sessions are not - surviving proof or valid executable inputs. The archived earlier handoff is - historical context only. diff --git a/e2e-test/clients/console-ui/e2e/fixtures/icebergInspection.py b/e2e-test/clients/console-ui/e2e/fixtures/icebergInspection.py index f2be9973f..91bcfbef4 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/icebergInspection.py +++ b/e2e-test/clients/console-ui/e2e/fixtures/icebergInspection.py @@ -19,7 +19,7 @@ properties={"write.parquet.row-group-limit": "2"}, ) for first in (0, 44): - table.append(pa.table({"id": list(range(first, first + 44)), "message": ["雪", "alpha", "beta", "gamma"] * 11})) + table.append(pa.table({"id": list(range(first, first + 44)), "message": ["\u96ea", "alpha", "beta", "gamma"] * 11})) print("Committed two snapshots with native Parquet data and Avro manifests") many = catalog.create_table( (namespace, "many_files"), diff --git a/e2e-test/clients/console-ui/e2e/fixtures/icebergRoundTrip.py b/e2e-test/clients/console-ui/e2e/fixtures/icebergRoundTrip.py index bf42aee4a..ee9338121 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/icebergRoundTrip.py +++ b/e2e-test/clients/console-ui/e2e/fixtures/icebergRoundTrip.py @@ -11,7 +11,7 @@ table = catalog.load_table((sys.argv[2], "events")) # The UI create response vends a writer grant; Console GETs vend read grants. table.io = load_file_io({**table.io.properties, **json.load(sys.stdin)}, location=table.location()) -expected = [{"id": 1, "message": "three nodes"}, {"id": 2, "message": "雪"}, {"id": 3, "message": "S3 + Iceberg"}] +expected = [{"id": 1, "message": "three nodes"}, {"id": 2, "message": "\u96ea"}, {"id": 3, "message": "S3 + Iceberg"}] table.append(pa.Table.from_pylist(expected, schema=pa.schema([("id", pa.int64()), ("message", pa.string())]))) actual = catalog.load_table((sys.argv[2], "events")).scan().to_arrow().to_pylist() assert sorted(actual, key=lambda row: row["id"]) == expected, actual diff --git a/e2e-test/clients/console-ui/e2e/fixtures/nativeServiceLifecycle.ts b/e2e-test/clients/console-ui/e2e/fixtures/nativeServiceLifecycle.ts index 4bd895773..b5b15d17d 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/nativeServiceLifecycle.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/nativeServiceLifecycle.ts @@ -23,23 +23,34 @@ export async function nativeServerInventory(request: APIRequestContext): Promise return response.json(); } -/** Probe the local dependency immediately before the single Access restart. */ +/** Observe local dependency convergence before the single Access restart. */ export async function accessDependencyReady(request: APIRequestContext, node: number) { const dependency = (await nativeServerInventory(request)).find(row => row.node_id === node && row.service_type === 'chunk-kv'); expect(dependency).toBeDefined(); expect(dependency.id).toMatch(/^chunk-kv-\d+$/); expect(dependency.mgmt_url).toMatch(/^https?:\/\//); const instance = Number(dependency.id.slice('chunk-kv-'.length)); - const deadline = performance.now() + 3000; - await step('native lifecycle: Access dependency readiness', () => expect.poll(async () => { - const remaining = Math.ceil(deadline - performance.now()); - if (remaining <= 0) return false; - const response = await request.get(new URL('/ready', dependency.mgmt_url).href, { timeout: remaining }); - const health = await response.json(); - expect([200, 503], await response.text()).toContain(response.status()); - expect(health.instance_id).toBe(instance); - return response.status() === 200 && health.ready === true; - }, { timeout: 3000, intervals: [100] }).toBe(true)); + const started = performance.now(); + const deadline = started + 10_000; + let observation: { status: number; health: unknown } | null = null; + const diagnostic = setTimeout(() => { + console.warn('[DEPENDENCY] Chunk-KV convergence exceeded 3000ms', JSON.stringify({ node, instance, observation })); + }, 3000); + try { + await step('native lifecycle: Access dependency readiness', () => expect.poll(async () => { + const remaining = Math.ceil(deadline - performance.now()); + if (remaining <= 0) return false; + const response = await request.get(new URL('/ready', dependency.mgmt_url).href, { timeout: remaining }); + const health = await response.json(); + observation = { status: response.status(), health }; + expect([200, 503], await response.text()).toContain(response.status()); + expect(health.instance_id).toBe(instance); + return response.status() === 200 && health.ready === true; + }, { timeout: 10_000, intervals: [100] }).toBe(true)); + } finally { + clearTimeout(diagnostic); + console.log('[DEPENDENCY] Chunk-KV convergence finished', JSON.stringify({ node, instance, elapsed_ms: Math.round(performance.now() - started), observation })); + } } async function siblingIdentities(target: string) { diff --git a/e2e-test/clients/console-ui/e2e/fixtures/parquetFiles.py b/e2e-test/clients/console-ui/e2e/fixtures/parquetFiles.py index 69d0c619d..6a5024864 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/parquetFiles.py +++ b/e2e-test/clients/console-ui/e2e/fixtures/parquetFiles.py @@ -15,7 +15,7 @@ def encode(table): print(json.dumps({ "empty": encode(pa.table({"id": pa.array([], type=pa.int64()), "message": pa.array([], type=pa.string())})), - "a": encode(pa.table({"id": [1, 2], "message": ["alpha", "雪"]})), + "a": encode(pa.table({"id": [1, 2], "message": ["alpha", "\u96ea"]})), "b": encode(pa.table({"id": [3, 4], "message": ["beta", "gamma"]})), "bad": encode(pa.table({"id": ["wrong type"], "message": ["reject"]})), })) diff --git a/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts b/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts index b1b9773bc..8c3544ebc 100644 --- a/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts @@ -95,9 +95,13 @@ test('native diagnostics: interrupted six-service plan reconciles without duplic steps: Object.fromEntries(kinds.map(kind => [kind, { state: kind === 'paxos-kv' ? 'deployed' : kind === 'chunkdb' ? 'deploying' : 'pending' }])), } }); expect(saved.ok(), await saved.text()).toBe(true); - let deploymentRequests = 0; + const lifecycleRequests: string[] = []; page.on('request', current => { - if (current.method() === 'POST' && /\/api\/nodes\/\d+\/(server|diskdb|services)\/deploy$/.test(new URL(current.url()).pathname)) deploymentRequests++; + const path = new URL(current.url()).pathname; + if (current.method() === 'POST' && ( + /\/api\/nodes\/\d+\/(server|diskdb|services)\/(deploy|start|restart)$/.test(path) + || /\/api\/services\/[^/]+\/(start|restart)$/.test(path) + )) lifecycleRequests.push(path); }); const dialog = page.getByRole('dialog', { name: 'Node 1 services', exact: true }); const open = async () => { @@ -111,10 +115,9 @@ test('native diagnostics: interrupted six-service plan reconciles without duplic await page.goto('/?domain=Cluster'); await page.reload(); await open(); - await expect(dialog).toContainText('Deployment was interrupted'); await expect(dialog.getByRole('listitem')).toHaveCount(6); - await dialog.getByRole('button', { name: 'Retry failed services', exact: true }).click(); - await expect(dialog.getByRole('button', { name: 'Done', exact: true })).toBeEnabled(); + await expect(dialog.getByRole('button', { name: 'Done', exact: true })).toBeEnabled({ timeout: 3000 }); + await expect(dialog).not.toContainText('Deployment was interrupted'); }); await step('native plan authoritative reconciliation', async () => { await expect.poll(async () => { @@ -125,14 +128,14 @@ test('native diagnostics: interrupted six-service plan reconciles without duplic const current = await request.get('/api/servers'); expect(current.ok()).toBe(true); expect(identity(await current.json())).toEqual(identity(original)); - expect(deploymentRequests).toBe(0); + expect(lifecycleRequests).toEqual([]); }); await step('native completed plan reload', async () => { await page.reload(); await open(); await expect(dialog.getByRole('button', { name: 'Done', exact: true })).toBeEnabled(); await expect(dialog).not.toContainText('Deployment was interrupted'); - expect(deploymentRequests).toBe(0); + expect(lifecycleRequests).toEqual([]); }); }); diff --git a/e2e-test/clients/console-ui/e2e/flows/30-kv-ops-basic.spec.ts b/e2e-test/clients/console-ui/e2e/flows/30-kv-ops-basic.spec.ts index 599278707..4d53c2120 100644 --- a/e2e-test/clients/console-ui/e2e/flows/30-kv-ops-basic.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/30-kv-ops-basic.spec.ts @@ -101,7 +101,7 @@ test.describe('kv ops · put/get/scan/delete', () => { await openKvPanel(page); const table = page.getByTestId('kv-scan-table'); await expect(table).toContainText('field00FFend'); - await expect(table).toContainText('helloFE8001中文'); + await expect(table).toContainText('helloFE8001\u4e2d\u6587'); await expect(table).not.toContainText('0x'); await expect(table.locator('[data-byte-format="hex"]')).toHaveText(['00FF', 'FE8001']); for (const run of await table.locator('[data-byte-format="hex"]').all()) { @@ -113,7 +113,7 @@ test.describe('kv ops · put/get/scan/delete', () => { } await table.getByText('field', { exact: true }).click(); const detail = page.getByRole('region', { name: 'Selected key' }); - await expect(detail).toContainText('helloFE8001中文'); + await expect(detail).toContainText('helloFE8001\u4e2d\u6587'); await expect(detail.locator('[data-byte-format="hex"]')).toHaveText(['00FF', 'FE8001']); for (const run of await detail.locator('[data-byte-format="hex"]').all()) { await expect(run).toHaveCSS('color', 'rgb(133, 133, 133)'); diff --git a/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts b/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts index 58f7731ad..14437a2f0 100644 --- a/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts @@ -1,6 +1,6 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -// Baseline: journal 4.6s (2026-10-31); partition/overlay 1.4s, unavailable 0.208s, placement 0.321s (2026-10-03). +// Baseline: journal 5.3s (2026-11-03); partition/overlay 1.4s, unavailable 0.208s, placement 0.321s (2026-10-03). import { test, expect } from '../fixtures/realBackend'; import { step } from '../fixtures/stepTimer'; import { journalRuntimeResponse, type JournalRuntimeTarget } from '../fixtures/journalRuntimeRequest'; @@ -24,9 +24,10 @@ test('native diagnostics: journal identity survives Chunk navigation and owner i expect(runtime.journal.chunk_record).not.toBeNull(); await page.getByRole('tab', { name: 'Journal', exact: true }).click(); const journal = page.getByRole('tabpanel', { name: 'Journal', exact: true }); - await expect(journal).toContainText(runtime.journal.chunk_record.chunk_id); + const storage = journal.getByRole('region', { name: 'Journal storage', exact: true }); + await expect(storage).toContainText(runtime.journal.chunk_record.chunk_id); for (const [name, value] of [['Stream chunk record generation', runtime.journal.generation], ['Stream writer epoch', runtime.journal.writer_epoch], ['Metadata group', runtime.journal.metadata_group_id]]) { - await expect(journal.locator('dt').filter({ hasText: new RegExp(`^${name}$`) }).locator('..').locator('dd')).toHaveText(value); + await expect(storage.locator('dt').filter({ hasText: new RegExp(`^${name}$`) }).locator('..').locator('dd')).toHaveText(value); } await expect(journal.getByLabel('Stream chunk record', { exact: true })).toContainText(runtime.journal.chunk_record.logical_end); await journal.getByRole('button', { name: 'Inspect Chunk', exact: true }).click(); diff --git a/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts b/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts index 96e53ddfb..3b49d4635 100644 --- a/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts @@ -9,7 +9,7 @@ import { bucketFailure, bucketReply } from '../fixtures/bucketDiagnostics'; // Baseline: 5.3s (2026-10-04); normal three-node fixture, actual multipart acceptance. test('S3 native multipart upload, HEAD, bounded preview and full round trip', async ({ page, request }) => { const bucket = `console-multipart-${process.pid}-${Date.now()}`; - const key = 'multipart 中文.txt'; + const key = 'multipart \u4e2d\u6587.txt'; const bucketPath = `/api/access/s3/${bucket}`; const objectPath = `${bucketPath}/${encodeURIComponent(key)}`; const bytes = Buffer.alloc(9 * 1024 * 1024, 'x'); diff --git a/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts b/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts index 0ef1a7041..9429fcc02 100644 --- a/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts @@ -165,7 +165,7 @@ async function verifyServices({ page, request, deployments }: UIContext) { async function s3RoundTrip({ page, request }: UIContext) { await step('three-node: S3 upload and exact read through UI', async () => { - const content = 'CROWDB three-node UI round trip\n雪\n'; + const content = 'CROWDB three-node UI round trip\n\u96ea\n'; await page.getByTestId('domain-s3').click(); await resourceAction(page, page.getByRole('navigation', { name: 'S3 buckets' }).getByRole('button', { name: 'S3', exact: true }), 'Create bucket…'); await page.getByLabel('New bucket', { exact: true }).fill('ui-three-node'); diff --git a/e2e-test/clients/iceberg-python/listing.py b/e2e-test/clients/iceberg-python/listing.py index 0ca7624c0..c16a01a58 100644 --- a/e2e-test/clients/iceberg-python/listing.py +++ b/e2e-test/clients/iceberg-python/listing.py @@ -28,7 +28,7 @@ def main(): endpoint_override=endpoint.netloc, force_virtual_addressing=False, ) prefix = f'{config["bucket"]}/{config["prefix"]}data/' - for suffix in ["a.json", "nested/b.json", "雪&<>+%.json"]: + for suffix in ["a.json", "nested/b.json", "\u96ea&<>+%.json"]: path = prefix + suffix file = PyArrowFile("s3://" + path, path, filesystem) assert not file.exists() @@ -39,7 +39,7 @@ def main(): assert stream.read() == b"{}" selected = filesystem.get_file_info(fs.FileSelector(prefix.removesuffix("/"), recursive=True)) names = sorted(file.path for file in selected if file.type == fs.FileType.File) - assert names == sorted(prefix + suffix for suffix in ["a.json", "nested/b.json", "雪&<>+%.json"]), names + assert names == sorted(prefix + suffix for suffix in ["a.json", "nested/b.json", "\u96ea&<>+%.json"]), names print(f"PyIceberg {pyiceberg.__version__} / PyArrow {pyarrow.__version__}: exact creation, HEAD/GET and prefix discovery passed") verify(config) if os.environ.get("CROWDB_E2E_NATIVE_LOADER_FLOW") == "1": diff --git a/e2e-test/clients/iceberg-python/namespace.py b/e2e-test/clients/iceberg-python/namespace.py index 0e7ada86a..95fcb6fe0 100644 --- a/e2e-test/clients/iceberg-python/namespace.py +++ b/e2e-test/clients/iceberg-python/namespace.py @@ -75,7 +75,7 @@ def main(): def verify_namespaces(uri, properties): writer = load_catalog("crowdb", **(properties | {"token": "w" * 32})) namespace = ("client-" + uuid.uuid4().hex,) - child = namespace + ("冰+a%2F",) + child = namespace + ("\u51b0+a%2F",) writer.create_namespace(namespace, {"owner": "original"}) assert writer.namespace_exists(namespace) assert writer.load_namespace_properties(namespace) == {"owner": "original"} diff --git a/e2e-test/clients/s3/batch_delete.py b/e2e-test/clients/s3/batch_delete.py index 6eb1e2d31..bdecf3df7 100644 --- a/e2e-test/clients/s3/batch_delete.py +++ b/e2e-test/clients/s3/batch_delete.py @@ -12,7 +12,7 @@ class BatchDeleteCases: def test_batch_delete_preserves_exact_keys_and_quiet(self): bucket = self.bucket + "-batch" - keys = ["escaped/<&>.bin", "unicode/边界%25+.bin", "literal\rline.bin"] + keys = ["escaped/<&>.bin", "unicode/\u8fb9\u754c%25+.bin", "literal\rline.bin"] self.client.create_bucket(Bucket=bucket) for key in keys + ["retained"]: self.client.put_object(Bucket=bucket, Key=key, Body=b"selected") @@ -35,7 +35,7 @@ def test_batch_delete_thousand_keys_and_unversioned_retry(self): if phase not in ("all", "initial", "retry", "new-publication"): raise ValueError("unknown batch-delete phase") bucket = self.bucket + "-batch-limit" - keys = [f"bulk/{index:04}/<&>边界" for index in range(1000)] + keys = [f"bulk/{index:04}/<&>\u8fb9\u754c" for index in range(1000)] selection = {"Objects": [{"Key": key} for key in keys]} if phase in ("all", "initial"): self.client.create_bucket(Bucket=bucket) diff --git a/e2e-test/clients/s3/default_client.py b/e2e-test/clients/s3/default_client.py index f23de0c47..f12f40937 100644 --- a/e2e-test/clients/s3/default_client.py +++ b/e2e-test/clients/s3/default_client.py @@ -67,7 +67,7 @@ def test_presigned_transfers_tamper_and_expiry(self): client = self.default_client() bucket = self.bucket + "-presign" client.create_bucket(Bucket=bucket) - params = {"Bucket": bucket, "Key": "exact/%25+边界"} + params = {"Bucket": bucket, "Key": "exact/%25+\u8fb9\u754c"} put = client.generate_presigned_url("put_object", Params=params, ExpiresIn=60) self.assertEqual(self.presigned_http("PUT", put, b"presigned bytes")[0], 200) get = client.generate_presigned_url("get_object", Params=params, ExpiresIn=60) diff --git a/e2e-test/clients/s3/object_copy.py b/e2e-test/clients/s3/object_copy.py index a86a708fd..c1046c729 100644 --- a/e2e-test/clients/s3/object_copy.py +++ b/e2e-test/clients/s3/object_copy.py @@ -8,7 +8,7 @@ class CopyCases: def test_server_side_copy_preserves_bytes_and_supported_metadata(self): source_bucket = f"{self.bucket}-copy-source" target_bucket = f"{self.bucket}-copy-target" - source_key = "encoded/雪 %?+&.json" + source_key = "encoded/\u96ea %?+&.json" payload = b'{"copy":"immutable"}' self.client.create_bucket(Bucket=source_bucket) self.client.create_bucket(Bucket=target_bucket) diff --git a/e2e-test/clients/s3/wire.py b/e2e-test/clients/s3/wire.py index d4a87ef74..5fabaad3d 100644 --- a/e2e-test/clients/s3/wire.py +++ b/e2e-test/clients/s3/wire.py @@ -7,7 +7,7 @@ class WireCases: def test_signed_raw_http_wire_contract(self): bucket = f"{self.bucket}-raw" - key = "raw/%25+边界.bin" + key = "raw/%25+\u8fb9\u754c.bin" path = f"/{bucket}/{quote(key, safe='/')}" payload = b"raw-http-object\x00payload" @@ -104,7 +104,7 @@ def test_fragmentation_and_storage_boundaries(self): sizes = [0, 1, 65505, 65506, 65507, 1024 * 1024 + 31, 4 * 1024 * 1024 + 127] keys = [] for index, size in enumerate(sizes): - key = f"encoded/边界-{index}-%00.bin" + key = f"encoded/\u8fb9\u754c-{index}-%00.bin" payload = bytes((offset * 31 + index) % 256 for offset in range(size)) digest = md5(payload) result = client.put_object( diff --git a/e2e-test/console_cli_readiness.py b/e2e-test/console_cli_readiness.py new file mode 100644 index 000000000..2b7484b31 --- /dev/null +++ b/e2e-test/console_cli_readiness.py @@ -0,0 +1,95 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Continuous negative readiness evidence during the original DiskIO outage.""" + +import json +from pathlib import Path +import sys +import time + + +OUTAGE_SECONDS = 40 +OBSERVATION_SECONDS = 8 + + +def check_unavailable(values, endpoints): + assert type(endpoints) is int and endpoints > 0, endpoints + assert len(values) == endpoints, values + for value in values: + assert value.get("ready") is False and value.get("live") is True, value + observation = value.get("observation", {}) + generation = observation.get("generation") + checked = observation.get("checked_at_unix_ms") + age = observation.get("age_ms") + assert type(generation) is int and generation > 0, value + assert type(checked) is int and checked > 0, value + assert type(age) is int and age >= 0, value + assert observation.get("max_age_ms") == 3000, value + assert isinstance(value.get("details", {}).get("chunks"), str), value + assert value["details"]["chunks"], value + state = observation.get("state") + if state == "current": + assert age <= 3000 and value.get("chunks") == "unavailable", value + assert "chunks" in value.get("reasons", []), value + else: + # age_ms truncates Duration, so an expired check may display 3000. + assert state == "expired" and age >= 3000, value + assert value.get("chunks") == "unknown", value + assert "observation-expired" in value.get("reasons", []), value + + +def check_progress(history, endpoints): + assert history, "No outage observations were retained" + for sample in history: + check_unavailable(sample["responses"], endpoints) + for endpoint in range(endpoints): + previous = None + generations = set() + for sample in history: + observation = sample["responses"][endpoint]["observation"] + current = (observation["generation"], observation["checked_at_unix_ms"]) + if previous is not None: + assert current[0] >= previous[0], (endpoint, previous, current) + if current[0] == previous[0]: + assert current[1] == previous[1], (endpoint, previous, current) + else: + assert current[1] > previous[1], (endpoint, previous, current) + generations.add(current[0]) + previous = current + assert len(generations) >= 3, ("Sampler did not advance twice", endpoint, history) + + +def observe_unavailable(artifacts, snapshot, endpoints): + started = time.monotonic() + deadline = started + OUTAGE_SECONDS + time.sleep(OUTAGE_SECONDS - OBSERVATION_SECONDS) + history = [] + failure = None + try: + while True: + sample = {"started_at_unix_ms": time.time_ns() // 1_000_000, + "elapsed_seconds": time.monotonic() - started, + "responses": snapshot()} + sample["finished_at_unix_ms"] = time.time_ns() // 1_000_000 + history.append(sample) + check_unavailable(sample["responses"], endpoints) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(0.25, remaining)) + check_progress(history, endpoints) + except BaseException as error: + failure = repr(error) + raise + finally: + report = {"outage_seconds": OUTAGE_SECONDS, + "observation_seconds": OBSERVATION_SECONDS, + "elapsed_seconds": time.monotonic() - started, + "failure": failure, "samples": history} + try: + (Path(artifacts) / "access-health-paused-history.json").write_text( + json.dumps(report, indent=2)) + except OSError as error: + if failure is None: + raise + print(f"Secondary readiness history write error: {error}", file=sys.stderr, flush=True) diff --git a/e2e-test/console_cli_recovery.py b/e2e-test/console_cli_recovery.py index dd6808bb7..a384e682c 100644 --- a/e2e-test/console_cli_recovery.py +++ b/e2e-test/console_cli_recovery.py @@ -8,6 +8,7 @@ from fixture import request from console_cli_io_fault import DiskIoOutage +from console_cli_readiness import OUTAGE_SECONDS, observe_unavailable def health_snapshot(cluster, label): @@ -37,12 +38,14 @@ def block_and_write(cluster, outage): clients.append(subprocess.Popen(cluster.arguments("s3", "object", "put", "writer-recovery", f"during-{number}", "--text", "during-fault"), env=cluster.environment(), stdout=output, stderr=output)) - time.sleep(40) + # Payload writes can fail before reaching the metadata journal. + clients.append(subprocess.Popen(cluster.arguments("s3", "bucket", "put", + "writer-recovery-metadata-fault"), env=cluster.environment(), + stdout=output, stderr=output)) + observe_unavailable(cluster.artifacts, + lambda: health_snapshot(cluster, "paused"), + len(cluster.servers)) outage.verify() - unavailable = health_snapshot(cluster, "paused") - assert all(value.get("ready") is False for value in unavailable), unavailable - assert all(value.get("observation", {}).get("state") == "current" - for value in unavailable), unavailable finally: for client in clients: try: @@ -87,7 +90,7 @@ def terminal_writer_recovery(cluster, cluster_id): pauses.append((identities[index], pid)) (cluster.artifacts / "fault-injection.json").write_text(json.dumps({ "diskio_processes": pauses, "fault": "owned-namespace DiskIO packet drop", - "outage_seconds": 40}, indent=2)) + "outage_seconds": OUTAGE_SECONDS}, indent=2)) block_and_write(cluster, DiskIoOutage(cluster)) # No application traffic until independent health observation reports recovery. started = time.monotonic() diff --git a/e2e-test/tests/console_cli_readiness_test.py b/e2e-test/tests/console_cli_readiness_test.py new file mode 100644 index 000000000..63100af76 --- /dev/null +++ b/e2e-test/tests/console_cli_readiness_test.py @@ -0,0 +1,109 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +import copy +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from console_cli_readiness import check_progress, check_unavailable, observe_unavailable + + +def failed(generation=24, checked=10000, age=2705, state="current"): + return {"ready": False, "live": True, + "chunks": "unavailable" if state == "current" else "unknown", + "details": {"chunks": "DiskIO read exceeded the readiness deadline"}, + "reasons": ["chunks"] if state == "current" else ["observation-expired"], + "observation": {"generation": generation, "checked_at_unix_ms": checked, + "age_ms": age, "max_age_ms": 3000, "state": state}} + + +def advancing(): + return [{"responses": [failed(24, 10000), failed(26, 10500), failed(25, 10100, 3071, "expired")]}, + {"responses": [failed(25, 12000), failed(27, 12500), failed(26, 12100)]}, + {"responses": [failed(26, 14000), failed(28, 14500), failed(27, 14100)]}] + + +class UnavailableProgressTest(unittest.TestCase): + def test_expired_negative_evidence_keeps_truthful_age_and_continuous_progress(self): + check_progress(advancing(), 3) + + def test_three_second_quantized_boundary_allows_both_states(self): + for state in ("current", "expired"): + check_unavailable([failed(age=3000, state=state)], 1) + + def test_stalled_sampler_fails_even_when_its_last_sample_is_current(self): + history = [{"responses": [failed()]} for _ in range(4)] + with self.assertRaisesRegex(AssertionError, "Sampler did not advance twice"): + check_progress(history, 1) + + def test_one_stalled_endpoint_cannot_hide_behind_healthy_siblings(self): + history = advancing() + for sample in history: + sample["responses"][1] = failed() + with self.assertRaisesRegex(AssertionError, "Sampler did not advance twice"): + check_progress(history, 3) + + def test_missing_error_ready_or_inconsistent_freshness_samples_are_rejected(self): + for value in ({"error": "health request timed out"}, + dict(failed(), ready=True), + failed(age=3071), + failed(age=2999, state="expired"), + dict(failed(), details={})): + with self.subTest(value=value), self.assertRaises(AssertionError): + check_unavailable([value], 1) + + def test_timestamp_refresh_without_a_new_generation_and_regression_are_rejected(self): + for generation, checked in ((24, 11000), (23, 12000), (25, 9999)): + history = copy.deepcopy(advancing()) + history[1]["responses"][0] = failed(generation, checked) + with self.subTest(generation=generation, checked=checked), self.assertRaises(AssertionError): + check_progress(history, 3) + + def test_missing_endpoint_is_rejected(self): + history = advancing() + history[1]["responses"].pop() + with self.assertRaises(AssertionError): + check_progress(history, 3) + + def test_observation_uses_the_last_eight_seconds_of_the_original_outage(self): + clock = [0.0] + calls = [] + + def sleep(seconds): + clock[0] += seconds + + def snapshot(): + calls.append(clock[0]) + generation = int(clock[0] // 2) + 1 + clock[0] += 0.1 + return [failed(generation, generation * 2000) for _ in range(3)] + + with tempfile.TemporaryDirectory() as directory, \ + patch("console_cli_readiness.time.monotonic", side_effect=lambda: clock[0]), \ + patch("console_cli_readiness.time.sleep", side_effect=sleep): + observe_unavailable(directory, snapshot, 3) + report = json.loads((Path(directory) / "access-health-paused-history.json").read_text()) + self.assertEqual(calls[0], 32) + self.assertLessEqual(calls[-1], 40) + self.assertAlmostEqual(clock[0], 40.1) + self.assertIsNone(report["failure"]) + self.assertEqual(len(report["samples"]), len(calls)) + + def test_failed_sample_is_retained_and_not_retried_until_it_passes(self): + with tempfile.TemporaryDirectory() as directory, \ + patch("console_cli_readiness.time.sleep"), \ + self.assertRaises(AssertionError): + try: + observe_unavailable(directory, lambda: [{"error": "unreachable"}], 1) + finally: + report = json.loads((Path(directory) / "access-health-paused-history.json").read_text()) + self.assertEqual(len(report["samples"]), 1) + self.assertEqual(report["samples"][0]["responses"], [{"error": "unreachable"}]) + self.assertIsNotNone(report["failure"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/e2e-test/tests/dataset_observation_test.py b/e2e-test/tests/dataset_observation_test.py index 8815e86bf..232c40999 100644 --- a/e2e-test/tests/dataset_observation_test.py +++ b/e2e-test/tests/dataset_observation_test.py @@ -132,7 +132,7 @@ def test_six_boundaries_preserve_original_counts_and_explicit_nonadditive_scope( def test_fragmented_unicode_and_ordinary_output_are_continuously_preserved(self): rows = list(messages()) - rows[0]["observation"]["unavailable"]["extra"] = "雪" + rows[0]["observation"]["unavailable"]["extra"] = "\u96ea" raw = b"ordinary report\n" + b"".join(encoded(row) for row in rows) + b"ordinary EOF without newline" child = TestChild(chunks=[raw[index:index + 17] for index in range(0, len(raw), 17)]) with tempfile.TemporaryDirectory() as directory: diff --git a/lib/crowdb-access-iceberg/tests/file_json_test.rs b/lib/crowdb-access-iceberg/tests/file_json_test.rs index 6e1cbbbf8..50f8ae351 100644 --- a/lib/crowdb-access-iceberg/tests/file_json_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_json_test.rs @@ -41,7 +41,7 @@ async fn json_sealing_streams_large_strings_and_split_utf8_without_changing_cano let validator = JsonSealer::new(store.clone(), 2, 1024 * 1024, 64).unwrap(); for bytes in [ br#"{"n":1e+2,"a":[null,true,false],"s":"{\"x\"}"}"#.to_vec(), - "{\"字段\":\"冰😀\"}".as_bytes().to_vec(), + "{\"\u{5b57}\u{6bb5}\":\"\u{51b0}😀\"}".as_bytes().to_vec(), format!("{{\"large\":\"{}\"}}", "x".repeat(100_000)).into_bytes(), ] { let size = if bytes.len() < 100 { 7 } else { 2048 }; diff --git a/lib/crowdb-access-iceberg/tests/file_listing_test.rs b/lib/crowdb-access-iceberg/tests/file_listing_test.rs index 3e587c034..a2e716442 100644 --- a/lib/crowdb-access-iceberg/tests/file_listing_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_listing_test.rs @@ -124,7 +124,13 @@ async fn two_table_principals_cannot_discover_each_others_published_names() { async fn pages_and_delimiters_preserve_order_without_duplicate_groups() { let fixture = TestFile::new(common::TestStore::default()).await; let repository = FileRepository::new(fixture.store.clone()); - for relative in ["a/one.json", "a/two.json", "b/one.json", "c.json", "雪&<>.json"] { + for relative in [ + "a/one.json", + "a/two.json", + "b/one.json", + "c.json", + "\u{96ea}&<>.json", + ] { repository .publish(fixture.context, &fixture.record(relative, b"{}")) .await diff --git a/lib/crowdb-access-iceberg/tests/file_location_test.rs b/lib/crowdb-access-iceberg/tests/file_location_test.rs index a0a702588..034578c75 100644 --- a/lib/crowdb-access-iceberg/tests/file_location_test.rs +++ b/lib/crowdb-access-iceberg/tests/file_location_test.rs @@ -16,7 +16,7 @@ fn canonical_locations_round_trip_without_names_or_key_normalization() { assert_eq!(table.to_string().parse::().unwrap(), table); for key in [ "metadata/v1.json", - "数据/a+b", + "\u{6570}\u{636e}/a+b", "a%2Fb", "a/b", "a//b", @@ -72,6 +72,6 @@ fn object_key_limit_counts_utf8_bytes_including_table_prefix() { assert_eq!(file.object_key().len(), MAX_OBJECT_KEY_BYTES); assert_eq!(file.to_string().parse::().unwrap(), file); assert!(table.file(&(key.clone() + "x")).is_err()); - assert!(table.file(&("x".repeat(available - 1) + "冰")).is_err()); + assert!(table.file(&("x".repeat(available - 1) + "\u{51b0}")).is_err()); assert!(FileLocation::from_object_key(&table.bucket(), &(file.object_key() + "x")).is_err()); } diff --git a/lib/crowdb-access-iceberg/tests/key_test.rs b/lib/crowdb-access-iceberg/tests/key_test.rs index 21c07fa36..a72dfd88e 100644 --- a/lib/crowdb-access-iceberg/tests/key_test.rs +++ b/lib/crowdb-access-iceberg/tests/key_test.rs @@ -116,7 +116,7 @@ fn key_byte_limit_is_enforced_before_encoding() { #[test] fn name_fields_preserve_utf8_delimiters_and_parent_identity() { let parent = NamespaceId::random(); - for name in ["a/b", "a%1Fb", "数据.表", "a:1"] { + for name in ["a/b", "a%1Fb", "\u{6570}\u{636e}.\u{8868}", "a:1"] { let suffix = NameSuffix { parent: Some(parent), name, diff --git a/lib/crowdb-access-iceberg/tests/manifest_variant_test.rs b/lib/crowdb-access-iceberg/tests/manifest_variant_test.rs index 4f9844e72..6101c97d7 100644 --- a/lib/crowdb-access-iceberg/tests/manifest_variant_test.rs +++ b/lib/crowdb-access-iceberg/tests/manifest_variant_test.rs @@ -234,7 +234,7 @@ fn variant_paths_require_normalized_jsonpath_with_bounded_depth() { "$", "$['']", "$['user.name']", - "$['位置']['纬度']", + "$['\u{4f4d}\u{7f6e}']['\u{7eac}\u{5ea6}']", "$['a'][0]", "$[42]", r"$['a\'b']", diff --git a/lib/crowdb-access-iceberg/tests/namespace_create_test.rs b/lib/crowdb-access-iceberg/tests/namespace_create_test.rs index b666de46d..e0d5ee131 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_create_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_create_test.rs @@ -25,7 +25,11 @@ fn request(fixture: &TestNamespace, names: &[&str]) -> NamespaceCreateRequest { }, principal: "writer".into(), identifier: NamespaceIdentifier::new(names.iter().map(|name| (*name).into()).collect()).unwrap(), - properties: NamespaceProperties::new(BTreeMap::from([("owner".into(), "数据库".into())])).unwrap(), + properties: NamespaceProperties::new(BTreeMap::from([( + "owner".into(), + "\u{6570}\u{636e}\u{5e93}".into(), + )])) + .unwrap(), } } @@ -72,7 +76,7 @@ async fn native_create_publishes_complete_resources_and_retries_existing_names_a assert_eq!( serde_json::from_slice::(&outcome.body).unwrap(), serde_json::json!({ - "namespace": names, "properties": {"owner": "数据库"}, + "namespace": names, "properties": {"owner": "\u{6570}\u{636e}\u{5e93}"}, }) ); let authority = reader diff --git a/lib/crowdb-access-iceberg/tests/namespace_model_test.rs b/lib/crowdb-access-iceberg/tests/namespace_model_test.rs index fc5c8d506..9e2fdbd5d 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_model_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_model_test.rs @@ -8,7 +8,11 @@ use crowdb_access_iceberg::namespace::{ #[test] fn multipart_identifiers_preserve_unicode_and_delimiter_like_text() { - let components = vec!["数据库".into(), "a.b / %1F".into(), "leaf".into()]; + let components = vec![ + "\u{6570}\u{636e}\u{5e93}".into(), + "a.b / %1F".into(), + "leaf".into(), + ]; let identifier = NamespaceIdentifier::new(components.clone()).unwrap(); assert_eq!( NamespaceIdentifier::decode(&identifier.encode().unwrap()).unwrap(), @@ -105,7 +109,7 @@ fn property_limits_are_byte_based_and_include_result_cardinality() { for (key, value, valid) in [ ("k".repeat(1024), "v".repeat(8192), true), ("k".repeat(1025), String::new(), false), - ("键".repeat(342), String::new(), false), + ("\u{952e}".repeat(342), String::new(), false), ("k".into(), "v".repeat(8193), false), ("\0".into(), String::new(), false), ("k".into(), "\0".into(), false), diff --git a/lib/crowdb-access-iceberg/tests/namespace_record_test.rs b/lib/crowdb-access-iceberg/tests/namespace_record_test.rs index d68b890f4..609473594 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_record_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_record_test.rs @@ -22,7 +22,11 @@ fn authority() -> NamespaceAuthority { pending_operation: None, creating_child: None, rename: None, - properties: NamespaceProperties::new(BTreeMap::from([("owner".into(), "数据库".into())])).unwrap(), + properties: NamespaceProperties::new(BTreeMap::from([( + "owner".into(), + "\u{6570}\u{636e}\u{5e93}".into(), + )])) + .unwrap(), } } @@ -132,7 +136,7 @@ fn child_ranges_are_parent_scoped_and_separate_namespace_and_table_indexes() { Some(NamespaceId::from_bytes(&[255; 16]).unwrap()), ] { let range = child_range(catalog, parent, CatalogScope::NamespaceName).unwrap(); - for name in ["a", "longer", "数据库"] { + for name in ["a", "longer", "\u{6570}\u{636e}\u{5e93}"] { let key = name_key(catalog, parent, name).unwrap().encode().unwrap(); assert!(range.contains(&key)); assert!( diff --git a/lib/crowdb-access-iceberg/tests/namespace_repository_test.rs b/lib/crowdb-access-iceberg/tests/namespace_repository_test.rs index 05a4a669b..cd24d42a2 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_repository_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_repository_test.rs @@ -19,9 +19,9 @@ use fixture::TestNamespace; #[tokio::test] async fn lookup_walks_stable_parents_without_writing_or_creating_ancestors() { let fixture = TestNamespace::new().await; - let parent = fixture.authority(None, &["数据库"]); + let parent = fixture.authority(None, &["\u{6570}\u{636e}\u{5e93}"]); fixture.publish(&parent).await; - let child = fixture.authority(Some(parent.namespace), &["数据库", "child"]); + let child = fixture.authority(Some(parent.namespace), &["\u{6570}\u{636e}\u{5e93}", "child"]); fixture.publish(&child).await; let repository = NamespaceRepository::new(fixture.store.clone()); let writes = fixture.store.writes.load(Ordering::SeqCst); @@ -29,7 +29,11 @@ async fn lookup_walks_stable_parents_without_writing_or_creating_ancestors() { repository.load(fixture.context, &child.identifier).await.unwrap(), Some(child) ); - for names in [vec!["child"], vec!["absent", "child"], vec!["数据库", "absent"]] { + for names in [ + vec!["child"], + vec!["absent", "child"], + vec!["\u{6570}\u{636e}\u{5e93}", "absent"], + ] { let identifier = NamespaceIdentifier::new(names.into_iter().map(String::from).collect()).unwrap(); assert!(!repository.exists(fixture.context, &identifier).await.unwrap()); } diff --git a/lib/crowdb-access-iceberg/tests/namespace_update_test.rs b/lib/crowdb-access-iceberg/tests/namespace_update_test.rs index bad8adaa1..b8c003741 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_update_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_update_test.rs @@ -27,7 +27,7 @@ fn request(fixture: &TestNamespace, authority: &NamespaceAuthority) -> Namespace identifier: authority.identifier.clone(), changes: PropertyChanges { removals: vec!["old".into(), "absent".into()], - updates: BTreeMap::from([("owner".into(), "数据库".into())]), + updates: BTreeMap::from([("owner".into(), "\u{6570}\u{636e}\u{5e93}".into())]), }, } } @@ -66,7 +66,10 @@ async fn property_publication_is_atomic_and_does_not_change_identity_or_admissio assert_eq!(selected.pending_operation, None); assert_eq!( selected.properties.entries(), - &BTreeMap::from([("keep".into(), "value".into()), ("owner".into(), "数据库".into()),]) + &BTreeMap::from([ + ("keep".into(), "value".into()), + ("owner".into(), "\u{6570}\u{636e}\u{5e93}".into()), + ]) ); let before = fixture.store.writes.load(Ordering::Relaxed); let replay = repository.update_properties(&request).await.unwrap().unwrap(); @@ -109,14 +112,20 @@ async fn independent_requests_apply_changes_to_current_properties_without_a_ledg repository.update_properties(&request).await.unwrap(); request.changes.updates.insert("owner".into(), "second".into()); repository.update_properties(&request).await.unwrap(); - request.changes.updates.insert("owner".into(), "数据库".into()); + request + .changes + .updates + .insert("owner".into(), "\u{6570}\u{636e}\u{5e93}".into()); repository.update_properties(&request).await.unwrap(); let selected = repository .load(fixture.context, &authority.identifier) .await .unwrap() .unwrap(); - assert_eq!(selected.properties.entries().get("owner").unwrap(), "数据库"); + assert_eq!( + selected.properties.entries().get("owner").unwrap(), + "\u{6570}\u{636e}\u{5e93}" + ); assert_eq!(selected.property_revision, authority.property_revision + 3); assert!(selected.pending_operation.is_none()); } @@ -184,7 +193,10 @@ async fn concurrent_property_writers_rebase_only_after_a_definitive_cas_loss() { .unwrap() .unwrap(); assert_eq!(selected.property_revision, authority.property_revision + 2); - assert_eq!(selected.properties.entries().get("owner").unwrap(), "数据库"); + assert_eq!( + selected.properties.entries().get("owner").unwrap(), + "\u{6570}\u{636e}\u{5e93}" + ); assert_eq!(selected.properties.entries().get("second").unwrap(), "retained"); assert_eq!(selected.properties.entries().get("keep").unwrap(), "value"); assert_eq!(selected.pending_operation, None); @@ -342,7 +354,10 @@ async fn four_real_property_conflicts_merge_the_observed_authority_without_get() "retained" ); } - assert_eq!(selected.properties.entries().get("owner").unwrap(), "数据库"); + assert_eq!( + selected.properties.entries().get("owner").unwrap(), + "\u{6570}\u{636e}\u{5e93}" + ); assert_eq!(selected.properties.entries().get("keep").unwrap(), "value"); } diff --git a/lib/crowdb-access-iceberg/tests/native_storage_path_test.rs b/lib/crowdb-access-iceberg/tests/native_storage_path_test.rs index e6e2edb6c..6ea66d3cf 100644 --- a/lib/crowdb-access-iceberg/tests/native_storage_path_test.rs +++ b/lib/crowdb-access-iceberg/tests/native_storage_path_test.rs @@ -28,7 +28,14 @@ fn native_uri_preserves_immutable_identity_and_exact_filename() { assert_eq!(table.to_string().len(), StorageTablePath::PREFIX_BYTES); assert_eq!(table.to_string().parse::().unwrap(), table); assert_eq!(table.uri(), format!("s3://iceberg/{table}")); - for filename in ["metadata/v1.json", "数据/a+b", "a%2Fb", "a//b", "a/", "a b"] { + for filename in [ + "metadata/v1.json", + "\u{6570}\u{636e}/a+b", + "a%2Fb", + "a//b", + "a/", + "a b", + ] { let location = table.logical_file(filename).unwrap(); assert_eq!(location.table(), table); assert_eq!(location.relative_key(), filename); @@ -87,7 +94,7 @@ fn filename_limit_counts_three_ids_and_utf8_bytes() { location ); assert!(table.file(&(filename + "x")).is_err()); - assert!(table.file(&("x".repeat(available - 1) + "冰")).is_err()); + assert!(table.file(&("x".repeat(available - 1) + "\u{51b0}")).is_err()); } #[test] diff --git a/lib/crowdb-access-iceberg/tests/record_test.rs b/lib/crowdb-access-iceberg/tests/record_test.rs index 8760fd981..8acdd914d 100644 --- a/lib/crowdb-access-iceberg/tests/record_test.rs +++ b/lib/crowdb-access-iceberg/tests/record_test.rs @@ -103,7 +103,7 @@ fn every_root_phase_round_trips_with_its_exact_catalog_and_deadline() { #[test] fn authority_round_trip_binds_identity_to_its_key() { let catalog = CatalogId::random(); - let record = StorageRecord::Authority(CatalogAuthority::new(catalog, "目录".into()).unwrap()); + let record = StorageRecord::Authority(CatalogAuthority::new(catalog, "\u{76ee}\u{5f55}".into()).unwrap()); let encoded = record.encode().unwrap(); let key = IcebergKey::Catalog { catalog, diff --git a/lib/crowdb-access-iceberg/tests/table_metadata_defaults_test.rs b/lib/crowdb-access-iceberg/tests/table_metadata_defaults_test.rs index 89e424cbd..1509b4484 100644 --- a/lib/crowdb-access-iceberg/tests/table_metadata_defaults_test.rs +++ b/lib/crowdb-access-iceberg/tests/table_metadata_defaults_test.rs @@ -24,7 +24,7 @@ fn primitive_defaults_check_type_range_and_canonical_encodings() { ("double", json!(1e100), json!("NaN")), ("decimal(4,2)", json!("12.34"), json!("123.45")), ("decimal(4,2)", json!("1234e-2"), json!("12.3")), - ("string", json!("雪"), json!(5)), + ("string", json!("\u{96ea}"), json!(5)), ( "uuid", json!("12345678-1234-1234-1234-123456789abc"), diff --git a/lib/crowdb-access-iceberg/tests/table_metadata_document_test.rs b/lib/crowdb-access-iceberg/tests/table_metadata_document_test.rs index 051e9d790..05158daa6 100644 --- a/lib/crowdb-access-iceberg/tests/table_metadata_document_test.rs +++ b/lib/crowdb-access-iceberg/tests/table_metadata_document_test.rs @@ -35,7 +35,7 @@ fn official_java_metadata_roundtrips_without_rewriting() { fn version_envelopes_preserve_original_bytes_and_unknown_fields() { for version in 1..=3 { let mut value = fixture::metadata(version); - value["future-extension"] = json!({"nested":[null,true,5,"雪"]}); + value["future-extension"] = json!({"nested":[null,true,5,"\u{96ea}"]}); value["location"] = json!(fixture::table().to_string().trim_end_matches('/')); let bytes = format!(" \n{}\n ", serde_json::to_string_pretty(&value).unwrap()).into_bytes(); let head = fixture::head( diff --git a/lib/crowdb-access-s3/tests/copy_test.rs b/lib/crowdb-access-s3/tests/copy_test.rs index 2a86f8165..57e23dad8 100644 --- a/lib/crowdb-access-s3/tests/copy_test.rs +++ b/lib/crowdb-access-s3/tests/copy_test.rs @@ -76,7 +76,7 @@ fn conditions_limits_and_self_copy_use_the_captured_record() { fn source_addresses_decode_once_and_reject_ambiguous_selectors() { assert_eq!( copy::source("/bucket/a%2Fb%20%25%2B%E9%9B%AA").unwrap(), - (b"bucket".to_vec(), "a/b %+雪".as_bytes().to_vec()) + (b"bucket".to_vec(), "a/b %+\u{96ea}".as_bytes().to_vec()) ); assert_eq!( copy::source("bucket/question%3FversionId%3Dliteral").unwrap().1, diff --git a/lib/crowdb-access-s3/tests/delete_test.rs b/lib/crowdb-access-s3/tests/delete_test.rs index eff7b43c1..7ddb0797c 100644 --- a/lib/crowdb-access-s3/tests/delete_test.rs +++ b/lib/crowdb-access-s3/tests/delete_test.rs @@ -57,9 +57,12 @@ fn batch_route_and_exact_keys() { ) .unwrap(); assert_eq!(route.operation, S3Operation::DeleteObjects); - let selection = DeleteSelection::parse("a&<雪a&<雪true".as_bytes()).unwrap(); + let selection = DeleteSelection::parse("a&<\u{96ea}a&<\u{96ea}true".as_bytes()).unwrap(); assert!(selection.quiet); - assert_eq!(selection.keys, vec!["a&<雪".as_bytes(), "a&<雪".as_bytes()]); + assert_eq!( + selection.keys, + vec!["a&<\u{96ea}".as_bytes(), "a&<\u{96ea}".as_bytes()] + ); } #[test] diff --git a/lib/crowdb-access-s3/tests/object_inspection_test.rs b/lib/crowdb-access-s3/tests/object_inspection_test.rs index a8c6d9e6e..b0f7cde4c 100644 --- a/lib/crowdb-access-s3/tests/object_inspection_test.rs +++ b/lib/crowdb-access-s3/tests/object_inspection_test.rs @@ -10,7 +10,7 @@ fn record(locations: &[Location]) -> ObjectRecord { let length = locations.iter().map(|location| location.logical_length).sum(); ObjectRecord { bucket_id: BucketId::new([7; 16]), - key: "folder/中文.bin".as_bytes().to_vec(), + key: "folder/\u{4e2d}\u{6587}.bin".as_bytes().to_vec(), logical_length: length, checksum: vec![1; 16], etag: "etag".into(), diff --git a/lib/crowdb-access-s3/tests/user_metadata_test.rs b/lib/crowdb-access-s3/tests/user_metadata_test.rs index f933a4c0e..baa13f86c 100644 --- a/lib/crowdb-access-s3/tests/user_metadata_test.rs +++ b/lib/crowdb-access-s3/tests/user_metadata_test.rs @@ -40,7 +40,7 @@ fn metadata_enforces_exact_combined_size_and_unique_valid_headers() { for (name, value) in [ ("x-amz-meta-", "empty-name"), ("x-amz-meta-a", "tab\tvalue"), - ("x-amz-meta-a", "雪"), + ("x-amz-meta-a", "\u{96ea}"), ] { input.clear(); input.insert(name, HeaderValue::from_bytes(value.as_bytes()).unwrap()); diff --git a/lib/crowdb-chunk-client/src/client.rs b/lib/crowdb-chunk-client/src/client.rs index 047797711..638b33b13 100644 --- a/lib/crowdb-chunk-client/src/client.rs +++ b/lib/crowdb-chunk-client/src/client.rs @@ -8,7 +8,7 @@ use disk_metrics::MetricsDiskWriter; mod readiness; #[cfg(feature = "test-util")] -pub use readiness::observe_readiness_phase_for_tests; +pub use readiness::{observe_disk_readiness_for_tests, observe_readiness_phase_for_tests}; mod connection; mod observation; mod writer_policy; diff --git a/lib/crowdb-chunk-client/src/client/disk_metrics.rs b/lib/crowdb-chunk-client/src/client/disk_metrics.rs index 3fbc412ed..af89b5b4f 100644 --- a/lib/crowdb-chunk-client/src/client/disk_metrics.rs +++ b/lib/crowdb-chunk-client/src/client/disk_metrics.rs @@ -242,6 +242,27 @@ impl DiskWriter for MetricsDiskWriter { result } + async fn read_priority( + &self, + seg: &Segment, + unit_bytes: u64, + segment_offset: u64, + length: u32, + ) -> Result { + let mut operation = self.metrics.diskio_read.start(); + let result = self + .inner + .read_priority(seg, unit_bytes, segment_offset, length) + .await; + if let Ok(data) = &result { + self.metrics + .diskio_read_bytes + .observe(u64::try_from(data.len()).unwrap_or(u64::MAX)); + operation.mark_success(); + } + result + } + async fn write_at_byte_offset( &self, seg: &Segment, diff --git a/lib/crowdb-chunk-client/src/client/readiness.rs b/lib/crowdb-chunk-client/src/client/readiness.rs index 04b550889..239ab1ef1 100644 --- a/lib/crowdb-chunk-client/src/client/readiness.rs +++ b/lib/crowdb-chunk-client/src/client/readiness.rs @@ -3,8 +3,12 @@ //! Read-only ChunkDB and physical DiskIO evidence through current client routes. -use crowdb_protocol::{common::HwStatus, diskdb::rpc::Segment}; -use std::{future::Future, time::Duration}; +use crowdb_kv_client::DiskRecord; +use crowdb_protocol::{ + common::{DiskId, HwStatus}, + diskdb::rpc::Segment, +}; +use std::{future::Future, sync::Arc, time::Duration}; use tokio::time::Instant; use crate::metrics::{OperationMetrics, ReadinessMetricsSnapshot}; @@ -24,61 +28,27 @@ impl ChunkIoClient { .topology .as_ref() .ok_or_else(|| IoError::Topology("readiness requires authoritative discovery".into()))?; - let (disks, chunks) = tokio::join!( - observe("hardware", deadline, &self.readiness.hardware, async { + let online = observe_dependencies( + deadline, + &self.readiness, + topology.disk_writer.clone(), + async { topology .hardware .list_all_disks() .await .map_err(|error| IoError::Topology(error.to_string())) - }), - observe("ChunkDB", deadline, &self.readiness.chunkdb, async { + }, + async { Box::pin(topology.chunkdb.observe_readiness()) .await .map_err(|error| IoError::ReadFailed(format!("ChunkDB read: {error}"))) - }), - ); - chunks?; - let disks = disks?; - let online: Vec<_> = disks - .iter() - .filter(|disk| disk.value.status == HwStatus::Up as i32) - .collect(); - if online.is_empty() { - return Err(IoError::Topology("no online disks are available".into())); - } - // Keep background observation work bounded independently of cluster size. - for batch in online.chunks(8) { - let mut reads = tokio::task::JoinSet::new(); - for disk in batch { - let writer = topology.disk_writer.clone(); - let metrics = self.readiness.clone(); - let id = disk.disk_id; - reads.spawn(async move { - let segment = Segment { - disk_id: Some(id), - unit_count: 1, - ..Default::default() - }; - observe("DiskIO", deadline, &metrics.diskio, async { - let bytes = writer.read(&segment, 4096, 0, 4096).await?; - if bytes.len() != 4096 { - return Err(IoError::ReadFailed("short disk header".into())); - } - Ok(()) - }) - .await - .map_err(|error| IoError::ReadFailed(format!("DiskIO disk {id:?}: {error}"))) - }); - } - while let Some(read) = reads.join_next().await { - read.map_err(|error| IoError::ReadFailed(error.to_string()))??; - } - } + }, + ) + .await?; total.mark_success(); Ok(format!( - "ChunkDB read and 4096-byte DiskIO reads succeeded on {} online disks", - online.len() + "ChunkDB read and 4096-byte DiskIO reads succeeded on {online} online disks" )) } @@ -87,6 +57,99 @@ impl ChunkIoClient { pub fn readiness_metrics(&self) -> ReadinessMetricsSnapshot { self.readiness.snapshot() } + + #[cfg(feature = "test-util")] + /// Exercise production dependency scheduling with controlled authoritative observations. + pub async fn observe_readiness_dependencies_for_tests( + deadline: Instant, + metrics: &Arc, + writer: Arc, + hardware: impl Future>>, + chunkdb: impl Future>, + ) -> Result { + observe_dependencies(deadline, metrics, writer, hardware, chunkdb).await + } +} + +async fn observe_dependencies( + deadline: Instant, + metrics: &Arc, + writer: Arc, + hardware: impl Future>>, + chunkdb: impl Future>, +) -> Result { + let (disks, chunks) = tokio::join!( + async { + let disks = observe("hardware", deadline, &metrics.hardware, hardware).await?; + // Disk enumeration depends on hardware, not on the independent ChunkDB read. + observe_online_disks(disks, writer, deadline, metrics).await + }, + observe("ChunkDB", deadline, &metrics.chunkdb, chunkdb), + ); + chunks?; + disks +} + +async fn observe_online_disks( + disks: Vec, + writer: Arc, + deadline: Instant, + metrics: &Arc, +) -> Result { + let online: Vec<_> = disks + .iter() + .filter(|disk| disk.value.status == HwStatus::Up as i32) + .collect(); + if online.is_empty() { + return Err(IoError::Topology("no online disks are available".into())); + } + // Keep background observation work bounded independently of cluster size. + for batch in online.chunks(8) { + let mut reads = tokio::task::JoinSet::new(); + for disk in batch { + let writer = writer.clone(); + let metrics = metrics.clone(); + let id = disk.disk_id; + reads.spawn(async move { observe_disk(writer.as_ref(), id, deadline, &metrics).await }); + } + while let Some(read) = reads.join_next().await { + read.map_err(|error| IoError::ReadFailed(error.to_string()))??; + } + } + Ok(online.len()) +} + +async fn observe_disk( + writer: &dyn DiskWriter, + id: DiskId, + deadline: Instant, + metrics: &crate::metrics::ReadinessMetrics, +) -> Result<()> { + let segment = Segment { + disk_id: Some(id), + unit_count: 1, + ..Default::default() + }; + observe("DiskIO", deadline, &metrics.diskio, async { + let bytes = writer.read_priority(&segment, 4096, 0, 4096).await?; + if bytes.len() != 4096 { + return Err(IoError::ReadFailed("short disk header".into())); + } + Ok(()) + }) + .await + .map_err(|error| IoError::ReadFailed(format!("DiskIO disk {id:?}: {error}"))) +} + +#[cfg(feature = "test-util")] +/// Exercise the production physical probe with an injected routed transport. +pub async fn observe_disk_readiness_for_tests( + writer: &dyn DiskWriter, + id: DiskId, + deadline: Instant, + metrics: &crate::metrics::ReadinessMetrics, +) -> Result<()> { + observe_disk(writer, id, deadline, metrics).await } async fn observe( diff --git a/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs b/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs index 174d93b10..f4479a00a 100644 --- a/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs +++ b/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs @@ -193,6 +193,17 @@ pub trait DiskWriter: Send + Sync { )) } + /// Read a bounded control range through the independently provisioned priority lane. + async fn read_priority( + &self, + seg: &Segment, + unit_bytes: u64, + segment_offset: u64, + length: u32, + ) -> Result { + self.read(seg, unit_bytes, segment_offset, length).await + } + /// Read while preserving backing metadata when the transport provides it. async fn read_payload( &self, diff --git a/lib/crowdb-chunk-client/src/disk_io/routing.rs b/lib/crowdb-chunk-client/src/disk_io/routing.rs index a3aa87255..17ba4af27 100644 --- a/lib/crowdb-chunk-client/src/disk_io/routing.rs +++ b/lib/crowdb-chunk-client/src/disk_io/routing.rs @@ -9,12 +9,15 @@ use async_trait::async_trait; use bytes::Bytes; use crowdb_diskio_client::{ DiskId, DiskioClient, DiskioClientConfig, DiskioError, Durability, NativeDiskIoRoutes, SegmentTarget, + TrafficLane, }; use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; use crowdb_protocol::diskdb::rpc::Segment; use crate::{DiskWriter, IoError, Result}; +mod reads; + /// Disk writer backed by the complete semantic DiskIO client. pub struct RoutedDiskWriter { client: Arc, @@ -370,24 +373,20 @@ impl DiskWriter for RoutedDiskWriter { segment_offset: u64, length: u32, ) -> Result { - let target = Self::target(seg, unit_bytes).map_err(|error| IoError::ReadFailed(error.to_string()))?; - let result = self - .client - .read_payload(target, segment_offset, length, self.client.normal_options()) - .await; - match result { - Err(DiskioError::TransportUnavailable(_) | DiskioError::TopologyUnavailable(_)) => { - // A restarted DiskIO can advertise a new endpoint while the - // retained generation still points at its old connection. - // Reads are safe to retry once after an authoritative refresh. - self.client.refresh().await.map_err(map_read_error)?; - self.client - .read_payload(target, segment_offset, length, self.client.normal_options()) - .await - .map_err(map_read_error) - } - result => result.map_err(map_read_error), - } + self.read_payload_in_lane(seg, unit_bytes, segment_offset, length, TrafficLane::Normal) + .await + } + + async fn read_priority( + &self, + seg: &Segment, + unit_bytes: u64, + segment_offset: u64, + length: u32, + ) -> Result { + self.read_payload_in_lane(seg, unit_bytes, segment_offset, length, TrafficLane::Priority) + .await + .map(crowdb_rpc_ffi::OwnedPayload::into_bytes) } } diff --git a/lib/crowdb-chunk-client/src/disk_io/routing/reads.rs b/lib/crowdb-chunk-client/src/disk_io/routing/reads.rs new file mode 100644 index 000000000..e7549ef35 --- /dev/null +++ b/lib/crowdb-chunk-client/src/disk_io/routing/reads.rs @@ -0,0 +1,44 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Preserve semantic routing and refresh behavior for both transport lanes. + +use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_rpc_ffi::OwnedPayload; + +use super::{map_read_error, DiskioError, IoError, Result, RoutedDiskWriter, TrafficLane}; + +impl RoutedDiskWriter { + pub(super) async fn read_payload_in_lane( + &self, + seg: &Segment, + unit_bytes: u64, + segment_offset: u64, + length: u32, + lane: TrafficLane, + ) -> Result { + let target = Self::target(seg, unit_bytes).map_err(|error| IoError::ReadFailed(error.to_string()))?; + let options = || { + let mut options = self.client.normal_options(); + options.lane = lane; + options + }; + let result = self + .client + .read_payload(target, segment_offset, length, options()) + .await; + match result { + Err(DiskioError::TransportUnavailable(_) | DiskioError::TopologyUnavailable(_)) => { + // A restarted DiskIO can advertise a new endpoint while the + // retained generation still points at its old connection. + // Reads are safe to retry once after an authoritative refresh. + self.client.refresh().await.map_err(map_read_error)?; + self.client + .read_payload(target, segment_offset, length, options()) + .await + .map_err(map_read_error) + } + result => result.map_err(map_read_error), + } + } +} diff --git a/lib/crowdb-chunk-client/tests/common/readiness_peer.rs b/lib/crowdb-chunk-client/tests/common/readiness_peer.rs new file mode 100644 index 000000000..62c7fe7d4 --- /dev/null +++ b/lib/crowdb-chunk-client/tests/common/readiness_peer.rs @@ -0,0 +1,172 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_diskio_client::{DiskId, DiskioClient, DiskioClientConfig, TestDiskRoute}; +use crowdb_protocol::diskio_fb::{ + FBDiskIoRetCode, FBDiskReadRequest, FBDiskReadResponse, FBDiskReadResponseArgs, +}; +use flatbuffers::FlatBufferBuilder; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + sync::Notify, + task::{JoinHandle, JoinSet}, +}; + +#[derive(Clone, Copy)] +pub enum TestProbeReply { + Ready, + Rejected, + Held, +} + +#[derive(Default)] +pub struct TestReadState { + pub bulk_started: Notify, + pub release_bulk: Notify, + pub release_probe: Notify, + pub headers: AtomicUsize, +} + +pub struct TestReadinessPeer { + pub state: Arc, + address: String, + worker: JoinHandle<()>, +} + +impl TestReadinessPeer { + pub async fn start(reply: TestProbeReply) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let state = Arc::new(TestReadState::default()); + let worker = tokio::spawn({ + let state = state.clone(); + async move { + let mut connections = JoinSet::new(); + loop { + tokio::select! { + value=listener.accept()=>{let(stream,_)=value.unwrap();connections.spawn(serve(stream,state.clone(),reply));}, + value=connections.join_next(),if !connections.is_empty()=>{value.unwrap().unwrap();} + } + } + } + }); + Self { + state, + address, + worker, + } + } + + pub fn client(&self) -> Arc { + Arc::new( + DiskioClient::connect_for_tests( + vec![TestDiskRoute { + disk_id: DiskId::new(1, 2), + rack_id: 1, + node_id: 1, + disk_group_id: 1, + instance_id: 1, + endpoint: self.address.clone(), + }], + DiskioClientConfig { + normal_connections_per_endpoint: 1, + priority_connections_per_endpoint: 1, + retry_attempts: 1, + ..Default::default() + }, + ) + .unwrap(), + ) + } +} + +impl Drop for TestReadinessPeer { + fn drop(&mut self) { + self.worker.abort(); + } +} + +async fn serve(mut stream: TcpStream, state: Arc, reply: TestProbeReply) { + loop { + let mut header = [0; 14]; + if stream.read_exact(&mut header).await.is_err() { + return; + } + assert_eq!(u16::from_le_bytes(header[..2].try_into().unwrap()), 0xca70); + let kind = u16::from_le_bytes(header[2..4].try_into().unwrap()); + let length = u32::from_le_bytes(header[4..8].try_into().unwrap()) as usize; + assert!(length <= 4096); + assert_eq!(&header[8..12], &[0; 4]); + let mut control = vec![0; length]; + stream.read_exact(&mut control).await.unwrap(); + if kind == 0xfffe { + assert_eq!(control.len(), 160); + let mut capabilities = vec![0; 160]; + capabilities[..4].copy_from_slice(&[0x44, 0x52, 4, 0]); + capabilities[4..12].copy_from_slice(&1_u64.to_le_bytes()); + header[13] = 0; + if stream.write_all(&header).await.is_err() { + return; + } + if stream.write_all(&capabilities).await.is_err() { + return; + } + continue; + } + let request = flatbuffers::root::(&control).unwrap(); + let id = request.id(); + let size = request.size() as usize; + assert!(matches!(size, 4096 | 65536)); + let bulk = size == 65536; + if !bulk { + state.headers.fetch_add(1, Ordering::Release); + if matches!(reply, TestProbeReply::Held) { + state.release_probe.notified().await; + } + } + let code = if !bulk && matches!(reply, TestProbeReply::Rejected) { + FBDiskIoRetCode::DiskNotExist + } else { + FBDiskIoRetCode::Success + }; + let mut builder = FlatBufferBuilder::new(); + let value = FBDiskReadResponse::create( + &mut builder, + &FBDiskReadResponseArgs { + id, + ret_code: code, + ..Default::default() + }, + ); + builder.finish(value, None); + let control = builder.finished_data(); + let bytes = if code == FBDiskIoRetCode::Success { size } else { 0 }; + let mut response = Vec::with_capacity(14 + control.len()); + response.extend_from_slice(&0xca70_u16.to_le_bytes()); + response.extend_from_slice(&(kind + 1).to_le_bytes()); + response.extend_from_slice(&u32::try_from(control.len()).unwrap().to_le_bytes()); + response.extend_from_slice(&u32::try_from(bytes).unwrap().to_le_bytes()); + response.extend_from_slice(&[14, 0]); + response.extend_from_slice(control); + if stream.write_all(&response).await.is_err() { + return; + } + if bulk { + if stream.write_all(&[7]).await.is_err() { + return; + } + state.bulk_started.notify_one(); + state.release_bulk.notified().await; + if stream.write_all(&vec![7; bytes - 1]).await.is_err() { + return; + } + } else if bytes != 0 && stream.write_all(&vec![9; bytes]).await.is_err() { + return; + } + } +} diff --git a/lib/crowdb-chunk-client/tests/common/readiness_schedule.rs b/lib/crowdb-chunk-client/tests/common/readiness_schedule.rs new file mode 100644 index 000000000..c1822fc8d --- /dev/null +++ b/lib/crowdb-chunk-client/tests/common/readiness_schedule.rs @@ -0,0 +1,106 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_chunk_client::{DiskWriter, IoError, Result}; +use crowdb_kv_client::DiskRecord; +use crowdb_protocol::{ + common::{DiskId, HwStatus}, + diskdb::rpc::{DiskValue, Segment}, +}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +pub enum TestHeaderReply { + Ready, + Rejected, + Short, + Held, +} + +pub struct TestReadinessWriter { + pub delay: Duration, + pub reply: TestHeaderReply, + pub started: AtomicUsize, + pub active: Arc, + pub maximum: AtomicUsize, + pub read_started: tokio::sync::Notify, +} + +impl TestReadinessWriter { + pub fn new(delay: Duration, reply: TestHeaderReply) -> Arc { + Arc::new(Self { + delay, + reply, + started: AtomicUsize::new(0), + active: Arc::default(), + maximum: AtomicUsize::new(0), + read_started: tokio::sync::Notify::new(), + }) + } +} + +struct TestActiveRead(Arc); +impl Drop for TestActiveRead { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } +} + +#[async_trait] +impl DiskWriter for TestReadinessWriter { + async fn write(&self, _: &Segment, _: u64, _: Bytes) -> Result<()> { + panic!("readiness must not write") + } + + async fn write_at_byte_offset(&self, _: &Segment, _: u64, _: u64, _: Bytes) -> Result<()> { + panic!("readiness must not write") + } + + async fn read_priority( + &self, + segment: &Segment, + unit_bytes: u64, + offset: u64, + length: u32, + ) -> Result { + assert!(segment.disk_id.is_some()); + assert_eq!((unit_bytes, offset, length), (4096, 0, 4096)); + self.started.fetch_add(1, Ordering::SeqCst); + let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; + self.maximum.fetch_max(active, Ordering::SeqCst); + let _active = TestActiveRead(self.active.clone()); + self.read_started.notify_one(); + tokio::time::sleep(self.delay).await; + match self.reply { + TestHeaderReply::Ready => Ok(Bytes::from(vec![7; 4096])), + TestHeaderReply::Rejected => Err(IoError::ReadFailed("injected physical rejection".into())), + TestHeaderReply::Short => Ok(Bytes::from_static(b"short")), + TestHeaderReply::Held => std::future::pending().await, + } + } +} + +pub fn test_disks(count: usize) -> Vec { + (0..count) + .map(|n| DiskRecord { + rack_id: 1, + node_id: 1, + disk_group_id: n as u64 + 1, + disk_id: DiskId { + high: 1, + low: n as u64 + 1, + }, + value: DiskValue { + status: HwStatus::Up as i32, + ..Default::default() + }, + }) + .collect() +} diff --git a/lib/crowdb-chunk-client/tests/readiness_priority_test.rs b/lib/crowdb-chunk-client/tests/readiness_priority_test.rs new file mode 100644 index 000000000..9114b23de --- /dev/null +++ b/lib/crowdb-chunk-client/tests/readiness_priority_test.rs @@ -0,0 +1,103 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/readiness_peer.rs"] +mod peer; +#[path = "common/read_payload.rs"] +mod storage; + +use crowdb_chunk_client::{ + client::observe_disk_readiness_for_tests, metrics::ReadinessMetrics, ChunkClientMetrics, ChunkIoClient, + RoutedDiskWriter, +}; +use crowdb_common::metrics::{MetricPoint, MetricsRegistry}; +use crowdb_diskio_client::{DiskId, OperationOptions, SegmentTarget}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkType}; +use crowdb_protocol::common::DiskId as ProtoDiskId; +use crowdb_rpc_ffi::OwnedPayload; +use peer::{TestProbeReply, TestReadinessPeer}; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; +use tokio::time::Instant; + +async fn exercise(reply: TestProbeReply) -> (crowdb_chunk_client::Result<()>, usize, ReadinessMetrics) { + let peer = TestReadinessPeer::start(reply).await; + let client = peer.client(); + let target = SegmentTarget::new(DiskId::new(1, 2), 0, 0, 16, 4096).unwrap(); + let bulk = tokio::spawn({ + let client = client.clone(); + async move { + client + .read(target, 0, 65536, OperationOptions::within(Duration::from_secs(5))) + .await + } + }); + tokio::time::timeout(Duration::from_secs(2), peer.state.bulk_started.notified()) + .await + .unwrap(); + let mut registry = MetricsRegistry::new(); + let aggregate = Arc::new(ChunkClientMetrics::register(&mut registry)); + let io = ChunkIoClient::from_parts( + Arc::new(storage::TestPayloadStorage { + chunk: Chunk::default(), + data: OwnedPayload::from(Vec::new()), + }), + Arc::new(RoutedDiskWriter::from_client(client.clone())), + ChunkType::S3, + ) + .with_metrics(&aggregate); + let (_, writer) = io.storage_parts(); + let metrics = ReadinessMetrics::default(); + let result = observe_disk_readiness_for_tests( + writer.as_ref(), + ProtoDiskId { high: 1, low: 2 }, + Instant::now() + Duration::from_millis(750), + &metrics, + ) + .await; + assert!(matches!( + registry.snapshot_named("chunkio.diskio.read.e2e.lh", 1.0), + Some(MetricPoint::Histogram { total: 1, .. }) + )); + assert!( + !bulk.is_finished(), + "the bulk response must remain held while readiness completes" + ); + let headers = peer.state.headers.load(Ordering::Acquire); + peer.state.release_probe.notify_one(); + peer.state.release_bulk.notify_one(); + let bytes = bulk.await.unwrap().unwrap(); + assert_eq!(bytes.len(), 65536); + assert!(bytes.iter().all(|b| *b == 7)); + client.shutdown(); + (result, headers, metrics) +} + +#[tokio::test] +async fn readiness_header_progresses_while_a_bulk_response_is_held() { + let (result, headers, metrics) = exercise(TestProbeReply::Ready).await; + assert!( + result.is_ok(), + "physical readiness must not wait behind a bulk TCP response: {result:?}" + ); + assert_eq!(headers, 1); + assert_eq!(metrics.snapshot().diskio.errors, 0); +} + +#[tokio::test] +async fn rejected_priority_header_still_fails_readiness() { + let (result, headers, metrics) = exercise(TestProbeReply::Rejected).await; + assert!(result.is_err()); + assert_eq!(headers, 1); + assert_eq!(metrics.snapshot().diskio.errors, 1); +} + +#[tokio::test] +async fn held_priority_header_still_owes_the_original_readiness_deadline() { + let (result, headers, metrics) = exercise(TestProbeReply::Held).await; + assert!(result.unwrap_err().to_string().contains("readiness deadline")); + assert_eq!(headers, 1); + assert_eq!(metrics.snapshot().diskio.errors, 1); +} diff --git a/lib/crowdb-chunk-client/tests/readiness_test.rs b/lib/crowdb-chunk-client/tests/readiness_test.rs index 9c6d96bba..efe59583f 100644 --- a/lib/crowdb-chunk-client/tests/readiness_test.rs +++ b/lib/crowdb-chunk-client/tests/readiness_test.rs @@ -1,12 +1,238 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use std::sync::Arc; +use std::sync::{atomic::Ordering, Arc}; use std::time::Duration; -use crowdb_chunk_client::{client::observe_readiness_phase_for_tests, metrics::ReadinessMetrics}; +use crowdb_chunk_client::{ + client::observe_readiness_phase_for_tests, metrics::ReadinessMetrics, ChunkIoClient, IoError, +}; use tokio::time::Instant; +#[path = "common/readiness_schedule.rs"] +mod schedule; + +#[tokio::test(start_paused = true)] +async fn disk_headers_start_after_hardware_without_waiting_for_chunkdb() { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = + schedule::TestReadinessWriter::new(Duration::from_millis(100), schedule::TestHeaderReply::Ready); + let start = Instant::now(); + let result = crowdb_chunk_client::ChunkIoClient::observe_readiness_dependencies_for_tests( + start + Duration::from_millis(750), + &metrics, + writer.clone(), + async { + tokio::time::sleep(Duration::from_millis(50)).await; + Ok(schedule::test_disks(1)) + }, + async { + tokio::time::sleep(Duration::from_millis(700)).await; + Ok(()) + }, + ) + .await; + assert!( + result.is_ok(), + "independent hardware/header and ChunkDB work must fit the original 750ms total: {result:?}" + ); + assert_eq!(result.unwrap(), 1); + assert_eq!(Instant::now() - start, Duration::from_millis(700)); + assert_eq!(writer.started.load(Ordering::SeqCst), 1); + assert_eq!(metrics.snapshot().diskio.errors, 0); +} + +#[tokio::test(start_paused = true)] +async fn held_chunkdb_still_fails_after_a_successful_physical_header() { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = + schedule::TestReadinessWriter::new(Duration::from_millis(100), schedule::TestHeaderReply::Ready); + let deadline = Instant::now() + Duration::from_millis(750); + let result = ChunkIoClient::observe_readiness_dependencies_for_tests( + deadline, + &metrics, + writer.clone(), + async { Ok(schedule::test_disks(1)) }, + std::future::pending(), + ) + .await; + assert!(result.unwrap_err().to_string().contains("ChunkDB read exceeded")); + assert_eq!(Instant::now(), deadline); + assert_eq!(writer.started.load(Ordering::SeqCst), 1); + assert_eq!(metrics.snapshot().chunkdb.errors, 1); + assert_eq!(metrics.snapshot().diskio.errors, 0); +} + +#[tokio::test(start_paused = true)] +async fn rejected_chunkdb_still_fails_after_a_successful_physical_header() { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = + schedule::TestReadinessWriter::new(Duration::from_millis(100), schedule::TestHeaderReply::Ready); + let result = ChunkIoClient::observe_readiness_dependencies_for_tests( + Instant::now() + Duration::from_millis(750), + &metrics, + writer.clone(), + async { Ok(schedule::test_disks(1)) }, + async { Err(IoError::ReadFailed("injected ChunkDB rejection".into())) }, + ) + .await; + assert!(result + .unwrap_err() + .to_string() + .contains("injected ChunkDB rejection")); + assert_eq!(writer.started.load(Ordering::SeqCst), 1); + assert_eq!(metrics.snapshot().chunkdb.errors, 1); + assert_eq!(metrics.snapshot().diskio.errors, 0); +} + +#[tokio::test(start_paused = true)] +async fn held_physical_header_still_owes_the_original_total_deadline() { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = schedule::TestReadinessWriter::new(Duration::ZERO, schedule::TestHeaderReply::Held); + let deadline = Instant::now() + Duration::from_millis(750); + let result = ChunkIoClient::observe_readiness_dependencies_for_tests( + deadline, + &metrics, + writer.clone(), + async { Ok(schedule::test_disks(1)) }, + async { Ok(()) }, + ) + .await; + assert!(result.unwrap_err().to_string().contains("DiskIO read exceeded")); + assert_eq!(Instant::now(), deadline); + assert_eq!(metrics.snapshot().diskio.errors, 1); + assert_eq!(writer.active.load(Ordering::SeqCst), 0); +} + +#[tokio::test(start_paused = true)] +async fn rejected_or_short_physical_headers_cannot_pass_with_ready_chunkdb() { + for (reply, message) in [ + (schedule::TestHeaderReply::Rejected, "injected physical rejection"), + (schedule::TestHeaderReply::Short, "short disk header"), + ] { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = schedule::TestReadinessWriter::new(Duration::ZERO, reply); + let result = ChunkIoClient::observe_readiness_dependencies_for_tests( + Instant::now() + Duration::from_millis(750), + &metrics, + writer.clone(), + async { Ok(schedule::test_disks(1)) }, + async { Ok(()) }, + ) + .await; + assert!(result.unwrap_err().to_string().contains(message)); + assert_eq!(metrics.snapshot().diskio.errors, 1); + assert_eq!(writer.started.load(Ordering::SeqCst), 1); + } +} + +#[tokio::test(start_paused = true)] +async fn failed_or_held_hardware_never_starts_a_physical_read() { + for held in [false, true] { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = schedule::TestReadinessWriter::new(Duration::ZERO, schedule::TestHeaderReply::Ready); + let result = ChunkIoClient::observe_readiness_dependencies_for_tests( + Instant::now() + Duration::from_millis(750), + &metrics, + writer.clone(), + async { + if held { + std::future::pending().await + } else { + Err(IoError::Topology("injected hardware rejection".into())) + } + }, + async { Ok(()) }, + ) + .await; + let message = result.unwrap_err().to_string(); + assert!(message.contains(if held { + "hardware read exceeded" + } else { + "injected hardware rejection" + })); + assert_eq!(metrics.snapshot().hardware.errors, 1); + assert_eq!(metrics.snapshot().diskio.observations, 0); + assert_eq!(writer.started.load(Ordering::SeqCst), 0); + } +} + +#[tokio::test(start_paused = true)] +async fn empty_or_offline_hardware_does_not_claim_physical_readiness() { + for count in [0, 1] { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = schedule::TestReadinessWriter::new(Duration::ZERO, schedule::TestHeaderReply::Ready); + let mut disks = schedule::test_disks(count); + for disk in &mut disks { + disk.value.status = crowdb_protocol::common::HwStatus::Offline as i32; + } + let result = ChunkIoClient::observe_readiness_dependencies_for_tests( + Instant::now() + Duration::from_millis(750), + &metrics, + writer.clone(), + async { Ok(disks) }, + async { Ok(()) }, + ) + .await; + assert!(result.unwrap_err().to_string().contains("no online disks")); + assert_eq!(writer.started.load(Ordering::SeqCst), 0); + assert_eq!(metrics.snapshot().diskio.observations, 0); + } +} + +#[tokio::test(start_paused = true)] +async fn physical_headers_keep_the_existing_eight_read_batch_bound() { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = + schedule::TestReadinessWriter::new(Duration::from_millis(100), schedule::TestHeaderReply::Ready); + let start = Instant::now(); + let result = ChunkIoClient::observe_readiness_dependencies_for_tests( + start + Duration::from_millis(750), + &metrics, + writer.clone(), + async { Ok(schedule::test_disks(17)) }, + async { Ok(()) }, + ) + .await; + assert_eq!(result.unwrap(), 17); + assert_eq!(writer.started.load(Ordering::SeqCst), 17); + assert_eq!(writer.maximum.load(Ordering::SeqCst), 8); + assert_eq!(Instant::now() - start, Duration::from_millis(300)); + assert_eq!(metrics.snapshot().diskio.observations, 17); +} + +#[tokio::test(start_paused = true)] +async fn cancelling_dependency_observation_aborts_physical_reads_and_both_phase_guards() { + let metrics = Arc::new(ReadinessMetrics::default()); + let writer = schedule::TestReadinessWriter::new(Duration::ZERO, schedule::TestHeaderReply::Held); + let worker = tokio::spawn({ + let metrics = metrics.clone(); + let writer = writer.clone(); + async move { + ChunkIoClient::observe_readiness_dependencies_for_tests( + Instant::now() + Duration::from_millis(750), + &metrics, + writer, + async { Ok(schedule::test_disks(1)) }, + std::future::pending(), + ) + .await + } + }); + writer.read_started.notified().await; + assert_eq!(metrics.snapshot().chunkdb.inflight, 1); + assert_eq!(metrics.snapshot().diskio.inflight, 1); + worker.abort(); + assert!(worker.await.unwrap_err().is_cancelled()); + tokio::task::yield_now().await; + assert_eq!(writer.active.load(Ordering::SeqCst), 0); + let snapshot = metrics.snapshot(); + assert_eq!(snapshot.chunkdb.inflight, 0); + assert_eq!(snapshot.chunkdb.errors, 1); + assert_eq!(snapshot.diskio.inflight, 0); + assert_eq!(snapshot.diskio.errors, 1); +} + #[tokio::test(start_paused = true)] async fn phase_deadlines_share_the_remaining_budget_and_identify_the_dependency() { let metrics = ReadinessMetrics::default(); diff --git a/lib/crowdb-chunkdb-client/src/client.rs b/lib/crowdb-chunkdb-client/src/client.rs index 49ad7bd7f..77232e373 100644 --- a/lib/crowdb-chunkdb-client/src/client.rs +++ b/lib/crowdb-chunkdb-client/src/client.rs @@ -212,6 +212,10 @@ impl ChunkdbClient { async fn endpoints_for_chunk(&self, chunk_id: Option<&ChunkId>) -> Result> { self.refresh_endpoints_if_due().await; + self.cached_endpoints_for_chunk(chunk_id).await + } + + async fn cached_endpoints_for_chunk(&self, chunk_id: Option<&ChunkId>) -> Result> { if let Some(id) = chunk_id { let binding = &self.range_binding; let owner = binding diff --git a/lib/crowdb-chunkdb-client/src/readiness.rs b/lib/crowdb-chunkdb-client/src/readiness.rs index d5d9fa913..9d9b79aad 100644 --- a/lib/crowdb-chunkdb-client/src/readiness.rs +++ b/lib/crowdb-chunkdb-client/src/readiness.rs @@ -31,7 +31,14 @@ impl ChunkdbClient { high: u64::from(purpose) << 56, low: 0, }; - match self.query_chunk(QueryChunkRequest { chunk_id: Some(id) }).await { + let reply = self + .with_probe_retry(&id, |transport, endpoint| async move { + transport + .send_query_chunk(&endpoint, &QueryChunkRequest { chunk_id: Some(id) }) + .await + }) + .await; + match reply { Ok(reply) if reply.chunk.as_ref().is_some_and(|chunk| chunk.id == Some(id)) => Ok(()), Err(ChunkdbClientError::NotFound(_)) => Ok(()), Ok(_) => Err(ChunkdbClientError::Rpc( diff --git a/lib/crowdb-chunkdb-client/src/retry.rs b/lib/crowdb-chunkdb-client/src/retry.rs index 807249f10..eab2b26eb 100644 --- a/lib/crowdb-chunkdb-client/src/retry.rs +++ b/lib/crowdb-chunkdb-client/src/retry.rs @@ -7,6 +7,11 @@ use crowdb_protocol::chunkdb::rpc::{AllocateChunkRequest, AllocateChunkResponse} use crowdb_protocol::common::ChunkId; use std::{sync::Arc, time::Duration}; +enum RouteRefresh { + Periodic, + AfterFailure, +} + impl ChunkdbClient { pub(super) async fn with_rpc_retry(&self, chunk_id: Option<&ChunkId>, op: F) -> Result where @@ -14,7 +19,7 @@ impl ChunkdbClient { Fut: std::future::Future>, { self.rpc_transport - .with_request_identity(self.retry_rpc(chunk_id, false, op)) + .with_request_identity(self.retry_rpc(chunk_id, false, RouteRefresh::Periodic, op)) .await } @@ -24,12 +29,30 @@ impl ChunkdbClient { Fut: std::future::Future>, { self.rpc_transport - .with_request_identity(self.retry_rpc(chunk_id, true, op)) + .with_request_identity(self.retry_rpc(chunk_id, true, RouteRefresh::Periodic, op)) + .await + } + + /// A real readiness point read must not enumerate authority before using + /// an initialized route. Cold discovery and failed-route recovery remain. + pub(crate) async fn with_probe_retry(&self, chunk_id: &ChunkId, op: F) -> Result + where + F: Fn(Arc, String) -> Fut, + Fut: std::future::Future>, + { + self.rpc_transport + .with_request_identity(self.retry_rpc(Some(chunk_id), true, RouteRefresh::AfterFailure, op)) .await } /// Execute a crowdb-rpc call with retry on transient errors. - async fn retry_rpc(&self, chunk_id: Option<&ChunkId>, read_only: bool, op: F) -> Result + async fn retry_rpc( + &self, + chunk_id: Option<&ChunkId>, + read_only: bool, + refresh: RouteRefresh, + op: F, + ) -> Result where F: Fn(Arc, String) -> Fut, Fut: std::future::Future>, @@ -39,7 +62,10 @@ impl ChunkdbClient { let mut attempts = 0u32; let mut backoff = self.retry.initial_backoff; loop { - let endpoints = self.endpoints_for_chunk(chunk_id).await?; + let endpoints = match refresh { + RouteRefresh::Periodic => self.endpoints_for_chunk(chunk_id).await?, + RouteRefresh::AfterFailure => self.cached_endpoints_for_chunk(chunk_id).await?, + }; let mut last_error = None; for endpoint in endpoints { match tokio::time::timeout_at(deadline, op(Arc::clone(&transport), endpoint)) diff --git a/lib/crowdb-console-shared/src/deployment/admission.rs b/lib/crowdb-console-shared/src/deployment/admission.rs index 4ef0d5f3d..a2dc9cb8f 100644 --- a/lib/crowdb-console-shared/src/deployment/admission.rs +++ b/lib/crowdb-console-shared/src/deployment/admission.rs @@ -15,6 +15,9 @@ use crate::config::NodeEntry; use crate::error::{Error, Result}; use crate::ssh::{Session, SshCreds}; +mod process_observation; +pub use process_observation::RemoteProcessObservation; + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SshNodeIdentity { pub discovery_id: String, diff --git a/lib/crowdb-console-shared/src/deployment/admission/process_observation.rs b/lib/crowdb-console-shared/src/deployment/admission/process_observation.rs new file mode 100644 index 000000000..00491aebd --- /dev/null +++ b/lib/crowdb-console-shared/src/deployment/admission/process_observation.rs @@ -0,0 +1,83 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! A single sampler owns its authenticated, read-only monitor connection. + +use crate::{ + config::NodeEntry, + error::{Error, Result}, + ssh::Session, +}; +use crowdb_protocol::mgmt::node::{NodeBinding, NodeControl}; +use serde_json::Value; +use std::time::Duration; +use tokio::time::Instant; + +/// Bound process observations reuse SSH without sharing sessions with mutations. +/// Dropping a read future discards its connection; the next cadence reconnects. +pub struct RemoteProcessObservation { + node: NodeEntry, + binding: NodeBinding, + connection: Option<(Session, Instant)>, +} + +impl RemoteProcessObservation { + /// Construct a reader for one accepted node using its provisioned local key. + /// + /// # Errors + /// Rejects password fallback or a missing explicit key. + pub fn new(node: NodeEntry, binding: NodeBinding) -> Result { + if node.ssh_password.is_some() || node.ssh_key.is_none() { + return Err(Error::Config( + "process observations require an explicit node key".into(), + )); + } + Ok(Self { + node, + binding, + connection: None, + }) + } + + /// Observe the same node and bootstrap, with no transport retry on failure. + /// + /// # Errors + /// Propagates authentication, command and identity failures. + pub async fn read(&mut self) -> Result { + // Keep no stored handle across an await: timeout cancellation must not + // retain a connection with an unfinished command or an unread reply. + let (mut session, connected) = match self.connection.take() { + Some((session, connected)) if connected.elapsed() < Duration::from_secs(60) => { + (session, connected) + } + previous => { + if let Some((session, _)) = previous { + session.close().await; + } + (super::connect(&self.node).await?, Instant::now()) + } + }; + let command = NodeControl::Processes { + cluster_id: self.binding.bootstrap.cluster_id.clone(), + node_id: self.node.id, + }; + let result = super::control(&mut session, &command).await.and_then(|value| { + if value["node_id"] != self.node.id + || value["cluster_id"] != self.binding.bootstrap.cluster_id + || value["bootstrap"] + != serde_json::to_value(&self.binding.bootstrap).map_err(super::config_error)? + { + return Err(Error::Config( + "process observation identity differs from accepted node".into(), + )); + } + Ok(value) + }); + if result.is_ok() { + self.connection = Some((session, connected)); + } else { + session.close().await; + } + result + } +} diff --git a/lib/crowdb-console-shared/tests/common/ssh_server.rs b/lib/crowdb-console-shared/tests/common/ssh_server.rs index 276f2dbb3..03d5b108d 100644 --- a/lib/crowdb-console-shared/tests/common/ssh_server.rs +++ b/lib/crowdb-console-shared/tests/common/ssh_server.rs @@ -1,7 +1,10 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use std::sync::Arc; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; use async_trait::async_trait; use russh::keys::key::{KeyPair, PublicKey}; @@ -12,12 +15,14 @@ use russh::{Channel, ChannelId, CryptoVec}; #[allow(dead_code)] pub enum TestReply { Stdout(Vec), + DelayedStdout(Vec, std::time::Duration), Failure(String), } pub struct TestSshServer { pub port: u16, server: tokio::task::JoinHandle<()>, + connections: Arc, } impl Drop for TestSshServer { @@ -28,6 +33,9 @@ impl Drop for TestSshServer { #[allow(dead_code)] impl TestSshServer { + pub fn connections(&self) -> usize { + self.connections.load(Ordering::Relaxed) + } pub async fn start(key: PublicKey) -> Self { Self::start_with_reply(key, None).await } @@ -51,12 +59,15 @@ impl TestSshServer { keys: vec![host], ..Default::default() }); + let connections = Arc::new(AtomicUsize::new(0)); + let accepted_count = Arc::clone(&connections); let server = tokio::spawn(async move { let mut sessions = tokio::task::JoinSet::new(); loop { tokio::select! { accepted = listener.accept() => { let (stream, _) = accepted.unwrap(); + accepted_count.fetch_add(1, Ordering::Relaxed); let config = config.clone(); let handler = TestHandler { key: key.clone(), reply: reply.clone() }; sessions.spawn(async move { @@ -69,7 +80,11 @@ impl TestSshServer { } } }); - Self { port, server } + Self { + port, + server, + connections, + } } } @@ -109,6 +124,11 @@ impl Handler for TestHandler { session.data(channel, CryptoVec::from_slice(bytes)); session.exit_status_request(channel, 0); } + TestReply::DelayedStdout(bytes, delay) => { + tokio::time::sleep(*delay).await; + session.data(channel, CryptoVec::from_slice(bytes)); + session.exit_status_request(channel, 0); + } TestReply::Failure(error) => { session.extended_data(channel, 1, CryptoVec::from_slice(error.as_bytes())); session.exit_status_request(channel, 1); diff --git a/lib/crowdb-console-shared/tests/process_observation_ssh_test.rs b/lib/crowdb-console-shared/tests/process_observation_ssh_test.rs new file mode 100644 index 000000000..5895ad900 --- /dev/null +++ b/lib/crowdb-console-shared/tests/process_observation_ssh_test.rs @@ -0,0 +1,99 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_console_shared::{config::NodeEntry, deployment::admission::RemoteProcessObservation}; +use crowdb_protocol::mgmt::{node::NodeBinding, SystemBootstrapIdentity}; +use serde_json::json; +use std::time::Duration; + +#[path = "common/ssh_server.rs"] +mod ssh_server; + +#[tokio::test] +async fn bound_process_reader_reuses_authentication_and_discards_failed_or_cancelled_reads() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("process-observation-ssh"); + std::env::set_var("CROWDB_KV_KNOWN_HOSTS", root.path().join("known-hosts")); + let key = russh::keys::key::KeyPair::generate_ed25519().unwrap(); + let path = root.path().join("node-key"); + russh::keys::encode_pkcs8_pem(&key, std::fs::File::create(&path).unwrap()).unwrap(); + let binding = NodeBinding { + node_id: 1, + bootstrap: SystemBootstrapIdentity { + cluster_id: "bound-cluster".into(), + operation_id: "bootstrap".into(), + configuration_digest: "a".repeat(64), + }, + management_seeds: vec![], + }; + let value = json!({"node_id": 2, "cluster_id": "bound-cluster", "bootstrap": binding.bootstrap, "processes": {"kv": 42}}); + let bytes = serde_json::to_vec(&value).unwrap(); + for scenario in [ + "success", + "rejection", + "wrong-node", + "wrong-bootstrap", + "cancelled", + ] { + let reply = match scenario { + "success" => ssh_server::TestReply::Stdout(bytes.clone()), + "rejection" => ssh_server::TestReply::Failure("monitor rejected observation".into()), + "wrong-node" => { + let mut foreign = value.clone(); + foreign["node_id"] = json!(3); + ssh_server::TestReply::Stdout(serde_json::to_vec(&foreign).unwrap()) + } + "wrong-bootstrap" => { + let mut foreign = value.clone(); + foreign["bootstrap"]["operation_id"] = json!("foreign-bootstrap"); + ssh_server::TestReply::Stdout(serde_json::to_vec(&foreign).unwrap()) + } + _ => ssh_server::TestReply::DelayedStdout(bytes.clone(), Duration::from_secs(1)), + }; + let server = + ssh_server::TestSshServer::start_with_result(key.clone_public_key().unwrap(), reply).await; + let node = NodeEntry { + id: 2, + rack_id: 1, + host: "127.0.0.1".into(), + ssh_port: server.port, + ssh_user: "operator".into(), + ssh_key: Some(path.to_string_lossy().into_owned()), + ssh_password: None, + ssh_credential_ref: None, + }; + let mut reader = RemoteProcessObservation::new(node.clone(), binding.clone()).unwrap(); + if scenario == "success" { + for _ in 0..5 { + assert_eq!(reader.read().await.unwrap()["processes"]["kv"], 42); + } + assert_eq!( + server.connections(), + 1, + "one authentication for repeated fresh reads" + ); + } else if scenario == "cancelled" { + for _ in 0..2 { + assert!(tokio::time::timeout(Duration::from_millis(100), reader.read()) + .await + .is_err()); + } + assert_eq!( + server.connections(), + 2, + "a cancelled read must not retain its session" + ); + } else { + for _ in 0..2 { + assert!(reader.read().await.is_err(), "{scenario}"); + } + assert_eq!( + server.connections(), + 2, + "a rejected response must discard its session" + ); + } + let mut password = node; + password.ssh_password = Some("unused".into()); + assert!(RemoteProcessObservation::new(password, binding.clone()).is_err()); + } +} diff --git a/lib/crowdb-kv-client/src/client/core.rs b/lib/crowdb-kv-client/src/client/core.rs index 590c23477..53a821425 100644 --- a/lib/crowdb-kv-client/src/client/core.rs +++ b/lib/crowdb-kv-client/src/client/core.rs @@ -141,16 +141,27 @@ pub struct CrowdbKvClient { impl CrowdbKvClient { #[must_use] pub fn new(config: ClientConfig) -> Self { - Self::build(config, None) + Self::build(config, None, false) + } + + /// Keep point reads independent of bulk response frames on this client's + /// existing transport, with one additional connection per endpoint. + #[must_use] + pub fn new_with_isolated_point_reads(config: ClientConfig) -> Self { + Self::build(config, None, true) } /// Create a client using an existing shared RPC transport. #[must_use] pub fn new_with_rpc_transport(config: ClientConfig, transport: Arc) -> Self { - Self::build(config, Some(transport)) + Self::build(config, Some(transport), false) } - fn build(config: ClientConfig, transport: Option>) -> Self { + fn build( + config: ClientConfig, + transport: Option>, + isolated_point_reads: bool, + ) -> Self { // A standalone client is normal at process startup. Keep the // distinction available for diagnostics without warning on each // test listener restart or each independently deployed process. @@ -174,14 +185,19 @@ impl CrowdbKvClient { metrics: Arc::new(ClientMetrics::default()), read_endpoint_policy: config.read_endpoint_policy, rpc_transport: Some(transport.unwrap_or_else(|| { - std::sync::Arc::new(crate::KvRpcTransport::with_pool_size( + let transport = crate::KvRpcTransport::with_pool_size( config.pool_size_per_endpoint, config.enable_nagle, config.quickack, config.event_write, config.send_queue_capacity, config.rpc_workers, - )) + ); + Arc::new(if isolated_point_reads { + transport.with_isolated_point_reads() + } else { + transport + }) })), } } diff --git a/lib/crowdb-kv-client/src/client/retry.rs b/lib/crowdb-kv-client/src/client/retry.rs index 79d8ff8bb..baa80abb0 100644 --- a/lib/crowdb-kv-client/src/client/retry.rs +++ b/lib/crowdb-kv-client/src/client/retry.rs @@ -84,7 +84,11 @@ impl CrowdbKvClient { self.metrics.record_unknown_leader_wait(); self.metrics.record_leader_query(); self.metrics.record_topology_refresh(); - if let Err(e) = self.topology.refresh().await { + if let Err(e) = self + .topology + .refresh_after_failure(store_id, group_id, endpoint) + .await + { if matches!(e, Error::NoSeeds) { warn!(error = %e, "topology refresh failed in wait_and_refresh_leader"); return endpoint.to_string(); @@ -117,7 +121,11 @@ impl CrowdbKvClient { ) -> String { self.metrics.record_topology_refresh(); let mut no_seeds = false; - if let Err(e) = self.topology.refresh().await { + if let Err(e) = self + .topology + .refresh_after_failure(store_id, group_id, current) + .await + { if matches!(e, Error::NoSeeds) { no_seeds = true; } diff --git a/lib/crowdb-kv-client/src/client/topology.rs b/lib/crowdb-kv-client/src/client/topology.rs index b8bf73d95..fee39b06b 100644 --- a/lib/crowdb-kv-client/src/client/topology.rs +++ b/lib/crowdb-kv-client/src/client/topology.rs @@ -19,6 +19,7 @@ use crate::error::{Error, Result}; use crate::ReadEndpointPolicy; mod address; +mod recovery; type GroupKey = (u64, u64); @@ -234,7 +235,7 @@ impl TopologyCache { } async fn fetch_and_merge(&self, base_generation: u64) -> Result<()> { - self.fetch_group_and_merge(base_generation, None) + self.fetch_group_and_merge(base_generation, None, None) .await .map(|_| ()) } @@ -247,7 +248,7 @@ impl TopologyCache { // A recent unrelated snapshot cannot answer a missing group's query. let generation = self.snapshot.load().generation; let result = self - .fetch_group_and_merge(generation, Some((store_id, group_id))) + .fetch_group_and_merge(generation, Some((store_id, group_id)), None) .await; *last = Instant::now(); result @@ -257,6 +258,7 @@ impl TopologyCache { &self, base_generation: u64, required_group: Option, + rejected_endpoint: Option<&str>, ) -> Result> { let seeds = self.seeds.read().unwrap().clone(); if seeds.is_empty() { @@ -287,7 +289,9 @@ impl TopologyCache { .map(|remote| remote.endpoint.clone()) } }) - .filter(|endpoint| !endpoint.is_empty()); + .filter(|endpoint| { + !endpoint.is_empty() && rejected_endpoint != Some(endpoint.as_str()) + }); if required_group.is_some() && endpoint.is_none() { continue; } diff --git a/lib/crowdb-kv-client/src/client/topology/recovery.rs b/lib/crowdb-kv-client/src/client/topology/recovery.rs new file mode 100644 index 000000000..7f7e018f3 --- /dev/null +++ b/lib/crowdb-kv-client/src/client/topology/recovery.rs @@ -0,0 +1,34 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Discovery after a cached endpoint has failed or rejected leadership. + +use super::TopologyCache; +use crate::error::Result; +use std::time::Instant; + +impl TopologyCache { + pub(in crate::client) async fn refresh_after_failure( + &self, + store_id: u64, + group_id: u64, + failed: &str, + ) -> Result<()> { + let mut last = self.refresh_gate.lock().await; + if self + .leader(store_id, group_id) + .is_some_and(|leader| leader != failed) + || last.elapsed() < self.min_refresh_interval + { + return Ok(()); + } + let generation = self.snapshot.load().generation; + // Reachability of a management seed does not establish freshness of + // its leader view. Keep looking if it repeats the rejected endpoint. + let result = self + .fetch_group_and_merge(generation, Some((store_id, group_id)), Some(failed)) + .await; + *last = Instant::now(); + result.map(|_| ()) + } +} diff --git a/lib/crowdb-kv-client/src/service/registry/diskio.rs b/lib/crowdb-kv-client/src/service/registry/diskio.rs index 7c59fd83b..bf44cd997 100644 --- a/lib/crowdb-kv-client/src/service/registry/diskio.rs +++ b/lib/crowdb-kv-client/src/service/registry/diskio.rs @@ -96,13 +96,30 @@ impl ServiceRegistryClient { { GetOutcome::Found { value, revision } => (value, revision), GetOutcome::NotFound => { + if current_scope.is_some() { + return Err(Error::Server( + "DiskIO configured scope requires a registered Rack".into(), + )); + } return self .kv .put_cas(0, 0, path.as_bytes(), payload, revision) .await - .map(|_| ()) + .map(|_| ()); } }; + // The Rack fence must precede the instance observation it protects. + // A competing first scope may have committed after our first read + // but before this Rack read; do not overwrite that accepted scope. + if self + .instance_record(path) + .await? + .as_ref() + .map_or(0, |(_, revision)| *revision) + != revision + { + continue; + } let operations = [ BatchOp::Put { key: path.as_bytes().to_vec().into(), diff --git a/lib/crowdb-kv-client/src/transport/rpc_transport.rs b/lib/crowdb-kv-client/src/transport/rpc_transport.rs index 9ddf16df8..b76d03732 100644 --- a/lib/crowdb-kv-client/src/transport/rpc_transport.rs +++ b/lib/crowdb-kv-client/src/transport/rpc_transport.rs @@ -16,7 +16,11 @@ //! window; `CrowdbKvClient` selects the transport via //! `with_rpc_transport`. +mod connections; mod owned; +mod point_reads; + +use connections::normalize_endpoint; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -59,6 +63,7 @@ pub struct KvRpcTransport { server: Arc, rpc: Arc, connections: ConnectionPoolIndex, + point_connections: Option, next_req_id: AtomicU64, } @@ -66,116 +71,16 @@ impl std::fmt::Debug for KvRpcTransport { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("KvRpcTransport") .field("next_req_id", &self.next_req_id.load(Ordering::Relaxed)) + .field("isolated_point_reads", &self.point_connections.is_some()) .finish_non_exhaustive() } } impl KvRpcTransport { - /// Create a new crowdb-rpc transport with `pool_size` connections - /// per endpoint, using the given RPC tunables and `workers` I/O - /// worker threads. The `RpcServer` is the client-side transport — - /// it does not listen but is used to establish connections to - /// remote endpoints. - #[must_use] - pub fn with_pool_size( - pool_size: usize, - enable_nagle: bool, - quickack: bool, - event_write: bool, - send_queue_capacity: u32, - workers: u32, - ) -> Self { - let server = Arc::new(RpcServer::with_engines(None, 1, workers)); - server.set_tcp_nodelay(!enable_nagle); - server.set_quickack(quickack); - server.set_event_write(event_write); - server.set_send_queue_capacity(send_queue_capacity); - server.start(); - server.register_conn_count_gauge("rpc.client.connections"); - let rpc = Arc::new(RpcClient::new()); - rpc.set_completion_pool_size(1024); - rpc.start_reaper(5_000_000_000, 500_000_000); - Self { - server, - rpc, - connections: ConnectionPoolIndex::new(pool_size, None), - next_req_id: AtomicU64::new(1), - } - } - - /// Create a new crowdb-rpc transport with a single connection per - /// endpoint and default tunables (backward-compatible). - #[must_use] - pub fn new() -> Self { - Self::with_pool_size(1, false, false, false, 4096, 2) - } - fn next_id(&self) -> u64 { self.next_req_id.fetch_add(1, Ordering::Relaxed) } - /// Clear all cached connections. Called after a server restart to - /// force reconnection on the next request. - pub fn clear_connections(&self) { - self.connections.clear(); - } - - /// Convert an `RpcError` to `Error`, dropping cached connections - /// for `endpoint` on connection-level errors (closed/reset) and - /// `Timeout`. `Timeout` can indicate a dead connection whose TCP - /// FIN has not yet been processed by the epoll worker (e.g. the - /// remote process was killed and restarted on the same port) — the - /// reaper fails the pending request but leaves the stale fd in the - /// pool, so retrying on the same connection would time out again. - /// Dropping forces `conn_for` to establish a fresh connection. - /// `SendQueueFull` is truly transient (the connection is alive but - /// the send queue is full) so the pool is preserved. Preserves the - /// endpoint in the error message. - fn map_rpc_err(&self, e: RpcError, endpoint: &str, generation: u64) -> Error { - if matches!( - e, - RpcError::ConnectionClosed | RpcError::ConnectionError | RpcError::Timeout - ) { - self.connections - .invalidate(&normalize_endpoint(endpoint), generation); - } - Error::Transport { - endpoint: endpoint.to_string(), - status: format!("rpc error: {e:?}"), - } - } - - /// Get or create a `Connection` for the given endpoint, round- - /// robining across the pool. The crowdb-rpc server listens on the - /// same port as the crowdb-rpc endpoint (no port derivation). - fn conn_for(&self, rpc_endpoint: &str) -> Result { - let normalized = normalize_endpoint(rpc_endpoint); - let (host, port) = parse_endpoint(&normalized).map_err(|e| Error::InvalidEndpoint { - endpoint: rpc_endpoint.to_string(), - reason: e, - })?; - self.connections - .get_or_try_install(&normalized, || { - let conn = self.server.connect(&host, port).map_err(|e| Error::Transport { - endpoint: rpc_endpoint.to_string(), - status: format!("rpc connect to {host}:{port}: {e:?}"), - })?; - self.rpc.attach(&conn); - Ok(conn) - }) - .map_err(|error| match error { - ConnectionPoolError::Closed => Error::Transport { - endpoint: rpc_endpoint.to_string(), - status: "connection closed during acquisition".into(), - }, - ConnectionPoolError::Connect(error) => error, - ConnectionPoolError::Capacity { max_endpoints } => Error::Transport { - endpoint: rpc_endpoint.to_string(), - status: format!("endpoint connection limit {max_endpoints} reached"), - }, - }) - } - /// Send a `Put` request via crowdb-rpc. Returns the /// `KvResponse` so the caller's retry logic is unchanged. #[allow(clippy::too_many_arguments)] @@ -296,78 +201,6 @@ impl KvRpcTransport { parse_kv_response(control.bytes()) } - /// Send a `Get` request via crowdb-rpc. - #[allow(clippy::too_many_arguments)] - pub async fn send_get( - &self, - rpc_endpoint: &str, - key: &[u8], - request_id: u64, - request_create_ms: u64, - group_id: u64, - read_mode: ReadMode, - min_slot: u64, - ) -> Result { - self.send_get_with_forwarded( - rpc_endpoint, - key, - request_id, - request_create_ms, - group_id, - read_mode, - min_slot, - false, - ) - .await - } - - /// Send a `Get` request with an explicit forwarded loop-guard flag. - #[allow(clippy::too_many_arguments)] - pub async fn send_get_with_forwarded( - &self, - rpc_endpoint: &str, - key: &[u8], - request_id: u64, - request_create_ms: u64, - group_id: u64, - read_mode: ReadMode, - min_slot: u64, - forwarded: bool, - ) -> Result { - let req_id = self.next_id(); - let conn = self.conn_for(rpc_endpoint)?; - let mut builder = FlatBufferBuilder::new(); - let fb_key = builder.create_vector(key); - let args = FBKvGetRequestArgs { - id: req_id, - rpc_create_nano: 0, - version: 1, - key: Some(fb_key), - request_id, - request_create_ms, - group_id, - read_mode: read_mode_to_fb(read_mode), - min_slot, - forwarded, - }; - let req = FBKvGetRequest::create(&mut builder, &args); - builder.finish(req, None); - let control = Buffer::from_bytes(builder.finished_data()); - let msg_type = FBMsgType::EKvGetRequest.0 as u16; - let fut = self - .rpc - .call(&self.server, &conn, req_id, control, None, msg_type) - .map_err(|e| self.map_rpc_err(e, rpc_endpoint, conn.generation()))?; - let resp = fut - .await - .map_err(|e| self.map_rpc_err(e, rpc_endpoint, conn.generation()))?; - let ctrl = resp.control.ok_or_else(|| Error::Transport { - endpoint: rpc_endpoint.to_string(), - status: "get response missing control buffer".into(), - })?; - parse_kv_response(ctrl.bytes()) - } - /// Send a `Delete` request via crowdb-rpc. #[allow(clippy::too_many_arguments)] pub async fn send_delete( @@ -1161,26 +994,3 @@ fn fb_ret_code_to_kv_error_code(code: FBKvClientRetCode) -> i32 { _ => KvErrorCode::KvErrorInternal as i32, } } - -fn normalize_endpoint(endpoint: &str) -> String { - let with_scheme = if endpoint.starts_with("http://") || endpoint.starts_with("https://") { - endpoint.to_string() - } else { - format!("http://{endpoint}") - }; - with_scheme.replacen("://0.0.0.0:", "://127.0.0.1:", 1) -} - -fn parse_endpoint(endpoint: &str) -> std::result::Result<(String, i32), String> { - let without_scheme = endpoint - .strip_prefix("http://") - .or_else(|| endpoint.strip_prefix("https://")) - .unwrap_or(endpoint); - let (host, port_str) = without_scheme - .rsplit_once(':') - .ok_or_else(|| format!("invalid endpoint: {endpoint}"))?; - let port: i32 = port_str - .parse() - .map_err(|_| format!("invalid port in endpoint: {endpoint}"))?; - Ok((host.to_string(), port)) -} diff --git a/lib/crowdb-kv-client/src/transport/rpc_transport/connections.rs b/lib/crowdb-kv-client/src/transport/rpc_transport/connections.rs new file mode 100644 index 000000000..4031b992f --- /dev/null +++ b/lib/crowdb-kv-client/src/transport/rpc_transport/connections.rs @@ -0,0 +1,152 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Shared transport owners and independently invalidated point/ordinary pools. + +use super::{ + Arc, AtomicU64, ConnectionPoolError, ConnectionPoolIndex, Error, KvRpcTransport, Result, RpcClient, + RpcError, RpcServer, SelectedConnection, +}; + +impl KvRpcTransport { + /// Create a new crowdb-rpc transport with `pool_size` connections + /// per endpoint, using the given RPC tunables and `workers` I/O + /// worker threads. The `RpcServer` is the client-side transport — + /// it does not listen but is used to establish connections to + /// remote endpoints. + #[must_use] + pub fn with_pool_size( + pool_size: usize, + enable_nagle: bool, + quickack: bool, + event_write: bool, + send_queue_capacity: u32, + workers: u32, + ) -> Self { + let server = Arc::new(RpcServer::with_engines(None, 1, workers)); + server.set_tcp_nodelay(!enable_nagle); + server.set_quickack(quickack); + server.set_event_write(event_write); + server.set_send_queue_capacity(send_queue_capacity); + server.start(); + server.register_conn_count_gauge("rpc.client.connections"); + let rpc = Arc::new(RpcClient::new()); + rpc.set_completion_pool_size(1024); + rpc.start_reaper(5_000_000_000, 500_000_000); + Self { + server, + rpc, + connections: ConnectionPoolIndex::new(pool_size, None), + point_connections: None, + next_req_id: AtomicU64::new(1), + } + } + + /// Create a new crowdb-rpc transport with a single connection per + /// endpoint and default tunables (backward-compatible). + #[must_use] + pub fn new() -> Self { + Self::with_pool_size(1, false, false, false, 4096, 2) + } + + /// Reserve one connection per endpoint for Get replies, independently of + /// scan and mutation frames. Uses the same I/O workers, correlation pool, + /// deadlines and tunables. Ordinary transports keep their existing pool. + #[must_use] + pub fn with_isolated_point_reads(mut self) -> Self { + self.point_connections = Some(ConnectionPoolIndex::new(1, None)); + self + } + + /// Clear both pools after an explicit reconnect request. + pub fn clear_connections(&self) { + self.connections.clear(); + if let Some(points) = &self.point_connections { + points.clear(); + } + } + + fn point_index(&self) -> &ConnectionPoolIndex { + self.point_connections.as_ref().unwrap_or(&self.connections) + } + + pub(super) fn map_rpc_err(&self, error: RpcError, endpoint: &str, generation: u64) -> Error { + map_error(error, endpoint, generation, &self.connections) + } + + pub(super) fn map_get_rpc_err(&self, error: RpcError, endpoint: &str, generation: u64) -> Error { + map_error(error, endpoint, generation, self.point_index()) + } + + pub(super) fn conn_for(&self, endpoint: &str) -> Result { + self.conn_for_in(endpoint, &self.connections) + } + + pub(super) fn conn_for_point(&self, endpoint: &str) -> Result { + self.conn_for_in(endpoint, self.point_index()) + } + + fn conn_for_in(&self, rpc_endpoint: &str, index: &ConnectionPoolIndex) -> Result { + let normalized = normalize_endpoint(rpc_endpoint); + let (host, port) = parse_endpoint(&normalized).map_err(|e| Error::InvalidEndpoint { + endpoint: rpc_endpoint.to_string(), + reason: e, + })?; + index + .get_or_try_install(&normalized, || { + let conn = self.server.connect(&host, port).map_err(|e| Error::Transport { + endpoint: rpc_endpoint.to_string(), + status: format!("rpc connect to {host}:{port}: {e:?}"), + })?; + self.rpc.attach(&conn); + Ok(conn) + }) + .map_err(|error| match error { + ConnectionPoolError::Closed => Error::Transport { + endpoint: rpc_endpoint.to_string(), + status: "connection closed during acquisition".into(), + }, + ConnectionPoolError::Connect(error) => error, + ConnectionPoolError::Capacity { max_endpoints } => Error::Transport { + endpoint: rpc_endpoint.to_string(), + status: format!("endpoint connection limit {max_endpoints} reached"), + }, + }) + } +} + +fn map_error(error: RpcError, endpoint: &str, generation: u64, index: &ConnectionPoolIndex) -> Error { + if matches!( + error, + RpcError::ConnectionClosed | RpcError::ConnectionError | RpcError::Timeout + ) { + index.invalidate(&normalize_endpoint(endpoint), generation); + } + Error::Transport { + endpoint: endpoint.to_string(), + status: format!("rpc error: {error:?}"), + } +} + +pub(super) fn normalize_endpoint(endpoint: &str) -> String { + let with_scheme = if endpoint.starts_with("http://") || endpoint.starts_with("https://") { + endpoint.to_string() + } else { + format!("http://{endpoint}") + }; + with_scheme.replacen("://0.0.0.0:", "://127.0.0.1:", 1) +} + +fn parse_endpoint(endpoint: &str) -> std::result::Result<(String, i32), String> { + let without_scheme = endpoint + .strip_prefix("http://") + .or_else(|| endpoint.strip_prefix("https://")) + .unwrap_or(endpoint); + let (host, port_str) = without_scheme + .rsplit_once(':') + .ok_or_else(|| format!("invalid endpoint: {endpoint}"))?; + let port: i32 = port_str + .parse() + .map_err(|_| format!("invalid port in endpoint: {endpoint}"))?; + Ok((host.to_string(), port)) +} diff --git a/lib/crowdb-kv-client/src/transport/rpc_transport/point_reads.rs b/lib/crowdb-kv-client/src/transport/rpc_transport/point_reads.rs new file mode 100644 index 000000000..6174dc7a6 --- /dev/null +++ b/lib/crowdb-kv-client/src/transport/rpc_transport/point_reads.rs @@ -0,0 +1,83 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Get requests retain their wire semantics on an optional isolated connection. + +use super::{ + parse_kv_response, read_mode_to_fb, Buffer, Error, FBKvGetRequest, FBKvGetRequestArgs, FBMsgType, + FlatBufferBuilder, KvResponse, KvRpcTransport, ReadMode, Result, +}; + +impl KvRpcTransport { + /// Send a `Get` request via crowdb-rpc. + #[allow(clippy::too_many_arguments)] + pub async fn send_get( + &self, + rpc_endpoint: &str, + key: &[u8], + request_id: u64, + request_create_ms: u64, + group_id: u64, + read_mode: ReadMode, + min_slot: u64, + ) -> Result { + self.send_get_with_forwarded( + rpc_endpoint, + key, + request_id, + request_create_ms, + group_id, + read_mode, + min_slot, + false, + ) + .await + } + + /// Send a `Get` request with an explicit forwarded loop-guard flag. + #[allow(clippy::too_many_arguments)] + pub async fn send_get_with_forwarded( + &self, + rpc_endpoint: &str, + key: &[u8], + request_id: u64, + request_create_ms: u64, + group_id: u64, + read_mode: ReadMode, + min_slot: u64, + forwarded: bool, + ) -> Result { + let req_id = self.next_id(); + let conn = self.conn_for_point(rpc_endpoint)?; + let mut builder = FlatBufferBuilder::new(); + let fb_key = builder.create_vector(key); + let args = FBKvGetRequestArgs { + id: req_id, + rpc_create_nano: 0, + version: 1, + key: Some(fb_key), + request_id, + request_create_ms, + group_id, + read_mode: read_mode_to_fb(read_mode), + min_slot, + forwarded, + }; + let req = FBKvGetRequest::create(&mut builder, &args); + builder.finish(req, None); + let control = Buffer::from_bytes(builder.finished_data()); + let msg_type = FBMsgType::EKvGetRequest.0 as u16; + let fut = self + .rpc + .call(&self.server, &conn, req_id, control, None, msg_type) + .map_err(|e| self.map_get_rpc_err(e, rpc_endpoint, conn.generation()))?; + let resp = fut + .await + .map_err(|e| self.map_get_rpc_err(e, rpc_endpoint, conn.generation()))?; + let ctrl = resp.control.ok_or_else(|| Error::Transport { + endpoint: rpc_endpoint.to_string(), + status: "get response missing control buffer".into(), + })?; + parse_kv_response(ctrl.bytes()) + } +} diff --git a/lib/crowdb-kv-client/tests/common/point_read_peer.rs b/lib/crowdb-kv-client/tests/common/point_read_peer.rs new file mode 100644 index 000000000..3939bbe3c --- /dev/null +++ b/lib/crowdb-kv-client/tests/common/point_read_peer.rs @@ -0,0 +1,219 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::{ + fb::FBMsgType, + kv_client_fb::{ + FBKvClientRetCode, FBKvGetRequest, FBKvResponse, FBKvResponseArgs, FBKvScanItem, FBKvScanItemArgs, + FBKvScanRequest, FBKvScanResponse, FBKvScanResponseArgs, + }, +}; +use flatbuffers::FlatBufferBuilder; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt, ReadHalf, WriteHalf}, + net::{TcpListener, TcpStream}, + sync::{mpsc, Notify}, + task::{JoinHandle, JoinSet}, +}; + +#[derive(Clone, Copy)] +pub enum TestPointReply { + Ready, + Rejected, + Held, + ClosedOnce, +} + +#[derive(Default)] +pub struct TestPointState { + pub bulk_started: Notify, + pub release_bulk: Notify, + pub accepted: AtomicUsize, + pub scans: AtomicUsize, + pub gets: AtomicUsize, +} + +pub struct TestPointReadPeer { + pub endpoint: String, + pub state: Arc, + worker: JoinHandle<()>, +} + +struct Reply { + bytes: Vec, + hold: bool, +} + +impl TestPointReadPeer { + pub async fn start(reply: TestPointReply) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = listener.local_addr().unwrap().to_string(); + let state = Arc::new(TestPointState::default()); + let worker = tokio::spawn({ + let state = state.clone(); + async move { + let mut connections = JoinSet::new(); + loop { + tokio::select! { + value=listener.accept()=>{ + let(stream,_)=value.unwrap(); + state.accepted.fetch_add(1,Ordering::Release); + connections.spawn(serve(stream,state.clone(),reply)); + }, + result=connections.join_next(),if !connections.is_empty()=>{result.unwrap().unwrap();} + } + } + } + }); + Self { + endpoint, + state, + worker, + } + } +} + +impl Drop for TestPointReadPeer { + fn drop(&mut self) { + self.worker.abort(); + } +} + +async fn serve(stream: TcpStream, state: Arc, reply: TestPointReply) { + let (reader, writer) = tokio::io::split(stream); + let (tx, rx) = mpsc::channel(8); + tokio::join!( + read_requests(reader, tx, state.clone(), reply), + write_replies(writer, rx, state) + ); +} + +async fn read_requests( + mut reader: ReadHalf, + tx: mpsc::Sender, + state: Arc, + reply: TestPointReply, +) { + loop { + let mut header = [0; 14]; + if reader.read_exact(&mut header).await.is_err() { + return; + } + assert_eq!(u16::from_le_bytes(header[..2].try_into().unwrap()), 0xca70); + let kind = u16::from_le_bytes(header[2..4].try_into().unwrap()); + let length = u32::from_le_bytes(header[4..8].try_into().unwrap()) as usize; + assert!(length <= 4096); + assert_eq!(&header[8..12], &[0; 4]); + let mut control = vec![0; length]; + reader.read_exact(&mut control).await.unwrap(); + let response = if kind == 0xfffe { + assert_eq!(control.len(), 160); + let mut capabilities = vec![0; 160]; + capabilities[..4].copy_from_slice(&[0x44, 0x52, 4, 0]); + capabilities[4..12].copy_from_slice(&1_u64.to_le_bytes()); + frame(kind, &capabilities, false) + } else if kind == FBMsgType::EKvScanRequest.0 as u16 { + let id = flatbuffers::root::(&control).unwrap().id(); + let hold = state.scans.fetch_add(1, Ordering::AcqRel) == 0; + scan_reply(id, hold) + } else { + assert_eq!(kind, FBMsgType::EKvGetRequest.0 as u16); + let id = flatbuffers::root::(&control).unwrap().id(); + let index = state.gets.fetch_add(1, Ordering::AcqRel); + if matches!(reply, TestPointReply::ClosedOnce) && index == 0 { + return; + } + if matches!(reply, TestPointReply::Held) { + continue; + } + point_reply(id, reply) + }; + if tx.send(response).await.is_err() { + return; + } + } +} + +async fn write_replies( + mut writer: WriteHalf, + mut rx: mpsc::Receiver, + state: Arc, +) { + while let Some(reply) = rx.recv().await { + if reply.hold { + if writer.write_all(&reply.bytes[..15]).await.is_err() { + return; + } + state.bulk_started.notify_one(); + state.release_bulk.notified().await; + if writer.write_all(&reply.bytes[15..]).await.is_err() { + return; + } + } else if writer.write_all(&reply.bytes).await.is_err() { + return; + } + } +} + +fn frame(kind: u16, control: &[u8], hold: bool) -> Reply { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&0xca70_u16.to_le_bytes()); + bytes.extend_from_slice(&kind.to_le_bytes()); + bytes.extend_from_slice(&u32::try_from(control.len()).unwrap().to_le_bytes()); + bytes.extend_from_slice(&0_u32.to_le_bytes()); + bytes.extend_from_slice(&[14, 0]); + bytes.extend_from_slice(control); + Reply { bytes, hold } +} + +fn scan_reply(id: u64, hold: bool) -> Reply { + let mut builder = FlatBufferBuilder::new(); + let key = builder.create_vector(b"/bulk/value"); + let value = builder.create_vector(&vec![7_u8; 65536]); + let item = FBKvScanItem::create( + &mut builder, + &FBKvScanItemArgs { + key: Some(key), + value: Some(value), + ..Default::default() + }, + ); + let items = builder.create_vector(&[item]); + let response = FBKvScanResponse::create( + &mut builder, + &FBKvScanResponseArgs { + id, + version: 1, + ok: true, + items: Some(items), + ..Default::default() + }, + ); + builder.finish(response, None); + frame(FBMsgType::EKvScanResponse.0 as u16, builder.finished_data(), hold) +} + +fn point_reply(id: u64, reply: TestPointReply) -> Reply { + let mut builder = FlatBufferBuilder::new(); + let response = FBKvResponse::create( + &mut builder, + &FBKvResponseArgs { + id, + version: 1, + ok: matches!(reply, TestPointReply::Ready | TestPointReply::ClosedOnce), + not_found: matches!(reply, TestPointReply::Ready | TestPointReply::ClosedOnce), + ret_code: if matches!(reply, TestPointReply::Rejected) { + FBKvClientRetCode::Unavailable + } else { + FBKvClientRetCode::Success + }, + ..Default::default() + }, + ); + builder.finish(response, None); + frame(FBMsgType::EKvResponse.0 as u16, builder.finished_data(), false) +} diff --git a/lib/crowdb-kv-client/tests/common/topology_recovery.rs b/lib/crowdb-kv-client/tests/common/topology_recovery.rs new file mode 100644 index 000000000..c15614182 --- /dev/null +++ b/lib/crowdb-kv-client/tests/common/topology_recovery.rs @@ -0,0 +1,87 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{routing::get, Json, Router}; +use crowdb_kv::cluster::{ + group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +pub struct TestDiscovery { + stores: Vec>, + servers: Vec>, + pub client: Arc, + pub fetches: Arc, +} + +impl TestDiscovery { + pub async fn start(transport_failure: bool, replacement: bool) -> Self { + let mut stores = Vec::new(); + for (id, role) in [(1, PxLocalReplicaRole::Leader), (2, PxLocalReplicaRole::Follower)] { + let store = Arc::new(PxKvStore::new(11, "127.0.0.1:0".parse().unwrap())); + store.add_group(PxGroup::new(1, PxLocalReplica::new(id, role))); + store.start().await.unwrap(); + stores.push(store); + } + let mut stale = stores[0].status(); + let failed = if transport_failure { + "127.0.0.1:1".to_owned() + } else { + stores[1].listen_addr().unwrap().to_string() + }; + stale.listen_addr = Some(failed.clone()); + let mut seeds = Vec::new(); + let mut servers = Vec::new(); + let fetches = Arc::new(AtomicUsize::new(0)); + for body in [ + stale.clone(), + if replacement { stores[0].status() } else { stale }, + ] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + seeds.push(format!("http://{}", listener.local_addr().unwrap())); + let requests = Arc::clone(&fetches); + servers.push(tokio::spawn(async move { + let app = Router::new().route( + "/topology", + get(move || { + let body = body.clone(); + requests.fetch_add(1, Ordering::Relaxed); + async move { Json(serde_json::json!({"stores": [body]})) } + }), + ); + axum::serve(listener, app).await.unwrap(); + })); + } + let mut config = ClientConfig::new(seeds); + config.retry.max_retries = 2; + config.retry.unknown_leader_wait = Duration::from_millis(10); + config.retry.backoff_base = Duration::from_secs(1); + let client = Arc::new(CrowdbKvClient::new(config)); + client.seed_leader(11, 1, failed); + Self { + stores, + servers, + client, + fetches, + } + } + + pub async fn stop(&self) { + for server in &self.servers { + server.abort(); + } + for store in &self.stores { + store.stop(); + } + for store in &self.stores { + store.join().await; + } + } +} diff --git a/lib/crowdb-kv-client/tests/point_read_isolation_test.rs b/lib/crowdb-kv-client/tests/point_read_isolation_test.rs new file mode 100644 index 000000000..4dcf9f5c8 --- /dev/null +++ b/lib/crowdb-kv-client/tests/point_read_isolation_test.rs @@ -0,0 +1,170 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_kv::rpc::{KvErrorCode, KvResponse}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient, Error, GetOutcome, KvRpcTransport, ReadMode, Result}; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; +use tokio::task::JoinHandle; + +#[path = "common/point_read_peer.rs"] +mod point_read_peer; +use point_read_peer::{TestPointReadPeer, TestPointReply}; + +const BUDGET: Duration = Duration::from_millis(750); + +async fn scan_value(transport: &KvRpcTransport, endpoint: &str) { + let page = transport + .send_scan( + endpoint, + b"/bulk/", + b"", + b"", + 1, + 1, + 0, + 1, + ReadMode::Linearizable, + 0, + false, + false, + 0, + false, + 0, + ) + .await + .unwrap(); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].value.len(), 65536); +} + +fn start_scan(transport: Arc, endpoint: String) -> JoinHandle<()> { + tokio::spawn(async move { scan_value(&transport, &endpoint).await }) +} + +async fn finish_scan(peer: &TestPointReadPeer, scan: JoinHandle<()>) { + peer.state.release_bulk.notify_one(); + tokio::time::timeout(BUDGET, scan).await.unwrap().unwrap(); +} + +async fn point(transport: &KvRpcTransport, endpoint: &str) -> Result { + transport + .send_get(endpoint, b"/chunk/point", 2, 0, 1, ReadMode::Linearizable, 0) + .await +} + +#[tokio::test] +async fn point_read_finishes_while_a_scan_frame_is_incomplete() { + let peer = TestPointReadPeer::start(TestPointReply::Ready).await; + let client = Arc::new(CrowdbKvClient::new_with_isolated_point_reads(ClientConfig::new( + vec![], + ))); + client.seed_leader(0, 1, peer.endpoint.clone()); + let scan = tokio::spawn({ + let client = client.clone(); + async move { + let page = client + .scan( + 0, + 1, + b"/bulk/", + b"", + b"", + 1, + ReadMode::Linearizable, + None, + false, + None, + ) + .await + .unwrap(); + assert_eq!(page.items[0].1.len(), 65536); + } + }); + tokio::time::timeout(BUDGET, peer.state.bulk_started.notified()) + .await + .unwrap(); + let get = tokio::time::timeout( + BUDGET, + client.get(0, 1, b"/chunk/point", ReadMode::Linearizable, None), + ) + .await; + let received = peer.state.gets.load(Ordering::Acquire); + finish_scan(&peer, scan).await; + assert_eq!( + received, 1, + "the peer received the point read while its scan response was held" + ); + assert!(matches!(get, Ok(Ok(GetOutcome::NotFound))), + "a real point read must finish within its original deadline despite an incomplete scan frame: {get:?}"); + assert_eq!(peer.state.accepted.load(Ordering::Acquire), 2); +} + +#[tokio::test] +async fn ordinary_transport_keeps_the_shared_pool_without_opt_in() { + let peer = TestPointReadPeer::start(TestPointReply::Ready).await; + let transport = Arc::new(KvRpcTransport::new()); + let scan = start_scan(transport.clone(), peer.endpoint.clone()); + tokio::time::timeout(BUDGET, peer.state.bulk_started.notified()) + .await + .unwrap(); + let get = tokio::time::timeout(BUDGET, point(&transport, &peer.endpoint)).await; + finish_scan(&peer, scan).await; + assert!( + get.is_err(), + "the control must retain shared-frame response ordering: {get:?}" + ); + assert_eq!(peer.state.gets.load(Ordering::Acquire), 1); + assert_eq!(peer.state.accepted.load(Ordering::Acquire), 1); +} + +#[tokio::test] +async fn isolated_point_reads_preserve_rejection() { + let peer = TestPointReadPeer::start(TestPointReply::Rejected).await; + let transport = KvRpcTransport::new().with_isolated_point_reads(); + let result = tokio::time::timeout(BUDGET, point(&transport, &peer.endpoint)) + .await + .unwrap() + .unwrap(); + assert!(!result.ok); + assert!(!result.not_found); + assert_eq!(result.error_code, KvErrorCode::KvErrorUnavailable as i32); + assert_eq!(peer.state.gets.load(Ordering::Acquire), 1); +} + +#[tokio::test] +async fn isolated_point_reads_keep_the_original_deadline() { + let peer = TestPointReadPeer::start(TestPointReply::Held).await; + let transport = KvRpcTransport::new().with_isolated_point_reads(); + assert!(tokio::time::timeout(BUDGET, point(&transport, &peer.endpoint)) + .await + .is_err()); + assert_eq!(peer.state.gets.load(Ordering::Acquire), 1); +} + +#[tokio::test] +async fn failed_point_generation_does_not_evict_a_healthy_scan_pool() { + let peer = TestPointReadPeer::start(TestPointReply::ClosedOnce).await; + let transport = Arc::new(KvRpcTransport::new().with_isolated_point_reads()); + let scan = start_scan(transport.clone(), peer.endpoint.clone()); + tokio::time::timeout(BUDGET, peer.state.bulk_started.notified()) + .await + .unwrap(); + let lost = tokio::time::timeout(BUDGET, point(&transport, &peer.endpoint)).await; + let recovered = tokio::time::timeout(BUDGET, point(&transport, &peer.endpoint)).await; + finish_scan(&peer, scan).await; + assert!(matches!(lost, Ok(Err(Error::Transport { .. }))), "{lost:?}"); + assert!(recovered.unwrap().unwrap().not_found); + scan_value(&transport, &peer.endpoint).await; + assert_eq!(peer.state.accepted.load(Ordering::Acquire), 3); + transport.clear_connections(); + assert!(point(&transport, &peer.endpoint).await.unwrap().not_found); + scan_value(&transport, &peer.endpoint).await; + assert_eq!( + peer.state.accepted.load(Ordering::Acquire), + 5, + "explicit clear must replace both pools; failed point reads must replace only their own generation" + ); +} diff --git a/lib/crowdb-kv-client/tests/storage_exclusion_test.rs b/lib/crowdb-kv-client/tests/storage_exclusion_test.rs index 86c667c9a..08568c263 100644 --- a/lib/crowdb-kv-client/tests/storage_exclusion_test.rs +++ b/lib/crowdb-kv-client/tests/storage_exclusion_test.rs @@ -202,6 +202,48 @@ async fn immutable_configured_scope_covers_disks_before_usage_advertisements_cat ); } +#[tokio::test] +async fn configured_scope_waits_for_its_rack_membership_fence() { + use crowdb_protocol::common::{DiskioHardwareScope, NodeValue, RackValue}; + let fixture = TestStorage::start().await; + let hardware = &fixture.hardware; + let svc = ServiceRegistryClient::from_shared(hardware.shared_kv()); + let scope = DiskioHardwareScope { + version: 1, + rack_id: 99, + node_id: 99, + disk_group_id: 0, + }; + assert!( + svc.heartbeat_diskio_scoped(99, "pending:7200", &scope, &[], &[]) + .await + .is_err(), + "a first scope cannot publish before its Rack admission fence exists" + ); + let pinned = svc.read_instance("diskio", 99).await.unwrap().unwrap(); + assert!(pinned.extra.unwrap().diskdb.unwrap().diskio_scope.is_none()); + hardware + .add_rack( + 99, + &RackValue { + node_ids: vec![99], + ..Default::default() + }, + ) + .await + .unwrap(); + hardware.add_node(99, 99, &NodeValue::default()).await.unwrap(); + svc.heartbeat_diskio_scoped(99, "ready:7200", &scope, &[], &[]) + .await + .unwrap(); + let mut other = scope.clone(); + other.disk_group_id = 3; + assert!(svc + .heartbeat_diskio_scoped(99, "other:7200", &other, &[], &[]) + .await + .is_err()); +} + #[tokio::test] async fn simultaneous_first_scope_selection_pins_one_configuration() { use crowdb_protocol::common::DiskioHardwareScope; @@ -215,9 +257,24 @@ async fn simultaneous_first_scope_selection_pins_one_configuration() { }; let mut second = first.clone(); second.disk_group_id = 3; - let (a, b) = tokio::join!( - svc.heartbeat_diskio_scoped(7, "a:7200", &first, &[], &[]), - svc.heartbeat_diskio_scoped(7, "b:7200", &second, &[], &[]) - ); - assert_ne!(a.is_ok(), b.is_ok(), "a={a:?}, b={b:?}"); + for instance_id in 7..39 { + let (a, b) = tokio::join!( + svc.heartbeat_diskio_scoped(instance_id, "a:7200", &first, &[], &[]), + svc.heartbeat_diskio_scoped(instance_id, "b:7200", &second, &[], &[]) + ); + assert_ne!(a.is_ok(), b.is_ok(), "instance={instance_id}, a={a:?}, b={b:?}"); + let (winner, loser) = if a.is_ok() { + (&first, &second) + } else { + (&second, &first) + }; + assert!(svc + .heartbeat_diskio_scoped(instance_id, "winner:7200", winner, &[], &[]) + .await + .is_ok()); + assert!(svc + .heartbeat_diskio_scoped(instance_id, "loser:7200", loser, &[], &[]) + .await + .is_err()); + } } diff --git a/lib/crowdb-kv-client/tests/topology_retry_discovery_test.rs b/lib/crowdb-kv-client/tests/topology_retry_discovery_test.rs new file mode 100644 index 000000000..332da0162 --- /dev/null +++ b/lib/crowdb-kv-client/tests/topology_retry_discovery_test.rs @@ -0,0 +1,88 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/topology_recovery.rs"] +mod topology_recovery; + +use crowdb_kv_client::{Error, GetOutcome, ReadMode}; +use std::time::Duration; +use topology_recovery::TestDiscovery; + +#[tokio::test] +async fn a_reachable_stale_seed_cannot_hide_a_replacement_after_transport_failure() { + let test = TestDiscovery::start(true, true).await; + let result = tokio::time::timeout( + Duration::from_millis(500), + test.client.put(11, 1, b"recovered", b"retained", None), + ) + .await; + let read = if matches!(result, Ok(Ok(_))) { + Some( + test.client + .get(11, 1, b"recovered", ReadMode::Linearizable, None) + .await, + ) + } else { + None + }; + test.stop().await; + assert!( + matches!(result, Ok(Ok(_))), + "stale first seed retained failed route/backoff: {result:?}" + ); + assert!(matches!(read, Some(Ok(GetOutcome::Found { value, .. })) if value.as_ref() == b"retained")); +} + +#[tokio::test] +async fn a_reachable_stale_seed_cannot_hide_a_replacement_after_no_hint_rejection() { + let test = TestDiscovery::start(false, true).await; + let result = tokio::time::timeout( + Duration::from_millis(500), + test.client.put_cas(11, 1, b"recovered", b"retained", 0), + ) + .await; + test.stop().await; + assert!( + matches!(result, Ok(Ok(_))), + "stale first seed retained rejected route: {result:?}" + ); +} + +#[tokio::test] +async fn rejecting_stale_routes_does_not_remove_the_retry_budget() { + let test = TestDiscovery::start(false, false).await; + let result = test.client.put_cas(11, 1, b"bounded", b"value", 0).await; + test.stop().await; + assert!( + matches!(result, Err(Error::RetriesExhausted { attempts: 3, .. })), + "{result:?}" + ); +} + +#[tokio::test] +async fn concurrent_route_failures_share_the_replacement_discovery() { + let test = TestDiscovery::start(true, true).await; + let result = tokio::time::timeout(Duration::from_millis(500), async { + let mut requests = tokio::task::JoinSet::new(); + for id in 0u8..32 { + let client = std::sync::Arc::clone(&test.client); + requests.spawn(async move { client.put(11, 1, &[id], b"retained", None).await }); + } + let mut results = Vec::new(); + while let Some(result) = requests.join_next().await { + results.push(result.unwrap()); + } + results + }) + .await; + let fetches = test.fetches.load(std::sync::atomic::Ordering::Relaxed); + test.stop().await; + assert!( + matches!(result, Ok(ref results) if results.iter().all(Result::is_ok)), + "{result:?}" + ); + assert_eq!( + fetches, 2, + "the stale and replacement seeds should each be fetched once" + ); +} diff --git a/lib/crowdb-kv/Cargo.toml b/lib/crowdb-kv/Cargo.toml index c76751882..d6fa5a6ec 100644 --- a/lib/crowdb-kv/Cargo.toml +++ b/lib/crowdb-kv/Cargo.toml @@ -21,7 +21,7 @@ workspace = true test-util = [] [dependencies] -tokio = { workspace = true, features = ["rt", "macros", "time", "net", "fs", "sync", "io-util"] } +tokio = { workspace = true, features = ["rt", "rt-multi-thread", "macros", "time", "net", "fs", "sync", "io-util"] } tokio-stream = "0.1" tokio-util = { version = "0.7", features = ["rt"] } bytes = "1" diff --git a/lib/crowdb-kv/src/cluster/group_election_leader.rs b/lib/crowdb-kv/src/cluster/group_election_leader.rs index ca1d6586e..f394333ec 100644 --- a/lib/crowdb-kv/src/cluster/group_election_leader.rs +++ b/lib/crowdb-kv/src/cluster/group_election_leader.rs @@ -503,8 +503,13 @@ impl PxGroup { } // Admin step-down via StepDown RPC. () = replica.admin_step_down_signal.notified() => { - self.step_down(&tenure_cancel, leader_term, StepDownReason::Admin); - return; + // Accepted requests fence the role before publishing the wakeup. + // A permit retained from an earlier tenure cannot dismiss a + // replica that has since become leader again. + if replica.role() != PxLocalReplicaRole::Leader { + self.step_down(&tenure_cancel, leader_term, StepDownReason::Admin); + return; + } } _ = ticker.tick() => { match self.run_heartbeat_round(cfg, leader_term).await { diff --git a/lib/crowdb-kv/src/wal/block_backend.rs b/lib/crowdb-kv/src/wal/block_backend.rs index bb795b1d6..1ada1f040 100644 --- a/lib/crowdb-kv/src/wal/block_backend.rs +++ b/lib/crowdb-kv/src/wal/block_backend.rs @@ -13,6 +13,20 @@ use parking_lot::Mutex; use super::io_backend::OpenOptions; use super::pipeline_backend::WalBlockAlignment; +/// Hand the multi-thread runtime's worker duties back before blocking on storage. +/// Borrowed WAL buffers stay on the caller's stack without an extra payload copy. +/// Current-thread runtimes retain their synchronous behavior: they cannot hand off +/// a worker and Tokio rejects `block_in_place` there. +pub(super) fn blocking_io(operation: impl FnOnce() -> T) -> T { + let can_handoff = tokio::runtime::Handle::try_current() + .is_ok_and(|handle| handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread); + if can_handoff { + tokio::task::block_in_place(operation) + } else { + operation() + } +} + #[derive(Clone, Default)] pub struct BlockDeviceController { inner: Arc, @@ -103,6 +117,16 @@ pub struct BlockDevice { #[cfg_attr(not(target_os = "linux"), allow(dead_code))] use_direct_io: bool, rmw_count: Arc, + #[cfg(feature = "test-util")] + io_gate: Arc>, +} + +#[cfg(feature = "test-util")] +struct TestIoGate { + sync: bool, + armed: AtomicBool, + entered: Arc, + release: Arc, } impl BlockDevice { @@ -134,6 +158,40 @@ impl BlockDevice { alignment, use_direct_io, rmw_count: Arc::new(AtomicU64::new(0)), + #[cfg(feature = "test-util")] + io_gate: Arc::new(std::sync::OnceLock::new()), + } + } + + /// Hold the next physical write or sync before entering the real file operation. + /// + /// # Panics + /// Panics if a gate was already installed on this device. + #[cfg(feature = "test-util")] + pub fn set_io_gate_for_tests( + &self, + sync: bool, + entered: Arc, + release: Arc, + ) { + assert!(self + .io_gate + .set(TestIoGate { + sync, + armed: AtomicBool::new(true), + entered, + release + }) + .is_ok()); + } + + #[cfg(feature = "test-util")] + fn pause_io_for_tests(&self, sync: bool) { + if let Some(gate) = self.io_gate.get() { + if gate.sync == sync && gate.armed.swap(false, Ordering::AcqRel) { + gate.entered.wait(); + gate.release.wait(); + } } } @@ -159,6 +217,10 @@ impl BlockDevice { } pub(crate) fn open_segment(&self, segment_path: &Path, opts: &OpenOptions) -> io::Result { + blocking_io(|| self.open_segment_file(segment_path, opts)) + } + + fn open_segment_file(&self, segment_path: &Path, opts: &OpenOptions) -> io::Result { let segment_path = segment_path.to_path_buf(); let mut std_opts = std::fs::OpenOptions::new(); std_opts.read(true).write(true); @@ -185,32 +247,34 @@ impl BlockDevice { #[allow(clippy::unused_self)] pub(super) fn rename_segment(&self, from: &Path, to: &Path) -> io::Result<()> { - std::fs::rename(from, to) + blocking_io(|| std::fs::rename(from, to)) } #[allow(clippy::unused_self)] pub(super) fn unlink_segment(&self, segment_path: &Path) -> io::Result<()> { - std::fs::remove_file(segment_path) + blocking_io(|| std::fs::remove_file(segment_path)) } #[allow(clippy::unused_self)] pub(super) fn list_layout(&self, layout_path: &Path) -> io::Result> { - let mut entries = Vec::new(); - let rd = std::fs::read_dir(layout_path)?; - for entry in rd.flatten() { - entries.push(entry.path()); - } - Ok(entries) + blocking_io(|| { + let mut entries = Vec::new(); + let rd = std::fs::read_dir(layout_path)?; + for entry in rd.flatten() { + entries.push(entry.path()); + } + Ok(entries) + }) } #[allow(clippy::unused_self)] pub(super) fn create_layout(&self, layout_path: &Path) -> io::Result<()> { - std::fs::create_dir_all(layout_path) + blocking_io(|| std::fs::create_dir_all(layout_path)) } #[allow(clippy::unused_self)] pub(super) fn contains_path(&self, path: &Path) -> bool { - path.exists() + blocking_io(|| path.exists()) } } @@ -227,12 +291,16 @@ pub(crate) struct BlockSegment { impl BlockSegment { pub(super) fn write_at(&self, data: &[u8], offset: u64) -> io::Result { + #[cfg(feature = "test-util")] + self.device.pause_io_for_tests(false); self.write_bytes_to_file(&self.file, data, offset)?; self.device.write_count.fetch_add(1, Ordering::AcqRel); Ok(data.len()) } pub(super) fn write_vectored_at(&self, bufs: &[std::io::IoSlice<'_>], offset: u64) -> io::Result { + #[cfg(feature = "test-util")] + self.device.pause_io_for_tests(false); let total_len: usize = bufs.iter().map(|b| b.len()).sum(); let mut cur_offset = offset; for buf in bufs { @@ -312,6 +380,8 @@ impl BlockSegment { } pub(super) fn fdatasync(&self) -> io::Result<()> { + #[cfg(feature = "test-util")] + self.device.pause_io_for_tests(true); self.device.fdatasync_count.fetch_add(1, Ordering::AcqRel); self.file.sync_data()?; Ok(()) diff --git a/lib/crowdb-kv/src/wal/wal_file.rs b/lib/crowdb-kv/src/wal/wal_file.rs index 80bf288b4..36c704009 100644 --- a/lib/crowdb-kv/src/wal/wal_file.rs +++ b/lib/crowdb-kv/src/wal/wal_file.rs @@ -39,7 +39,7 @@ impl WalFile { WalFileInner::File(f) => f.write_at(data, offset).await, WalFileInner::Uring(f) => f.write_at(data, offset).await, WalFileInner::MemBlock(f) => f.write_at(data, offset), - WalFileInner::Block(f) => f.write_at(data, offset), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.write_at(data, offset)), } } @@ -58,7 +58,7 @@ impl WalFile { WalFileInner::File(f) => f.write_vectored_at(bufs, offset).await, WalFileInner::Uring(f) => f.write_vectored_at(bufs, offset).await, WalFileInner::MemBlock(f) => f.write_vectored_at(bufs, offset), - WalFileInner::Block(f) => f.write_vectored_at(bufs, offset), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.write_vectored_at(bufs, offset)), } } @@ -71,7 +71,7 @@ impl WalFile { WalFileInner::File(f) => f.read_at(buf, offset).await, WalFileInner::Uring(f) => f.read_at(buf, offset).await, WalFileInner::MemBlock(f) => f.read_at(buf, offset), - WalFileInner::Block(f) => f.read_at(buf, offset), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.read_at(buf, offset)), } } @@ -84,7 +84,7 @@ impl WalFile { WalFileInner::File(f) => f.read_exact_at(buf, offset).await, WalFileInner::Uring(f) => f.read_exact_at(buf, offset).await, WalFileInner::MemBlock(f) => f.read_exact_at(buf, offset), - WalFileInner::Block(f) => f.read_exact_at(buf, offset), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.read_exact_at(buf, offset)), } } @@ -97,7 +97,7 @@ impl WalFile { WalFileInner::File(f) => f.fdatasync().await, WalFileInner::Uring(f) => f.fdatasync().await, WalFileInner::MemBlock(f) => f.fdatasync(), - WalFileInner::Block(f) => f.fdatasync(), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.fdatasync()), } } @@ -110,7 +110,7 @@ impl WalFile { WalFileInner::File(f) => f.fsync().await, WalFileInner::Uring(f) => f.fsync().await, WalFileInner::MemBlock(f) => f.fsync(), - WalFileInner::Block(f) => f.fsync(), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.fsync()), } } @@ -133,7 +133,7 @@ impl WalFile { WalFileInner::File(f) => f.len().await, WalFileInner::Uring(f) => f.len(), WalFileInner::MemBlock(f) => f.len(), - WalFileInner::Block(f) => f.len(), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.len()), } } @@ -146,7 +146,7 @@ impl WalFile { WalFileInner::File(f) => f.truncate(len).await, WalFileInner::Uring(f) => f.truncate(len), WalFileInner::MemBlock(f) => f.truncate(len), - WalFileInner::Block(f) => f.truncate(len), + WalFileInner::Block(f) => block_backend::blocking_io(|| f.truncate(len)), } } } diff --git a/lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs b/lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs index 6a678a14a..1965f3d88 100644 --- a/lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs +++ b/lib/crowdb-kv/tests/group_test/g3_leader_change_test.rs @@ -218,3 +218,65 @@ async fn leader_wait_does_not_accept_role_before_a_current_quorum_lease() { assert!(put_via_leader(&cluster, b"ready-leader", b"once", 3).await); cluster.shutdown().await; } + +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn prior_tenure_admin_wakeup_cannot_step_down_a_newly_elected_leader() { + use crowdb_kv::cluster::group::PxGroup; + use crowdb_kv::cluster::group_election::spawn; + use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; + use crowdb_kv::common::config::PxElectionConfig; + use std::sync::Arc; + + let config = PxElectionConfig::for_tests(); + let mut group = PxGroup::new(731, PxLocalReplica::new(19, PxLocalReplicaRole::Leader)); + group.set_election_config(config); + let group = Arc::new(group); + let replica = group.local_replica(); + let previous_term = replica.current_term_snapshot(); + // The admin request arrives before the previous leader loop consumes its + // wakeup. Its immediate role fence is real; the Notify permit is retained. + assert!( + replica + .handle_step_down(&StepDownRequestPayload { + term: previous_term, + target_leader_id: replica.id, + reason: "step down before the leader loop starts".into(), + }) + .accepted + ); + assert_eq!(replica.role(), PxLocalReplicaRole::Follower); + + let cancel = group.tenure_cancel(); + let driver = spawn(Arc::downgrade(&group), config, cancel.clone()); + for _ in 0..10 { + tokio::task::yield_now().await; + } + tokio::time::advance(Duration::from_millis(config.election_max_ms + 1)).await; + // Keep this task runnable so paused time cannot auto-advance to another + // election and conceal a stale-wakeup step-down of this exact tenure. + for _ in 0..30 { + tokio::task::yield_now().await; + } + let elected_term = replica.current_term_snapshot(); + let role = replica.role(); + let fresh_step_down = (role == PxLocalReplicaRole::Leader).then(|| { + replica.handle_step_down(&StepDownRequestPayload { + term: elected_term, + target_leader_id: replica.id, + reason: "a new request still fences the current tenure".into(), + }) + }); + cancel.cancel(); + driver.await.unwrap(); + assert!( + elected_term > previous_term, + "the real driver must complete a new election" + ); + assert_eq!( + role, + PxLocalReplicaRole::Leader, + "a prior-tenure admin wakeup stepped down the new leader" + ); + assert!(fresh_step_down.unwrap().accepted); + assert_eq!(replica.role(), PxLocalReplicaRole::Follower); +} diff --git a/lib/crowdb-kv/tests/wal_test.rs b/lib/crowdb-kv/tests/wal_test.rs index 802100e54..1c39a8e6b 100644 --- a/lib/crowdb-kv/tests/wal_test.rs +++ b/lib/crowdb-kv/tests/wal_test.rs @@ -30,6 +30,9 @@ mod pipeline_backend_test; #[path = "wal_test/block_backend_test.rs"] mod block_backend_test; +#[path = "wal_test/block_runtime_test.rs"] +mod block_runtime_test; + #[path = "wal_test/block_restore_test.rs"] mod block_restore_test; diff --git a/lib/crowdb-kv/tests/wal_test/block_runtime_test.rs b/lib/crowdb-kv/tests/wal_test/block_runtime_test.rs new file mode 100644 index 000000000..2211f4829 --- /dev/null +++ b/lib/crowdb-kv/tests/wal_test/block_runtime_test.rs @@ -0,0 +1,141 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Real block-backed WAL I/O must leave a single Tokio worker responsive. + +use std::sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, Arc, Barrier, +}; +use std::time::Duration; + +use crowdb_kv::paxos::roles::PxBallot; +use crowdb_kv::wal::{BlockDevice, IoBackend, OpenOptions, WALRecord, WalConfig, WalEngine}; + +async fn held_wal_io(sync: bool, expected_progress: bool) { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("block-runtime-held"); + let device = BlockDevice::ssd(); + let backend = Arc::new(IoBackend::BlockDevice(device.clone())); + let wal = WalEngine::create( + Arc::clone(&backend), + WalConfig { + wal_disks: vec![directory.path().into()], + ..Default::default() + }, + 1, + ) + .await + .unwrap(); + // Finish initial segment creation before installing the one-operation gate. + wal.append(&WALRecord::from_promised(1, 1, 1, PxBallot::new(0, 1))) + .await + .unwrap(); + let entered = Arc::new(Barrier::new(2)); + let release = Arc::new(Barrier::new(2)); + device.set_io_gate_for_tests(sync, Arc::clone(&entered), Arc::clone(&release)); + let acknowledged = Arc::new(AtomicBool::new(false)); + let controller_acknowledged = Arc::clone(&acknowledged); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (progress_tx, progress_rx) = mpsc::channel(); + let controller = std::thread::spawn(move || { + entered.wait(); + let _ = started_tx.send(()); + let progressed = progress_rx.recv_timeout(Duration::from_secs(1)).is_ok(); + let premature_ack = controller_acknowledged.load(Ordering::Acquire); + release.wait(); + (progressed, premature_ack) + }); + let independent = tokio::spawn(async move { + started_rx.await.unwrap(); + let _ = progress_tx.send(()); + }); + wal.append(&WALRecord::from_promised(1, 1, 2, PxBallot::new(0, 1))) + .await + .unwrap(); + acknowledged.store(true, Ordering::Release); + independent.await.unwrap(); + let (progressed, premature_ack) = controller.join().unwrap(); + assert!( + !premature_ack, + "WAL acknowledged before the held durable I/O completed" + ); + assert_eq!( + progressed, expected_progress, + "held BlockDevice WAL I/O blocked the only Tokio worker" + ); + assert!(device.fdatasync_count() >= 2); + wal.flush_all().await.unwrap(); + drop(wal); + let recovered = crowdb_kv::wal::replay::replay_group(&backend, &[directory.path().into()], 1) + .await + .unwrap(); + assert_eq!( + recovered + .records + .iter() + .map(|record| record.slot) + .collect::>(), + vec![1, 2] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn single_worker_progresses_while_block_wal_write_is_held() { + held_wal_io(false, true).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn single_worker_progresses_while_block_wal_sync_is_held() { + held_wal_io(true, true).await; +} + +#[tokio::test(flavor = "current_thread")] +async fn current_thread_fallback_remains_synchronous_without_panicking() { + held_wal_io(true, false).await; +} + +async fn block_io_round_trip() { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("block-runtime-reopen"); + let path = directory.path().join("block"); + let backend = IoBackend::BlockDevice(BlockDevice::ssd()); + let mut file = backend.open(&path, OpenOptions::create_rw()).await.unwrap(); + file.write_at(b"first", 7).await.unwrap(); + file.write_vectored_at( + &[std::io::IoSlice::new(b"next"), std::io::IoSlice::new(b"last")], + 21, + ) + .await + .unwrap(); + file.fdatasync().await.unwrap(); + drop(file); + let mut reopened = backend.open(&path, OpenOptions::read_only()).await.unwrap(); + let mut first = [0; 5]; + reopened.read_exact_at(&mut first, 7).await.unwrap(); + let mut next = [0; 8]; + reopened.read_exact_at(&mut next, 21).await.unwrap(); + assert_eq!(&first, b"first"); + assert_eq!(&next, b"nextlast"); + let mut short = [0; 8]; + assert_eq!(reopened.read_at(&mut short, 4093).await.unwrap(), 3); + assert_eq!(&short[..3], &[0; 3]); + let mut beyond = [0; 1]; + assert_eq!( + reopened + .read_exact_at(&mut beyond, 8192) + .await + .unwrap_err() + .kind(), + std::io::ErrorKind::UnexpectedEof + ); + assert!(backend.open(&path, OpenOptions::create_new_rw()).await.is_err()); +} + +#[tokio::test(flavor = "current_thread")] +async fn current_thread_block_io_preserves_order_rmw_and_durable_reopen() { + block_io_round_trip().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn single_worker_block_io_preserves_order_rmw_and_durable_reopen() { + block_io_round_trip().await; +} diff --git a/lib/crowdb-protocol/tests/chunk_task_value_test.rs b/lib/crowdb-protocol/tests/chunk_task_value_test.rs index fad1ccb4f..2b72b9e81 100644 --- a/lib/crowdb-protocol/tests/chunk_task_value_test.rs +++ b/lib/crowdb-protocol/tests/chunk_task_value_test.rs @@ -50,7 +50,7 @@ fn task_value_rejects_wrong_identifier_and_malformed_buffer() { #[test] fn task_value_bounds_last_error_without_splitting_utf8() { let mut value = task_value(); - value.last_error = "界".repeat(400); + value.last_error = "\u{754c}".repeat(400); let decoded = decode_chunk_task_value(&encode_chunk_task_value(&value)).unwrap(); assert!(decoded.last_error.len() <= 1_024); assert!(decoded.last_error.is_char_boundary(decoded.last_error.len())); diff --git a/lib/crowdb-tree/include/crowdb-tree/btree/tree.h b/lib/crowdb-tree/include/crowdb-tree/btree/tree.h index bc95ff3f1..267feb80d 100644 --- a/lib/crowdb-tree/include/crowdb-tree/btree/tree.h +++ b/lib/crowdb-tree/include/crowdb-tree/btree/tree.h @@ -717,8 +717,10 @@ class Crowdbtree void commit_prepared_snapshot(const PreparedSnapshot &prepared); void finalize_prepared_snapshot(PreparedSnapshot &prepared); - void acquire_snapshot_slot(); - void release_snapshot_slot(); + void acquire_snapshot_slot(); + void release_snapshot_slot(); + Status retry_snapshot_anchor(); + void snapshot_prepare_async(std::function on_done); void snapshot_write_next_async(std::shared_ptr prepared, size_t idx, std::function on_done); @@ -833,6 +835,9 @@ class Crowdbtree // snapshot_async's async write phase, where write_mutex_ itself can't be // held (see acquire_snapshot_slot's doc comment and snapshot_async's). std::atomic snapshot_inflight_{false}; + // Protected by snapshot_inflight_: retry an uncertain anchor barrier before + // preparing or writing another image over the backend's staged snapshot. + bool snapshot_anchor_pending_ = false; // Block indices that were empty in the previous snapshot (two-generation // rule for block deletion). A block is only deleted after it's empty in diff --git a/lib/crowdb-tree/src/snapshot/retry.cpp b/lib/crowdb-tree/src/snapshot/retry.cpp new file mode 100644 index 000000000..e83b9da85 --- /dev/null +++ b/lib/crowdb-tree/src/snapshot/retry.cpp @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "async_completion_adapter.h" +#include "crowdb-common/log.h" +#include "crowdb-tree/backend/async_page_store.h" +#include "crowdb-tree/crowdb-tree.h" + +#include +#include +#include + +namespace crowdb::tree +{ + +Status Crowdbtree::retry_snapshot_anchor() +{ + if (!snapshot_anchor_pending_) { + return Status::Ok(); + } + Status status = opt_.page_store->sync(); + if (status.ok()) { + snapshot_anchor_pending_ = false; + } + else { + CRB_LOG_ERROR("[{}] snapshot pending anchor sync failed: {}", name_, status.to_string()); + } + return status; +} + +void Crowdbtree::snapshot_prepare_async(std::function on_done) +{ + if (snapshot_anchor_pending_) { + Status submitted = opt_.async_page_store->submit_fsync( + detail::own_async_completion([this, on_done](const Status &status) mutable { + if (!status.ok()) { + CRB_LOG_ERROR("[{}] snapshot pending anchor sync failed: {}", name_, status.to_string()); + release_snapshot_slot(); + on_done(status, 0); + return; + } + snapshot_anchor_pending_ = false; + snapshot_prepare_async(std::move(on_done)); + })); + if (!submitted.ok()) { + release_snapshot_slot(); + on_done(std::move(submitted), 0); + } + return; + } + auto prepared = std::make_shared(); + Status status; + { + std::scoped_lock lock(write_mutex_); + status = prepare_snapshot_locked(prepared.get()); + } + if (!status.ok()) { + CRB_LOG_ERROR("[{}] snapshot prepare failed: {}", name_, status.to_string()); + release_snapshot_slot(); + on_done(std::move(status), 0); + return; + } + snapshot_write_next_async(std::move(prepared), 0, std::move(on_done)); +} + +} // namespace crowdb::tree diff --git a/lib/crowdb-tree/src/snapshot/write.cpp b/lib/crowdb-tree/src/snapshot/write.cpp index cb457f1cb..f0502ab6a 100644 --- a/lib/crowdb-tree/src/snapshot/write.cpp +++ b/lib/crowdb-tree/src/snapshot/write.cpp @@ -23,6 +23,11 @@ Status Crowdbtree::snapshot(uint64_t *out_last_applied, uint64_t *out_snapshot_s } auto snap_t0 = std::chrono::steady_clock::now(); acquire_snapshot_slot(); + Status pending = retry_snapshot_anchor(); + if (!pending.ok()) { + release_snapshot_slot(); + return pending; + } PreparedSnapshot prepared; Status ps; { @@ -98,8 +103,9 @@ Status Crowdbtree::snapshot(uint64_t *out_last_applied, uint64_t *out_snapshot_s release_snapshot_slot(); return aw; } - auto fsync2_t0 = std::chrono::steady_clock::now(); - Status sync2 = opt_.page_store->sync(); + auto fsync2_t0 = std::chrono::steady_clock::now(); + snapshot_anchor_pending_ = true; + Status sync2 = opt_.page_store->sync(); if (metrics_.fsync_l != nullptr) { auto ns = std::chrono::duration_cast(std::chrono::steady_clock::now() - fsync2_t0).count(); @@ -111,6 +117,7 @@ Status Crowdbtree::snapshot(uint64_t *out_last_applied, uint64_t *out_snapshot_s release_snapshot_slot(); return sync2; } + snapshot_anchor_pending_ = false; // Observe metadata write bytes: segments + directory + anchor. if (metrics_.snapshot_meta_write_bw != nullptr) { @@ -172,18 +179,7 @@ void Crowdbtree::snapshot_async( } if (opt_.async_page_store != nullptr) { acquire_snapshot_slot(); - auto prepared = std::make_shared(); - Status ps; - { - std::scoped_lock lk(write_mutex_); - ps = prepare_snapshot_locked(prepared.get()); - } - if (!ps.ok()) { - release_snapshot_slot(); - on_done(ps, 0); - return; - } - snapshot_write_next_async(std::move(prepared), 0, std::move(on_done)); + snapshot_prepare_async(std::move(on_done)); return; } // No async backend wired -- run the synchronous path in @@ -264,14 +260,17 @@ void Crowdbtree::snapshot_write_next_async( // NOLINT(readabi on_done(st3, 0); return; } - Status fs2 = opt_.async_page_store->submit_fsync( + snapshot_anchor_pending_ = true; + Status fs2 = opt_.async_page_store->submit_fsync( detail::own_async_completion([this, prepared, on_done](const Status &st4) mutable { if (!st4.ok()) { + CRB_LOG_ERROR("[{}] snapshot anchor sync failed: {}", name_, st4.to_string()); commit_prepared_snapshot(*prepared); release_snapshot_slot(); on_done(st4, 0); return; } + snapshot_anchor_pending_ = false; finalize_prepared_snapshot(*prepared); uint64_t last_applied = prepared->last_applied_slot; release_snapshot_slot(); @@ -295,6 +294,11 @@ Status Crowdbtree::persist_compaction_snapshot(std::vector prefe std::set selected_blocks, PreparedSnapshot *prepared) { acquire_snapshot_slot(); + Status pending = retry_snapshot_anchor(); + if (!pending.ok()) { + release_snapshot_slot(); + return pending; + } Status prepare; { std::scoped_lock lk(write_mutex_); diff --git a/lib/crowdb-tree/tests/integration/chunk_snapshot_retry_test.cpp b/lib/crowdb-tree/tests/integration/chunk_snapshot_retry_test.cpp new file mode 100644 index 000000000..e9fb5ea6e --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_snapshot_retry_test.cpp @@ -0,0 +1,115 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "backend/chunk/chunk_page_store.h" +#include "crowdb-tree/crowdb-tree.h" + +#include + +#include +#include +#include +#include + +namespace crowdb::tree::detail +{ +namespace +{ + +Status snapshot(Crowdbtree &tree, bool asynchronous) +{ + if (!asynchronous) { + return tree.snapshot(); + } + Status result; + std::atomic done{false}; + tree.snapshot_async([&](Status status, uint64_t /*unused*/) { + result = std::move(status); + done.store(true, std::memory_order_release); + done.notify_one(); + }); + done.wait(false, std::memory_order_acquire); + return result; +} + +void check_snapshot_retry(bool asynchronous, bool write_while_unavailable = false) +{ + auto catalog = std::make_shared(1); + auto transport = std::make_shared(); + ChunkPageStore store({.tree_id = 201, .owner_epoch = 1, .pack_bytes = 4096, .iu_size = 1}, catalog, transport); + Config options; + options.page_store = &store; + options.async_page_store = asynchronous ? &store : nullptr; + Crowdbtree tree(options); + ASSERT_TRUE(tree.apply(1, Batch{{{"key", OpKind::kPut, "old"}}}).ok()); + ASSERT_TRUE(tree.flush().ok()); + ASSERT_TRUE(snapshot(tree, asynchronous).ok()); + auto prior = catalog->load(201); + ASSERT_NE(prior, nullptr); + + ASSERT_TRUE(tree.apply(2, Batch{{{"key", OpKind::kPut, "new"}}}).ok()); + ASSERT_TRUE(tree.flush().ok()); + store.inject_mirror_write_failures(1U << 1U); + const Status failed = snapshot(tree, asynchronous); + ASSERT_EQ(failed.code(), Code::kUnavailable) << failed.to_string(); + ASSERT_EQ(catalog->load(201)->generation, prior->generation); + { + ChunkPageStore durable_store({.tree_id = 201, .owner_epoch = 1, .pack_bytes = 4096, .iu_size = 1}, catalog, + transport); + Config durable_options; + durable_options.page_store = &durable_store; + std::unique_ptr durable; + ASSERT_TRUE(Crowdbtree::open(durable_options, &durable).ok()); + std::string value; + uint64_t slot = 0; + ASSERT_TRUE(durable->get(Slice("key"), &slot, &value)); + EXPECT_EQ(slot, 1U); + EXPECT_EQ(value, "old"); + } + if (write_while_unavailable) { + ASSERT_TRUE(tree.apply(3, Batch{{{"key", OpKind::kPut, "newest"}}}).ok()); + ASSERT_TRUE(tree.flush().ok()); + } + const Status still_unavailable = snapshot(tree, asynchronous); + ASSERT_EQ(still_unavailable.code(), Code::kUnavailable) << still_unavailable.to_string(); + ASSERT_EQ(catalog->load(201)->generation, prior->generation); + store.inject_mirror_write_failures(0); + + const Status retried = snapshot(tree, asynchronous); + ASSERT_TRUE(retried.ok()) << retried.to_string(); + ASSERT_GT(catalog->load(201)->generation, prior->generation); + ChunkPageStore reopened_store({.tree_id = 201, .owner_epoch = 1, .pack_bytes = 4096, .iu_size = 1}, catalog, + transport); + options.page_store = &reopened_store; + options.async_page_store = asynchronous ? &reopened_store : nullptr; + std::unique_ptr reopened; + ASSERT_TRUE(Crowdbtree::open(options, &reopened).ok()); + std::string value; + uint64_t slot = 0; + ASSERT_TRUE(reopened->get(Slice("key"), &slot, &value)); + EXPECT_EQ(slot, write_while_unavailable ? 3U : 2U); + EXPECT_EQ(value, write_while_unavailable ? "newest" : "new"); +} + +TEST(ChunkSnapshotRetry, PublicationFailureDoesNotPoisonSynchronousSnapshotRetry) +{ + check_snapshot_retry(false); +} + +TEST(ChunkSnapshotRetry, PublicationFailureDoesNotPoisonAsynchronousSnapshotRetry) +{ + check_snapshot_retry(true); +} + +TEST(ChunkSnapshotRetry, SynchronousRetryPreservesWritesAfterFailedPublication) +{ + check_snapshot_retry(false, true); +} + +TEST(ChunkSnapshotRetry, AsynchronousRetryPreservesWritesAfterFailedPublication) +{ + check_snapshot_retry(true, true); +} + +} // namespace +} // namespace crowdb::tree::detail From 1f69095d16ab49a2bf5264282db659a6dafe4bce Mon Sep 17 00:00:00 2001 From: Gian Date: Wed, 4 Nov 2026 16:25:51 +0800 Subject: [PATCH 20/24] Correct drain logs and finalize physical test review --- app/crowdb-chunk-kv-server/src/main.rs | 8 +- doc/dev/physical-node-test-plan.md | 41 +++++--- doc/working/plan-ci-stability-review.md | 124 +++++++++++------------- 3 files changed, 87 insertions(+), 86 deletions(-) diff --git a/app/crowdb-chunk-kv-server/src/main.rs b/app/crowdb-chunk-kv-server/src/main.rs index 7ef7f3ec7..8320ab374 100644 --- a/app/crowdb-chunk-kv-server/src/main.rs +++ b/app/crowdb-chunk-kv-server/src/main.rs @@ -590,10 +590,12 @@ async fn main() -> std::process::ExitCode { refresh_task, )); info!("chunk KV background workers stopped"); - if let Err(error) = service.drain_writers().await { - warn!(%error, "WAL writer drain incomplete; restart must retain lease exclusion"); + match service.drain_writers().await { + Ok(()) => info!("chunk KV WAL writer drain complete"), + Err(error) => { + warn!(%error, "WAL writer drain incomplete; restart must retain lease exclusion"); + } } - info!("chunk KV WAL writer drain complete"); if let Err(error) = process_lease.withdraw_observation().await { warn!(%error, "failed to unregister chunk KV instance"); } diff --git a/doc/dev/physical-node-test-plan.md b/doc/dev/physical-node-test-plan.md index b217a7c43..b94b8bcf8 100644 --- a/doc/dev/physical-node-test-plan.md +++ b/doc/dev/physical-node-test-plan.md @@ -554,22 +554,37 @@ PID/start-time, and complete-audit assertions. - **P16.2 New and old data**: verify a new KV batch, new ChunkDB allocate/query, and all historical data separately. Preserve distinctions among acknowledgments, acknowledged deletions, and OutcomeUnknown. -- **P16.3 Full occupancy**: scan all this run's DiskDB occupancy read-only, +- **P16.3 Full occupancy**: scan all this run's DiskDB occupancy, covering every disk, zone, and continuation without truncation. Verify live mirror segments do not overlap across chunks; retain actual live, allocated, free, and reserved counts. -- **P16.4 Cutoff and metadata-change evidence**: use one recorded scan cutoff - for busy allocations and live Chunk metadata, then retain the completion - barrier and journal fence. A bounded current-version scan is not an MVCC - snapshot: matching busy-allocation or `/chunk/` mutations during that interval - must fail the unchanged stability guard. Capture the exact failed prefix and - slot interval immediately, before journal GC or expensive failure collection, - with one bounded 20-second read-only diagnostic. Retain raw and decoded keys, - values, operations and source identities to classify the changed fields. - Diagnostic failure must preserve the original audit exception. GC gaps or - truncation leave classification unknown; they cannot establish a consistent - scan or waive fixed-cutoff, scope, completeness or non-overlap assertions. - A later unchanged scan is a separate attempt, not proof about the failed one. +- **P16.4 Consistent inventory**: a bounded current-version scan is not an MVCC + snapshot. Record one initial cutoff for complete busy-allocation and + `/chunk/` scans. Either prove both prefixes unchanged through a valid + completion fence, or reconstruct both inventories at that fence by replaying + every intervening journal put and delete. Do not ignore metadata fields or + discard changed records. Apply all scope, completeness, allocation, bounds + and non-overlap checks to the resulting inventory. +- **P16.5 Completion fence and journal**: a coalesced ReadIndex may return a + conservative earlier read point. For reconstruction, submit one isolated + test-control marker to the same Group1 after both scans, outside the audited + prefixes. Record intent before submission, use a unique key, and require a + positive acknowledgment with a matching transport correlation ID and a + chosen revision beyond the initial cutoff. Record the logical request ID + separately. An unknown outcome fails without automatic resubmission. All + scans and replay are read-only; the marker is a separate control write. + Fetch the complete bounded journal interval for both prefixes without + splitting operations sharing a slot. Reject GC gaps, missing history, + truncation, a read slot below the fence, and foreign or unordered operations. + Retain raw scans, operations, reconstructed maps, and marker outcome. +- **P16.6 Failed stability evidence**: matching busy-allocation or `/chunk/` + mutations must fail an unchanged-inventory guard. Capture the exact failed + prefix and slot interval immediately, before journal GC or expensive failure + collection, with one bounded 20-second read-only diagnostic. Retain raw and + decoded keys, values, operations and source identities. Diagnostic failure + preserves the original audit exception. A GC gap or truncation leaves the + changed fields unknown. A later consistent audit is a separate qualification, + not proof about the failed scan or permission to weaken its assertions. ## 7. Category F: first-failure diagnosis, repair, and cleanup diff --git a/doc/working/plan-ci-stability-review.md b/doc/working/plan-ci-stability-review.md index 88d1c3d1f..d64d3b154 100644 --- a/doc/working/plan-ci-stability-review.md +++ b/doc/working/plan-ci-stability-review.md @@ -7,80 +7,64 @@ Upstream: [Physical Node Test Plan](../dev/physical-node-test-plan.md), [container acceptance](../../e2e-test/README.md), and [CI workflow](../../.github/workflows/ci.yml). -Goal: track the unresolved physical qualification and probe-observation work. +Goal: track the remaining physical-test stability issue. -## Current results +## Execution status -On 2026-11-04, the current-image physical P09 and independent P14 passed. -P13 failed its survivor lease-evidence assertion; its data and recovery -observations passed, but its final exit, stderr, and health qualification did -not run. P16 failed its fixed-cutoff occupancy audit after the data and capacity -checks passed. Earlier physical results remain scoped to their original image; -these targeted results do not constitute full-matrix acceptance. +The requested test execution and resource closure finished on 2026-11-04. +One complex issue remains recorded below under the user's instruction to finish +independent tests and defer complex repairs. Completed tasks are removed. -The three CI jobs associated with runs -[41329198767](https://github.com/buzzcrow/crowdb/actions/runs/41329198767) and -[41361029738](https://github.com/buzzcrow/crowdb/actions/runs/41361029738) -were rerun locally: UnitTests passed 1,181 tests and Console API passed 92 -cases. Console Native Operations first failed one case; its other 18 cases -passed, the failed case passed separately, and a second complete run passed all -19. The user subsequently approved a ten-second limit for this backend -dependency-convergence prerequisite, with diagnostics after three seconds; -UI and process-observation assertion budgets stay unchanged. The updated -focused case and complete Native job passed (19/19, 420.334 seconds). The two -dependency checks took 6 ms and 5 ms, so the three-second diagnostic did not -fire. The original failed migration window remains retained; this pass does -not establish its eventual recovery or the cause of its control-plane delays. -Evidence: `ci/requested-full-rerun-20261104/approved-convergence/`. - -The artifact root below is +All evidence paths below are relative to `.crowdb-runtime/artifacts/physical-nodes/physical-20261031T194149Z/`. -The five current test containers and seven scoped volumes remain retained. -## Unfinished tasks +- **Local CI reruns**: UnitTests 1,181 passed, Console API 92 passed, and Console + Native Operations 19 passed with the approved ten-second backend convergence + budget and three-second diagnostic. UI/process budgets stay unchanged. The + original Native failure is retained; its eventual recovery and control-plane + delay cause remain unproven. Evidence: + `ci/requested-full-rerun-20261104/approved-convergence/`. +- **Physical qualification**: P09 and independent P14 passed. P13 supplementary + qualification passed its corrected expired-lease matcher, positive exit + statuses, strict stderr, 24,311 healthy responses and unchanged 35-process + scope. P16 supplementary consistent-inventory qualification passed: 8,961 + busy records, 1,641 chunk records, 8,756 physical segments, no missing busy + allocations or overlaps, 35 unchanged process identities and 370 healthy + responses. Its 16 controls passed. Evidence: + `repair/readiness-probe-budget-2s/saved-p13-supplemental-review/original-gates/result.json` + and `repair/readiness-probe-budget-2s/p16-consistent-audit-staging/physical-attempt-2/result.json`. + Original failures remain retained. Results stay scoped to their actual source + and image; this is not fresh full-matrix acceptance. +- **Probe review**: all 141 CI warnings occurred in fault/restart scenarios; + 119 have same-endpoint recovery evidence. The other 22 Native records lack + exact saved Ready generations. P14's five and P13's 40 physical failures have + subsequent-generation recovery. No stable-load failure is established, so the + production two-second probe budget stays unchanged. Evidence: + `repair/readiness-probe-budget-2s/probe-review-completion-20261104/review.json`. +- **Resource closure**: all seven exact owned containers and volumes were + retired after complete service/crash-log capture and verified B-to-A copying. + Resource absence, closed listeners, evidence hashes and SSH access passed. + Docker's varying mount order caused one pre-delete refusal; a checked + complete-entry sort allowed the exact comparison, without dropping fields. + That refusal is retained. No new volume archives or broad pruning. Evidence: + `repair/readiness-probe-budget-2s/resource-closure-preparation/driver/disposition.json`. -For each issue, stop after two unsuccessful repair attempts, record the result, -and continue independent work. Complex issues are recorded for later analysis. +## Unfinished task -- [ ] **P16 occupancy audit**: Determine why `/chunk/` metadata changed during - the fixed-cutoff audit over slots 2415597..=2415605. The immediate read-only - journal capture found two chunk updates and no busy updates, but did not - establish predecessor values, changed fields, or whether allocations overlap. - Obtain sufficient evidence for a consistent audit, preserve the original - completeness and nonoverlap checks, and requalify P16. Files: - `repair/readiness-probe-budget-2s/host-readiness-contract-continuation/independent-p16-attempt-1/run/`, - especially `P16-immediate-journal/`. -- [ ] **P13 survivor qualification**: Integrate and verify the staged test - correction for the valid expired-lease fencing path, then rerun the complete - P13 action with its original data, observer, exit-status, stderr, and health - checks. The staged focused controls passed, but do not replace the original - failed physical qualification. Files: - `repair/readiness-probe-budget-2s/p13-lease-evidence-staging/` and - `repair/readiness-probe-budget-2s/host-readiness-contract-continuation/targeted-physical-driver/run/`. -- [ ] **P13 WAL drain**: Investigate the two survivor processes' incomplete WAL - writer drain during lease fencing, including stalled partition writes and - expired-lease withdrawal. The current logs do not prove a successful drain or - identify its cause. Files: the P13 survivor application and RPC logs under - `faults/readiness-probe-budget-2s-P13-survivor-exit-logs/`. -- [ ] **Review failures with a two-second probe budget**: Inspect the saved failures, - particularly those without per-endpoint recovery evidence, and decide whether - any timeout should change. Keep sampling after failures and report the - configured budget, actual elapsed time, error, and recovery; do not silently - increase the two-second budget. P14 recorded five 2000 ms ChunkDB deadlines - during its fault, with 2000/2000/2001/2007/2000 ms elapsed and recovery in - the next generation. The CI reruns recorded further fault/restart warnings, - some with exact recovery unknown. Files: - `repair/readiness-probe-budget-2s/host-readiness-contract-continuation/p13-probe-inventory/`, - `repair/readiness-probe-budget-2s/host-readiness-contract-continuation/independent-p14-attempt-1/run/`, - and `ci/requested-full-rerun-20261104/attempt-{1,2}/access-warning-audit.json`. - The approved ten-second convergence runs add eight warnings with the same - 2000 ms dependency and 2100 ms outer budgets; actual elapsed times are 0, - 2000, or 2001 ms. Their exact per-endpoint recovery remains unconfirmed. - File: `ci/requested-full-rerun-20261104/approved-convergence/access-warning-audit.json`. -- [ ] **Finish physical qualification and resource closure**: After addressing or - explicitly disposing of the remaining failures, reconcile all results by - source and image, preserve their logs, and decide the exact cleanup of the - owned test resources. Physical work is stopped at the user's request after - CI verification and publication. Do not represent mixed-image results as a - fresh full-matrix pass. The previous detailed plan and its deferred review - descriptions remain at `ci/requested-full-rerun-20261104/plan-before-open-items.md`. +- [ ] **P13 WAL release failure path**: During lease-fenced shutdown, B's two + original survivor processes reported `partition writes are stalled`. Identify + the affected partition/stream and first typed failure in + `release_drained_writer`, journal release, stream command dispatch/completion, + or worker release. Existing logs do not distinguish these paths or prove a + successful historical release. Reproduce the smallest proven cause before + selecting a repair, then verify successor lease exclusion and acknowledged + data replay. Stop after two unsuccessful repair attempts and retain evidence. + Files: `app/crowdb-chunk-kv-server/src/server/drain.rs`, + `lib/crowdb-chunk-kv/src/partition/drain.rs`, + `lib/crowdb-chunk-kv/src/partition/journal.rs`, and + `lib/crowdb-chunk-stream/src/stream.rs`. + Evidence: `repair/readiness-probe-budget-2s/wal-drain-log-repair/unresolved-issue.json`. + The separate unconditional success-log bug in + `app/crowdb-chunk-kv-server/src/main.rs` is fixed; three graceful-drain tests, + Rust fmt and focused clippy passed. The physical image was not rebuilt for + this logging-only change; it does not repair the release path. From 26cf5d101f175815c34756c660de75b3a61e12b8 Mon Sep 17 00:00:00 2001 From: Gian Date: Wed, 4 Nov 2026 18:41:29 +0800 Subject: [PATCH 21/24] Fix publication outcomes, S3 listing, and service plan observation --- .../src/services/useNodeServicePlans.test.tsx | 12 +- .../ui/src/services/useNodeServicePlans.ts | 12 +- .../useNodeServicePlansObservation.test.tsx | 52 +++ .../s3/design-crowdb-access-s3.md | 2 + .../console/design-crowdb-console-ui.md | 22 ++ doc/working/plan-ci-stability-review.md | 102 ++--- e2e-test/tests/kv/slot_publication.rs | 17 +- lib/crowdb-access-s3/src/bucket.rs | 22 +- .../src/bucket/listed_record.rs | 47 +++ lib/crowdb-access-s3/src/bucket/listing.rs | 25 +- lib/crowdb-access-s3/src/metadata/key.rs | 51 +++ .../src/metadata/multipart.rs | 9 +- .../tests/bucket_listing_collision_test.rs | 196 ++++++++++ .../tests/common/bucket_listing_collision.rs | 197 ++++++++++ .../tests/lease_fenced_release_test.rs | 222 +++++++++++ .../binding/chunk_slots/publication/epochs.rs | 24 +- .../tests/chunk_slot_publication_test.rs | 147 +++++++ .../tests/common/chunk_slot_publication.rs | 66 ++++ .../tests/common/publication_peer.rs | 360 ++++++++++++++++++ 19 files changed, 1480 insertions(+), 105 deletions(-) create mode 100644 lib/crowdb-access-s3/src/bucket/listed_record.rs create mode 100644 lib/crowdb-access-s3/tests/bucket_listing_collision_test.rs create mode 100644 lib/crowdb-access-s3/tests/common/bucket_listing_collision.rs create mode 100644 lib/crowdb-chunk-stream/tests/lease_fenced_release_test.rs create mode 100644 lib/crowdb-kv-client/tests/chunk_slot_publication_test.rs create mode 100644 lib/crowdb-kv-client/tests/common/chunk_slot_publication.rs create mode 100644 lib/crowdb-kv-client/tests/common/publication_peer.rs diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx b/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx index 9b7c073c8..1f4d56d59 100644 --- a/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx +++ b/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx @@ -65,10 +65,11 @@ describe('node service plans', () => { existing.push({ node_id: 3, service_type: 'diskio', pid: 20 }); vi.mocked(listServers).mockResolvedValue(existing as Awaited>); let ownershipReady = false; - vi.mocked(serviceRequest).mockImplementation(async path => { + vi.mocked(serviceRequest).mockImplementation(async (path, method) => { if (path === '/service-plans') return { 1: { revision: 7, steps: first }, 2: { revision: 7, steps: second } }; if (path === '/chunk-storage-readiness') return { ready: ownershipReady, reason: ownershipReady ? undefined : 'Waiting: storage' }; if (path === '/deployment-defaults') return { 'access-server': { instance_id: '2', http_port: 9092, s3_port: 9091 } }; + if (path === '/nodes/2/services/deploy' && method === 'POST') existing.push({ node_id: 2, service_type: 'access-server', pid: 31 }); return { revision: 8, ready: true }; }); const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); @@ -78,7 +79,7 @@ describe('node service plans', () => { expect(deployments()).toHaveLength(0); expect(result.current.plans[2]['access-server'].state).toBe('waiting'); ownershipReady = true; - vi.mocked(listServers).mockResolvedValue([...existing, { node_id: 2, service_type: 'chunk-kv', pid: 30 }] as Awaited>); + existing.push({ node_id: 2, service_type: 'chunk-kv', pid: 30 }); await act(async () => { await vi.advanceTimersByTimeAsync(600); }); expect(deployments()).toHaveLength(1); expect(result.current.plans[2]['access-server'].state).toBe('deployed'); @@ -142,15 +143,18 @@ describe('node service plans', () => { for (const kind of serviceOrder) steps[kind] = { state: 'disabled' }; steps['chunk-kv'] = { state: 'waiting' }; let restarted = false; + let chunkKvRunning = false; vi.mocked(listServers).mockImplementation(async () => [ { node_id: 1, service_type: 'chunkdb', pid: 12 }, { node_id: 1, service_type: 'diskio', pid: 13 }, { node_id: 2, service_type: 'diskio', pid: restarted ? 14 : undefined }, + ...(chunkKvRunning ? [{ node_id: 1, service_type: 'chunk-kv', pid: 15 }] : []), ] as Awaited>); const request = vi.mocked(serviceRequest).getMockImplementation()!; vi.mocked(serviceRequest).mockImplementation(async (path, method, body) => { if (path === '/service-plans') return { 1: { revision: 1, steps } }; if (path === '/deployment-defaults') return { 'chunk-kv': { instance_id: '1', http_port: 12010, rpc_port: 12110 } }; + if (path === '/nodes/1/services/deploy' && method === 'POST') chunkKvRunning = true; return request(path, method, body); }); const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); @@ -178,11 +182,13 @@ describe('node service plans', () => { expect(serviceRequest).toHaveBeenCalledWith('/nodes/1/service-plan', 'PUT', expect.objectContaining({ revision: 7 })); }); it('keeps a deployment queued when ownership changes after its readiness probe', async () => { - vi.mocked(listServers).mockResolvedValue([{ node_id: 1, service_type: 'diskio', pid: 10 }] as Awaited>); + const existing = [{ node_id: 1, service_type: 'diskio', pid: 10 }] as Awaited>; + vi.mocked(listServers).mockResolvedValue(existing); let ownershipPublished = false; const request = vi.mocked(serviceRequest).getMockImplementation()!; vi.mocked(serviceRequest).mockImplementation(async (path, method, body) => { if (path.endsWith('/services/deploy') && !ownershipPublished) throw new Error('HTTP 409: Waiting: every configured disk group needs live DiskIO ownership'); + if (path.endsWith('/services/deploy') && method === 'POST') existing.push({ node_id: 1, service_type: 'chunkdb', pid: 11 } as typeof existing[number]); return request(path, method, body); }); const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlans.ts b/app/crowdb-web/ui/src/services/useNodeServicePlans.ts index f6e9b4643..079ea5fc8 100644 --- a/app/crowdb-web/ui/src/services/useNodeServicePlans.ts +++ b/app/crowdb-web/ui/src/services/useNodeServicePlans.ts @@ -26,6 +26,11 @@ export type NodeServicePlan = Record; export type ServiceOverrides = Partial> & { keepalive_interval_secs?: number }>>; export const newPlan = (): NodeServicePlan => Object.fromEntries(serviceOrder.map(kind => [kind, { state: 'pending' }])) as NodeServicePlan; +function serviceFailureDetail(kind: ServiceKind, overrides: ServiceOverrides | undefined, error: unknown) { + const listeners = Object.entries(overrides?.[kind] ?? {}).map(([name, port]) => `${name}=${port}`).join(', '); + return `${serviceLabels[kind]}${listeners ? ` (${listeners})` : ''}: ${String(error)}`.slice(0, 4096); +} + function observationDelay(plans: Record, hasDisks: boolean, waitingForStorage: boolean, immediate: boolean) { if (immediate) return 0; const steps = Object.values(plans).flatMap(plan => serviceOrder.map(kind => plan[kind])); @@ -148,7 +153,7 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< if (disposed || stopped.current) return; for (const [key, initial] of Object.entries(plansRef.current)) { if (disposed || stopped.current) break; - if (!serviceOrder.some(kind => ['pending', 'waiting', 'failed'].includes(initial[kind].state))) continue; + if (!serviceOrder.some(kind => ['pending', 'waiting', 'failed', 'deployed'].includes(initial[kind].state))) continue; const id = Number(key); let plan = plansRef.current[id]; const write = async (kind: ServiceKind, step: ServiceStep) => { plan = { ...plan, [kind]: step }; await persist(id, plan); }; @@ -198,7 +203,7 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< ? { state: 'waiting', detail: startingService } : registered.process_observed === false ? { state: 'waiting', detail: observingService } - : { state: 'failed', detail: 'Registered service is stopped; restart it from the Node menu before resuming.' }; + : { state: 'failed', detail: serviceFailureDetail(kind, overrides.current[id], 'Registered service is stopped; restart it from the Node menu before resuming.') }; if (plan[kind].state !== step.state || plan[kind].detail !== step.detail) await write(kind, step); continue; } @@ -257,8 +262,7 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< await write(kind, { state: 'waiting', detail: dependency[1] }); continue; } - const listeners = Object.entries(overrides.current[id]?.[kind] ?? {}).map(([name, port]) => `${name}=${port}`).join(', '); - const detail = `${serviceLabels[kind]}${listeners ? ` (${listeners})` : ''}: ${String(error)}`.slice(0, 4096); + const detail = serviceFailureDetail(kind, overrides.current[id], error); // A node without a fixed ChunkDB slot is valid. Keep it as a // warning so the rest of the node plan and existing balance // services continue running. diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlansObservation.test.tsx b/app/crowdb-web/ui/src/services/useNodeServicePlansObservation.test.tsx index 861b7de82..61284d522 100644 --- a/app/crowdb-web/ui/src/services/useNodeServicePlansObservation.test.tsx +++ b/app/crowdb-web/ui/src/services/useNodeServicePlansObservation.test.tsx @@ -216,3 +216,55 @@ it('keeps unattempted work waiting across an unavailable inventory and resumes o expect(result.current.plans[1]['paxos-kv'].state).toBe('deployed'); noLifecycleMutation(); }); + +it.each(['stopped', 'unknown'] as const)('continues observing a completed plan after a transient PID becomes %s', async firstLoss => { + const steps = failedPlan(); + steps['paxos-kv'] = { state: 'deployed' }; + steps.diskdb = { state: 'failed', detail: 'crowdb-disk-db (rpc_port=7100): Previously reported stopped' }; + const overrides = { diskdb: { rpc_port: 7100 } }; + vi.mocked(serviceRequest).mockImplementation(async path => path === '/service-plans' + ? { 1: { revision: 7, steps, overrides } } : { revision: 8, ready: true }); + let state: ProcessState = 'unknown'; + vi.mocked(listServers).mockImplementation(async () => [kv('running'), { + ...diskdb(state === 'running', false), process_observed: state !== 'unknown', + }]); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + const preserved = () => { + expect(result.current.plans[1]['paxos-kv']).toEqual(steps['paxos-kv']); + for (const kind of serviceOrder) if (kind !== 'paxos-kv' && kind !== 'diskdb') { + expect(result.current.plans[1][kind]).toEqual({ state: 'disabled' }); + } + const writes = vi.mocked(serviceRequest).mock.calls.filter(([path, method]) => path === '/nodes/1/service-plan' && method === 'PUT'); + for (const [, , body] of writes) expect(body).toMatchObject({ overrides }); + noLifecycleMutation(); + }; + await act(async () => {}); + expect(result.current.plans[1].diskdb).toEqual(steps.diskdb); + preserved(); + state = 'running'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1].diskdb.state).toBe('deployed'); + expect(serviceOrder.every(kind => ['deployed', 'disabled'].includes(result.current.plans[1][kind].state))).toBe(true); + preserved(); + state = firstLoss; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1].diskdb.state).toBe(firstLoss === 'unknown' ? 'waiting' : 'failed'); + preserved(); + if (firstLoss === 'unknown') { + expect(result.current.plans[1].diskdb.detail).toMatch(/process|observation/i); + state = 'stopped'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1].diskdb.state).toBe('failed'); + } + expect(result.current.plans[1].diskdb.detail).toContain('crowdb-disk-db'); + expect(result.current.plans[1].diskdb.detail).toContain('7100'); + preserved(); + state = 'unknown'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1].diskdb.state).toBe('failed'); + preserved(); + state = 'running'; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1].diskdb.state).toBe('deployed'); + preserved(); +}); diff --git a/doc/design/access-server/s3/design-crowdb-access-s3.md b/doc/design/access-server/s3/design-crowdb-access-s3.md index 0ab9c4dc1..b48c004e7 100644 --- a/doc/design/access-server/s3/design-crowdb-access-s3.md +++ b/doc/design/access-server/s3/design-crowdb-access-s3.md @@ -128,6 +128,8 @@ Bucket paths accept a single trailing slash. ListObjectsV2 supports URL encoding of XML-incompatible keys and selected prefix/delimiter fields. ListBuckets reports the Unix epoch as a stable CreationDate placeholder because bucket records do not retain creation timestamps. +Bucket listings skip object and multipart rows sharing the metadata interval only +after validating their complete key and value identities; unknown or corrupt rows fail. User metadata is accepted through x-amz-meta-* headers on ordinary PUT, multipart initiation and CopyObject. Nonempty HTTP token names are normalized diff --git a/doc/design/console/design-crowdb-console-ui.md b/doc/design/console/design-crowdb-console-ui.md index 09aa7cc71..243492726 100644 --- a/doc/design/console/design-crowdb-console-ui.md +++ b/doc/design/console/design-crowdb-console-ui.md @@ -607,6 +607,10 @@ resource menu. Data browsing is read-only; Query/Scan remain center query tools. deployment is queued. A fresh, identity-matched running process automatically reconciles that registered service's progress to deployed. This also reconciles an interrupted deployment whose owned process is later confirmed running. +- A plan whose selected steps are all deployed continues read-only process + observation every two seconds. A short-lived running process cannot freeze + progress after it exits. Confirmed stopped diagnostics retain the service name + and configured listener overrides; successful siblings and overrides persist. - Read-only recovery never sends Start, Restart, or Deploy again. Missing or unverified process evidence does not clear a recorded failure. Explicit Retry is required to initiate another failed lifecycle mutation; an accepted startup @@ -643,6 +647,24 @@ startup becomes failed and continues observation. A later fresh running process reconciles progress to deployed automatically without a lifecycle request. The existing Retry failed services action initiates a needed lifecycle mutation. +A previously completed plan after a confirmed stopped process: + +```text ++--------------------------------------------------------------------+ +| Node 1 services | +| | +| crowdb-access-server disabled | +| crowdb-chunk-kv disabled | +| crowdb-chunk-db disabled | +| crowdb-disk-db failed | +| crowdb-disk-db (rpc_port=7100): Registered service is stopped | +| crowdb-disk-io disabled | +| crowdb-paxos-kv deployed | +| | +| [Cancel] [Retry failed services] | ++--------------------------------------------------------------------+ +``` + ## 14. DiskGroup and disk management - **DISK-01:** The physical tree ends at Disk: Rack → Node → Storage → DiskGroup diff --git a/doc/working/plan-ci-stability-review.md b/doc/working/plan-ci-stability-review.md index d64d3b154..78de0755f 100644 --- a/doc/working/plan-ci-stability-review.md +++ b/doc/working/plan-ci-stability-review.md @@ -3,68 +3,40 @@ # CI Stability Review Plan -Upstream: [Physical Node Test Plan](../dev/physical-node-test-plan.md), -[container acceptance](../../e2e-test/README.md), and -[CI workflow](../../.github/workflows/ci.yml). - -Goal: track the remaining physical-test stability issue. - -## Execution status - -The requested test execution and resource closure finished on 2026-11-04. -One complex issue remains recorded below under the user's instruction to finish -independent tests and defer complex repairs. Completed tasks are removed. - -All evidence paths below are relative to -`.crowdb-runtime/artifacts/physical-nodes/physical-20261031T194149Z/`. - -- **Local CI reruns**: UnitTests 1,181 passed, Console API 92 passed, and Console - Native Operations 19 passed with the approved ten-second backend convergence - budget and three-second diagnostic. UI/process budgets stay unchanged. The - original Native failure is retained; its eventual recovery and control-plane - delay cause remain unproven. Evidence: - `ci/requested-full-rerun-20261104/approved-convergence/`. -- **Physical qualification**: P09 and independent P14 passed. P13 supplementary - qualification passed its corrected expired-lease matcher, positive exit - statuses, strict stderr, 24,311 healthy responses and unchanged 35-process - scope. P16 supplementary consistent-inventory qualification passed: 8,961 - busy records, 1,641 chunk records, 8,756 physical segments, no missing busy - allocations or overlaps, 35 unchanged process identities and 370 healthy - responses. Its 16 controls passed. Evidence: - `repair/readiness-probe-budget-2s/saved-p13-supplemental-review/original-gates/result.json` - and `repair/readiness-probe-budget-2s/p16-consistent-audit-staging/physical-attempt-2/result.json`. - Original failures remain retained. Results stay scoped to their actual source - and image; this is not fresh full-matrix acceptance. -- **Probe review**: all 141 CI warnings occurred in fault/restart scenarios; - 119 have same-endpoint recovery evidence. The other 22 Native records lack - exact saved Ready generations. P14's five and P13's 40 physical failures have - subsequent-generation recovery. No stable-load failure is established, so the - production two-second probe budget stays unchanged. Evidence: - `repair/readiness-probe-budget-2s/probe-review-completion-20261104/review.json`. -- **Resource closure**: all seven exact owned containers and volumes were - retired after complete service/crash-log capture and verified B-to-A copying. - Resource absence, closed listeners, evidence hashes and SSH access passed. - Docker's varying mount order caused one pre-delete refusal; a checked - complete-entry sort allowed the exact comparison, without dropping fields. - That refusal is retained. No new volume archives or broad pruning. Evidence: - `repair/readiness-probe-budget-2s/resource-closure-preparation/driver/disposition.json`. - -## Unfinished task - -- [ ] **P13 WAL release failure path**: During lease-fenced shutdown, B's two - original survivor processes reported `partition writes are stalled`. Identify - the affected partition/stream and first typed failure in - `release_drained_writer`, journal release, stream command dispatch/completion, - or worker release. Existing logs do not distinguish these paths or prove a - successful historical release. Reproduce the smallest proven cause before - selecting a repair, then verify successor lease exclusion and acknowledged - data replay. Stop after two unsuccessful repair attempts and retain evidence. - Files: `app/crowdb-chunk-kv-server/src/server/drain.rs`, - `lib/crowdb-chunk-kv/src/partition/drain.rs`, - `lib/crowdb-chunk-kv/src/partition/journal.rs`, and - `lib/crowdb-chunk-stream/src/stream.rs`. - Evidence: `repair/readiness-probe-budget-2s/wal-drain-log-repair/unresolved-issue.json`. - The separate unconditional success-log bug in - `app/crowdb-chunk-kv-server/src/main.rs` is fixed; three graceful-drain tests, - Rust fmt and focused clippy passed. The physical image was not rebuilt for - this logging-only change; it does not repair the release path. +The three requested CI repairs and local verification finished on 2026-11-04. +The user authorized pushing the completed changes on 2026-11-04. + +- ContainerKV: 50 fixtures and 58 client phases pass, including two-slot isolation. + Explicit CAS conflicts remain visible; an unknown publication outcome can no + longer become a retryable conflict through a failed reconciliation read. +- Deployment services: 33 fixtures and 41 client phases pass. Strict bucket-row + classification fixes the reproduced multipart metadata collision. The original + restart HTTP 500 lacks its offending row, so historical attribution stays qualified. +- Console native-data: the original ordered 14-case baseline passes. The fixed + image passes the original browser case and all nine affected node cases; retain + five Dataset baseline passes whose relevant inputs did not change. This is + incremental coverage, not a claim that all 14 ran on the fixed image. + Complete plans keep observing current processes. All 381 UI tests and types pass. + +CI evidence from repository root: +`.crowdb-runtime/artifacts/ci/new-failures-20261104/local-verification/`. +`console-fixed-coverage.json` records exact case membership and image qualification. +Original failures and repair RED/GREEN evidence remain in the three job directories. + +## Remaining task: deferred historical attribution + +- [ ] **P13 first aggregate WAL release error**: Identify its exact historical + partition/phase and durable release outcome if further attribution is required. + Six failing streams map to exact partitions, but the saved aggregate error does + not identify the first branch. A real-worker reproduction establishes that + idle liveness failure followed by authority revocation returns `WriteStalled` + before owner release with zero payload IO; healthy idle release and transient + recovery both pass. This warning alone does not establish an unfinished payload + write or corruption. No production fencing change is justified by that evidence. + The next investigation needs typed partition/stream/epoch/lifecycle/release-phase + and first stalled cause in a local server reproduction. Preserve lease + exclusion and replay. The three controls, fmt and focused clippy pass. + Evidence from repository root: + `.crowdb-runtime/artifacts/physical-nodes/physical-20261031T194149Z/repair/readiness-probe-budget-2s/wal-drain-research-20261104/findings.md`. + +Completed execution, probe review and resource-closure tasks have been removed. diff --git a/e2e-test/tests/kv/slot_publication.rs b/e2e-test/tests/kv/slot_publication.rs index 2705fb81e..33d970de3 100644 --- a/e2e-test/tests/kv/slot_publication.rs +++ b/e2e-test/tests/kv/slot_publication.rs @@ -121,8 +121,16 @@ async fn complete_epoch_publication_is_atomic_idempotent_and_storage_independent client.publish_service_epochs(&transfer), client.publish_service_epochs(&transfer) ); - assert!(a.is_ok() && b.is_ok(), "first: {a:?}; second: {b:?}"); + assert!( + matches!( + (&a, &b), + (Ok(()), Ok(()) | Err(crowdb_kv_client::Error::CasBusy)) + | (Err(crowdb_kv_client::Error::CasBusy), Ok(())) + ), + "first: {a:?}; second: {b:?}" + ); let after = client.read_service_snapshot().await.unwrap(); + assert_eq!(after.authority().head(), transfer.head()); for current in ChunkSlot::all() { assert_eq!(after.authority().owner(current), transfer.owner(current)); assert_eq!( @@ -134,6 +142,13 @@ async fn complete_epoch_publication_is_atomic_idempotent_and_storage_independent client.read_storage().await.unwrap().bindings(), storage.bindings() ); + // A busy caller reconciles the completed publication before submitting again. + client.publish_service_epochs(&transfer).await.unwrap(); + let repeated = client.read_service_snapshot().await.unwrap(); + assert_eq!(repeated.authority().head(), after.authority().head()); + assert_eq!(repeated.authority().bindings(), after.authority().bindings()); + assert_eq!(repeated.service().head(), after.service().head()); + assert_eq!(repeated.service().bindings(), after.service().bindings()); let reused = epochs(&service, 4, Some((slot, 1, 2))); assert!(client.publish_service_epochs(&reused).await.is_err()); let rollback = epochs(&service, 4, None); diff --git a/lib/crowdb-access-s3/src/bucket.rs b/lib/crowdb-access-s3/src/bucket.rs index c489fdd12..83406047f 100644 --- a/lib/crowdb-access-s3/src/bucket.rs +++ b/lib/crowdb-access-s3/src/bucket.rs @@ -3,12 +3,13 @@ //! Durable bucket-name lifecycle on the deliberately slow control path. +mod listed_record; mod listing; pub use listing::{BucketListPage, BucketListRequest, BucketListSigner}; use crate::metadata::{ BucketId, BucketNameRecord, ChunkKvMetadataStore, MetadataKey, MetadataRecordError, MetadataStoreError, - ObjectRecord, PutIfAbsentOutcome, TenantId, + PutIfAbsentOutcome, TenantId, }; const LIST_MAX_BYTES: usize = 4 * 1024 * 1024; @@ -132,15 +133,16 @@ pub async fn list_buckets( .await?; values .into_iter() - // Object keys can share this legacy interval when their random bucket - // ID starts with a low byte. Decode them as objects before treating a - // failed bucket decode as corrupt bucket metadata. - .filter_map(|value| match BucketNameRecord::decode(&value.value) { - Ok(record) if !record.tombstone => Some(Ok(record)), - Ok(_) => None, - Err(_) if ObjectRecord::decode(&value.value).is_ok() => None, - Err(error) => Some(Err(error.into())), - }) + // Resource keys can share this legacy interval when their random bucket + // ID starts with a low byte. Validate their complete persisted identities + // before treating a row as corrupt bucket metadata. + .filter_map( + |value| match listed_record::decode_bucket_row(tenant, &value.key, &value.value) { + Ok(Some(record)) if !record.tombstone => Some(Ok(record)), + Ok(_) => None, + Err(error) => Some(Err(error.into())), + }, + ) .collect() } diff --git a/lib/crowdb-access-s3/src/bucket/listed_record.rs b/lib/crowdb-access-s3/src/bucket/listed_record.rs new file mode 100644 index 000000000..c56473a9a --- /dev/null +++ b/lib/crowdb-access-s3/src/bucket/listed_record.rs @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Classify rows in the legacy bucket interval by complete persisted identity. + +use crate::metadata::{ + BucketNameRecord, MetadataKey, MetadataRecordError, MultipartPartRecord, MultipartSessionRecord, + ObjectRecord, TenantId, +}; + +pub(super) fn decode_bucket_row( + tenant: &TenantId, + key: &[u8], + value: &[u8], +) -> Result, MetadataRecordError> { + let error = match BucketNameRecord::decode(value) { + Ok(record) => { + if record.tenant == *tenant + && MetadataKey::bucket_name(tenant, &record.name).is_ok_and(|expected| expected == key) + { + return Ok(Some(record)); + } + MetadataRecordError::Invalid + } + Err(error) => error, + }; + if ObjectRecord::decode(value).is_ok_and(|record| { + MetadataKey::object(tenant, record.bucket_id, &record.key).is_ok_and(|expected| expected == key) + }) || MultipartSessionRecord::decode_unbound(value).is_ok_and(|record| { + MetadataKey::multipart_session(tenant, record.bucket_id, &record.upload_id) == key + }) || MultipartPartRecord::decode_unbound(value).is_ok_and(|record| { + MetadataKey::multipart_part(tenant, record.bucket_id, &record.upload_id, record.number) + .is_ok_and(|expected| expected == key) + || MetadataKey::multipart_part_generation( + tenant, + record.bucket_id, + &record.upload_id, + record.number, + record.revision, + ) + .is_ok_and(|expected| expected == key) + }) || MetadataKey::matches_multipart_index(tenant, key, value) + { + return Ok(None); + } + Err(error) +} diff --git a/lib/crowdb-access-s3/src/bucket/listing.rs b/lib/crowdb-access-s3/src/bucket/listing.rs index 47059c299..02a79ba21 100644 --- a/lib/crowdb-access-s3/src/bucket/listing.rs +++ b/lib/crowdb-access-s3/src/bucket/listing.rs @@ -8,10 +8,8 @@ use hmac::{Hmac, Mac}; use serde::{Deserialize, Serialize}; use sha2::Sha256; -use super::BucketError; -use crate::metadata::{ - BucketNameRecord, ChunkKvMetadataStore, MetadataKey, MetadataStoreError, ObjectRecord, TenantId, -}; +use super::{listed_record::decode_bucket_row, BucketError}; +use crate::metadata::{BucketNameRecord, ChunkKvMetadataStore, MetadataKey, MetadataStoreError, TenantId}; const MAX_CURSOR_BYTES: usize = 64 * 1024; const CURSOR_CONTEXT: &[u8] = b"crowdb-list-buckets-v1\0"; @@ -87,20 +85,13 @@ impl BucketListSigner { let mut buckets = Vec::new(); let mut continuation = None; for (index, value) in page.items.into_iter().enumerate() { - match BucketNameRecord::decode(&value.value) { - Ok(record) => { - if record.tenant != *tenant - || MetadataKey::bucket_name(tenant, &record.name)? != value.key - || !record.name.starts_with(request.prefix) - { - return Err(crate::metadata::MetadataRecordError::Invalid.into()); - } - if !record.tombstone { - buckets.push(record); - } + if let Some(record) = decode_bucket_row(tenant, &value.key, &value.value)? { + if !record.name.starts_with(request.prefix) { + return Err(crate::metadata::MetadataRecordError::Invalid.into()); + } + if !record.tombstone { + buckets.push(record); } - Err(_) if ObjectRecord::decode(&value.value).is_ok() => {} - Err(error) => return Err(error.into()), } if buckets.len() == request.max_buckets || index + 1 == count { if index + 1 < count || page.continuation.is_some() { diff --git a/lib/crowdb-access-s3/src/metadata/key.rs b/lib/crowdb-access-s3/src/metadata/key.rs index 051213c21..91dd04420 100644 --- a/lib/crowdb-access-s3/src/metadata/key.rs +++ b/lib/crowdb-access-s3/src/metadata/key.rs @@ -341,6 +341,38 @@ impl MetadataKey { key.extend_from_slice(&revision.to_be_bytes()); Ok(key) } + + pub(crate) fn matches_multipart_index(tenant: &TenantId, key: &[u8], value: &[u8]) -> bool { + let namespace = namespace_prefix(tenant); + let Some(pointer) = value.strip_prefix(namespace.as_slice()) else { + return false; + }; + if pointer.len() != 34 { + return false; + } + let mut bucket = [0; 16]; + bucket.copy_from_slice(&pointer[..16]); + let bucket = BucketId::new(bucket); + let mut upload = [0; 16]; + upload.copy_from_slice(&pointer[17..33]); + if upload == [0; 16] || value != Self::multipart_session(tenant, bucket, &upload) { + return false; + } + let prefix = Self::multipart_session_prefix(tenant, bucket); + let Some(encoded) = key + .strip_prefix(prefix.as_slice()) + .and_then(|key| key.strip_suffix(&upload)) + else { + return false; + }; + if encoded.len() > MAX_KEY_BYTES * 2 + 2 { + return false; + } + let Some(object) = decode_ordered_component(encoded) else { + return false; + }; + Self::multipart_upload_index(tenant, bucket, &object, &upload).is_ok_and(|expected| expected == key) + } } #[derive(Clone, Debug, Eq, PartialEq)] @@ -392,6 +424,25 @@ fn append_ordered_bytes_prefix(output: &mut Vec, value: &[u8]) { } } +fn decode_ordered_component(mut encoded: &[u8]) -> Option> { + let mut decoded = Vec::new(); + while let Some((&first, remaining)) = encoded.split_first() { + encoded = remaining; + if first != 0 { + decoded.push(first); + continue; + } + let (&escape, remaining) = encoded.split_first()?; + match escape { + 0 => return remaining.is_empty().then_some(decoded), + 0xff => decoded.push(0), + _ => return None, + } + encoded = remaining; + } + None +} + fn increment_lexicographic(value: &mut Vec) { while let Some(byte) = value.pop() { if byte != u8::MAX { diff --git a/lib/crowdb-access-s3/src/metadata/multipart.rs b/lib/crowdb-access-s3/src/metadata/multipart.rs index c7b795173..32378161b 100644 --- a/lib/crowdb-access-s3/src/metadata/multipart.rs +++ b/lib/crowdb-access-s3/src/metadata/multipart.rs @@ -240,14 +240,19 @@ impl MultipartPartRecord { upload_id: &[u8; 16], number: u16, ) -> Result { - let record: Self = decode(PART_MAGIC, bytes)?; - record.validate()?; + let record = Self::decode_unbound(bytes)?; if record.bucket_id != bucket || &record.upload_id != upload_id || record.number != number { return Err(MultipartRecordError::Identity); } Ok(record) } + pub(crate) fn decode_unbound(bytes: &[u8]) -> Result { + let record: Self = decode(PART_MAGIC, bytes)?; + record.validate()?; + Ok(record) + } + fn validate(&self) -> Result<(), MultipartRecordError> { if self.upload_id == [0; 16] || self.number == 0 || self.number > 10_000 || self.revision == 0 { return Err(MultipartRecordError::Invalid); diff --git a/lib/crowdb-access-s3/tests/bucket_listing_collision_test.rs b/lib/crowdb-access-s3/tests/bucket_listing_collision_test.rs new file mode 100644 index 000000000..fca8176a4 --- /dev/null +++ b/lib/crowdb-access-s3/tests/bucket_listing_collision_test.rs @@ -0,0 +1,196 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/bucket_client.rs"] +mod bucket_client; +#[path = "common/bucket_listing_collision.rs"] +mod fixture; + +use crowdb_access_s3::{ + bucket::{self, BucketError, BucketListRequest, BucketListSigner}, + metadata::{BucketId, MetadataKey, ObjectRecord, TenantId}, +}; +use fixture::{multipart_rows, paginated, session, store}; + +#[tokio::test] +async fn simple_listing_skips_valid_multipart_records_sharing_the_bucket_interval() { + let tenant = TenantId::new(b"tenant".to_vec()).unwrap(); + let mut failures = Vec::new(); + for (kind, key, value) in multipart_rows(&tenant) { + assert!( + key >= MetadataKey::bucket_name_prefix(&tenant) && key < MetadataKey::bucket_name_end(&tenant) + ); + let store = store(&tenant, key, value).await; + match bucket::list_buckets(&store, &tenant, 100).await { + Ok(buckets) => assert_eq!( + buckets.iter().map(|b| b.name.as_slice()).collect::>(), + vec![b"expected"] + ), + Err(error) => failures.push(format!("{kind}: {error:?}")), + } + } + assert!( + failures.is_empty(), + "valid multipart resource poisoned simple bucket listing: {failures:?}" + ); +} + +#[tokio::test] +async fn paginated_listing_skips_valid_multipart_records_sharing_the_bucket_interval() { + let tenant = TenantId::new(b"tenant".to_vec()).unwrap(); + let mut failures = Vec::new(); + for (kind, key, value) in multipart_rows(&tenant) { + let store = store(&tenant, key, value).await; + match paginated(&store, &tenant).await { + Ok(names) => assert_eq!(names, vec![b"expected".to_vec()]), + Err(error) => failures.push(format!("{kind}: {error:?}")), + } + } + assert!( + failures.is_empty(), + "valid multipart resource poisoned paginated bucket listing: {failures:?}" + ); +} + +#[tokio::test] +async fn malformed_bucket_rows_are_not_hidden_as_multipart_resources() { + let tenant = TenantId::new(b"tenant".to_vec()).unwrap(); + for value in [b"corrupt-bucket".to_vec(), session().encode().unwrap()] { + let key = MetadataKey::bucket_name(&tenant, b"bad-bucket").unwrap(); + let store = store(&tenant, key, value).await; + assert!(matches!( + bucket::list_buckets(&store, &tenant, 100).await, + Err(BucketError::Record(_)) + )); + assert!(matches!( + paginated(&store, &tenant).await, + Err(BucketError::Record(_)) + )); + } +} + +#[tokio::test] +async fn legacy_objects_are_skipped_only_under_their_exact_key() { + let tenant = TenantId::new(b"tenant".to_vec()).unwrap(); + let record = ObjectRecord { + bucket_id: session().bucket_id, + key: b"legacy-object".to_vec(), + logical_length: 1, + checksum: vec![9; 16], + etag: "etag".into(), + created_at_ms: 100, + modified_at_ms: 100, + content_type: "text/plain".into(), + attributes: Vec::new(), + data_reference: vec![7], + data_length: 1, + }; + let bytes = record.encode().unwrap(); + let good = store( + &tenant, + MetadataKey::object(&tenant, record.bucket_id, &record.key).unwrap(), + bytes.clone(), + ) + .await; + assert_eq!(bucket::list_buckets(&good, &tenant, 100).await.unwrap().len(), 1); + assert_eq!( + paginated(&good, &tenant).await.unwrap(), + vec![b"expected".to_vec()] + ); + let bad = store( + &tenant, + MetadataKey::bucket_name(&tenant, b"bad-object").unwrap(), + bytes, + ) + .await; + assert!(matches!( + bucket::list_buckets(&bad, &tenant, 100).await, + Err(BucketError::Record(_)) + )); + assert!(matches!( + paginated(&bad, &tenant).await, + Err(BucketError::Record(_)) + )); +} + +#[tokio::test] +async fn foreign_only_page_keeps_its_cursor_and_returns_later_buckets_once() { + let tenant = TenantId::new(b"tenant".to_vec()).unwrap(); + let (_, key, value) = multipart_rows(&tenant).remove(0); + let store = store(&tenant, key, value).await; + let signer = BucketListSigner::new(b"secret".to_vec()).unwrap(); + let options = |continuation| BucketListRequest { + prefix: b"", + continuation, + max_buckets: 1, + max_scan_items: 1, + max_scan_bytes: 65536, + now: 150, + ttl: 30, + }; + let first = signer.list(&store, &tenant, &options(None)).await.unwrap(); + assert!(first.buckets.is_empty()); + assert!(first.continuation.is_some()); + let second = signer + .list(&store, &tenant, &options(first.continuation.as_deref())) + .await + .unwrap(); + assert_eq!( + second + .buckets + .iter() + .map(|b| b.name.as_slice()) + .collect::>(), + vec![b"expected"] + ); + assert!(second.continuation.is_none()); +} + +#[tokio::test] +async fn multipart_rows_with_foreign_or_corrupt_identities_remain_errors() { + let tenant = TenantId::new(b"tenant".to_vec()).unwrap(); + let other = TenantId::new(b"other-tenant".to_vec()).unwrap(); + let mut bad = Vec::new(); + for (kind, key, value) in multipart_rows(&tenant) { + let mut wrong_value = value.clone(); + wrong_value[0] ^= 0x40; + bad.push((format!("{kind} unknown format"), key.clone(), wrong_value)); + let mut wrong_key = key.clone(); + let last = wrong_key.len() - 1; + wrong_key[last] ^= 1; + bad.push((format!("{kind} foreign key"), wrong_key, value)); + } + let original = session(); + let mut foreign = original.clone(); + foreign.bucket_id = BucketId::new([3; 16]); + bad.push(( + "foreign bucket session".into(), + MetadataKey::multipart_session(&tenant, original.bucket_id, &original.upload_id), + foreign.encode().unwrap(), + )); + let (_, index_key, index_value) = multipart_rows(&tenant).pop().unwrap(); + let foreign_pointer = MetadataKey::multipart_session(&other, original.bucket_id, &original.upload_id); + bad.push(("foreign tenant index".into(), index_key, foreign_pointer)); + let mut malformed_index = MetadataKey::multipart_session_prefix(&tenant, original.bucket_id); + malformed_index.extend_from_slice(b"object\0\x01"); + malformed_index.extend_from_slice(&original.upload_id); + bad.push(( + "invalid ordered index component".into(), + malformed_index, + index_value, + )); + for (kind, key, value) in bad { + let store = store(&tenant, key, value).await; + assert!( + matches!( + bucket::list_buckets(&store, &tenant, 100).await, + Err(BucketError::Record(_)) + ), + "simple listing accepted {kind}" + ); + assert!( + matches!(paginated(&store, &tenant).await, Err(BucketError::Record(_))), + "paginated listing accepted {kind}" + ); + } +} diff --git a/lib/crowdb-access-s3/tests/common/bucket_listing_collision.rs b/lib/crowdb-access-s3/tests/common/bucket_listing_collision.rs new file mode 100644 index 000000000..aecb5f7f6 --- /dev/null +++ b/lib/crowdb-access-s3/tests/common/bucket_listing_collision.rs @@ -0,0 +1,197 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::bucket_client; + +use async_trait::async_trait; +use crowdb_access_s3::{ + bucket::{BucketError, BucketListRequest, BucketListSigner}, + metadata::{ + BucketId, BucketNameRecord, ChunkKvMetadataStore, MetadataKey, MultipartPartRecord, MultipartPhase, + MultipartSessionRecord, TenantId, + }, +}; +use crowdb_chunk_kv_client::{ChunkKvTransport, Result}; +use crowdb_protocol::{chunk_kv::*, chunkdb::rpc::Location}; +use std::sync::Arc; + +struct TestTransport(Vec); +type TestMetadataRow = (&'static str, Vec, Vec); + +#[async_trait] +impl ChunkKvTransport for TestTransport { + async fn point(&self, _: &str, _: &PointRequest) -> Result { + panic!("listing must only scan"); + } + + async fn seek(&self, _: &str, _: &SeekRequest) -> Result { + panic!("listing must only scan"); + } + + async fn scan(&self, _: &str, request: &ScanRequest) -> Result { + let mut items: Vec<_> = self + .0 + .iter() + .filter(|item| { + request.start.as_ref().map_or(true, |start| item.key >= *start) + && request.end.as_ref().map_or(true, |end| item.key < *end) + && request + .continuation + .as_ref() + .map_or(true, |cursor| item.key > cursor.last_key) + }) + .cloned() + .collect(); + let more = items.len() > request.limit as usize; + items.truncate(request.limit as usize); + let continuation = more.then(|| ScanContinuation { + direction: request.direction, + last_key: items.last().unwrap().key.clone(), + partition_id: request.routing.partition_id, + owner_epoch: request.routing.owner_epoch, + map_revision: request.routing.map_revision, + }); + Ok(ChunkKvResponse { + map_revision: 1, + journal_position: None, + result: Ok(OperationResult::Scan { items, continuation }), + }) + } +} + +pub(super) fn session() -> MultipartSessionRecord { + let mut bucket = [0x11; 16]; + bucket[0] = 1; + MultipartSessionRecord { + bucket_id: BucketId::new(bucket), + object_key: b"multipart\0object".to_vec(), + upload_id: [7; 16], + revision: 1, + phase: MultipartPhase::Open, + created_ms: 100, + expires_ms: 200, + content_type: "application/octet-stream".into(), + attributes: Vec::new(), + max_parts: 10, + max_part_bytes: 100, + max_object_bytes: 500, + max_staged_bytes: 1000, + part_count: 0, + staged_bytes: 0, + pending: None, + selection: None, + legacy_observation: None, + } +} + +pub(super) fn multipart_rows(tenant: &TenantId) -> Vec { + let session = session(); + let session_key = MetadataKey::multipart_session(tenant, session.bucket_id, &session.upload_id); + let session_bytes = session.encode().unwrap(); + assert_eq!( + MultipartSessionRecord::decode( + &session_bytes, + session.bucket_id, + &session.object_key, + &session.upload_id + ) + .unwrap(), + session + ); + let part = MultipartPartRecord { + bucket_id: session.bucket_id, + upload_id: session.upload_id, + number: 1, + revision: 1, + modified_ms: 150, + length: 5, + raw_md5: [9; 16], + locations: vec![Location { + chunk_id: Some(crowdb_protocol::common::ChunkId { high: 1, low: 2 }), + offset: 10, + length: 39, + logical_offset: 0, + logical_length: 5, + }], + }; + let part_bytes = part.encode().unwrap(); + assert_eq!( + MultipartPartRecord::decode(&part_bytes, part.bucket_id, &part.upload_id, 1).unwrap(), + part + ); + vec![ + ("session", session_key.clone(), session_bytes), + ( + "part", + MetadataKey::multipart_part(tenant, part.bucket_id, &part.upload_id, 1).unwrap(), + part_bytes.clone(), + ), + ( + "part-generation", + MetadataKey::multipart_part_generation(tenant, part.bucket_id, &part.upload_id, 1, 1).unwrap(), + part_bytes, + ), + ( + "upload-index", + MetadataKey::multipart_upload_index( + tenant, + session.bucket_id, + &session.object_key, + &session.upload_id, + ) + .unwrap(), + session_key, + ), + ] +} + +pub(super) async fn store( + tenant: &TenantId, + extra_key: Vec, + extra_value: Vec, +) -> ChunkKvMetadataStore { + let mapping = BucketNameRecord { + tenant: tenant.clone(), + name: b"expected".to_vec(), + bucket_id: session().bucket_id, + tombstone: false, + }; + let mut items = vec![ + RpcValue { + key: MetadataKey::bucket_name(tenant, &mapping.name).unwrap(), + value: mapping.encode(), + revision: 1, + }, + RpcValue { + key: extra_key, + value: extra_value, + revision: 2, + }, + ]; + items.sort_by(|a, b| a.key.cmp(&b.key)); + let client = Arc::new(bucket_client::client(Arc::new(TestTransport(items)))); + client.refresh_catalog().await.unwrap(); + ChunkKvMetadataStore::new(client) +} + +pub(super) async fn paginated( + store: &ChunkKvMetadataStore, + tenant: &TenantId, +) -> std::result::Result>, BucketError> { + BucketListSigner::new(b"secret".to_vec())? + .list( + store, + tenant, + &BucketListRequest { + prefix: b"", + continuation: None, + max_buckets: 100, + max_scan_items: 100, + max_scan_bytes: 65536, + now: 150, + ttl: 30, + }, + ) + .await + .map(|page| page.buckets.into_iter().map(|bucket| bucket.name).collect()) +} diff --git a/lib/crowdb-chunk-stream/tests/lease_fenced_release_test.rs b/lib/crowdb-chunk-stream/tests/lease_fenced_release_test.rs new file mode 100644 index 000000000..8d3464442 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/lease_fenced_release_test.rs @@ -0,0 +1,222 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, Result, StreamConfig, StreamError, StreamName, + StreamOwnerAuthority, StreamPurpose, StreamRecord, StreamRecordState, StreamRegistry, + StreamWriterSession, +}; +use std::{ + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; + +const HOUR: Duration = Duration::from_secs(3600); + +struct TestAuthority { + deadline: Instant, + fenced: AtomicBool, + releases: AtomicUsize, +} + +#[async_trait] +impl StreamOwnerAuthority for TestAuthority { + fn deadline(&self) -> Result { + if self.fenced.load(Ordering::Acquire) { + Err(StreamError::StaleWriter) + } else { + Ok(self.deadline) + } + } + + async fn release(&self) -> Result<()> { + self.releases.fetch_add(1, Ordering::AcqRel); + Ok(()) + } +} + +struct TestRegistry { + store: Arc, + session: Arc, +} + +#[async_trait] +impl StreamRegistry for TestRegistry { + async fn load(&self, name: StreamName) -> Result> { + self.store.load(name).await + } + + async fn create(&self, record: StreamRecord) -> Result<()> { + self.store.create(record).await + } + + async fn claim_writer(&self, _name: StreamName, _epoch: u64) -> Result> { + Ok(self.session.clone()) + } +} + +struct TestIdleWriter { + stream: ChunkStream, + store: Arc, + authority: Arc, + session: Arc, +} + +impl TestIdleWriter { + async fn create(low: u64) -> Self { + // This fixed std::Instant deadline does not advance with Tokio's + // paused clock. Revocation below is explicit, rather than a TTL race. + let authority = Arc::new(TestAuthority { + deadline: Instant::now() + Duration::from_secs(24 * 3600), + fenced: AtomicBool::new(false), + releases: AtomicUsize::new(0), + }); + let session = StreamWriterSession::new(authority.clone()); + // An empty 8 MiB chunk exceeds the initial lookahead, so no successor + // preparation can contribute a liveness attempt or release error. + let store = Arc::new(MemoryStreamStore::new(8 * 1024 * 1024)); + let registry = Arc::new(TestRegistry { + store: store.clone(), + session: session.clone(), + }); + let stream = ChunkStream::create( + StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(StreamName { high: 748, low }, None, StreamPurpose::Wal) + }, + 1, + StreamConfig { + liveness_interval: HOUR, + ..StreamConfig::default() + }, + registry, + store.clone(), + store.clone(), + ) + .await + .unwrap(); + for _ in 0..10 { + tokio::task::yield_now().await; + } + assert_eq!( + store.chunks_for_tests().await.len(), + 1, + "no successor may affect the renewal count" + ); + Self { + stream, + store, + authority, + session, + } + } + + fn assert_no_payload_or_release(&self) { + assert_eq!(self.stream.tail(), 0); + assert_eq!(self.stream.metrics().completed, 0); + assert_eq!(self.store.chunk_write_count(), 0); + assert_eq!(self.store.cursor_advance_count(), 0); + assert_eq!(self.authority.releases.load(Ordering::Acquire), 0); + assert!(!self.session.is_released()); + } +} + +async fn enter_failed_liveness_maintenance(fixture: &TestIdleWriter) { + fixture.store.fail_next_renewals(usize::MAX); + tokio::time::advance(HOUR).await; + wait_for_attempts(&fixture.store, 1).await; + tokio::time::advance(Duration::from_millis(100)).await; + wait_for_attempts(&fixture.store, 2).await; + tokio::time::advance(Duration::from_millis(100)).await; + wait_for_attempts(&fixture.store, 4).await; + assert_eq!(fixture.store.liveness_attempt_count_for_tests(), 4); + fixture.assert_no_payload_or_release(); +} + +async fn wait_for_attempts(store: &MemoryStreamStore, attempts: u64) { + // Runnable bounded yields prevent paused Tokio time from auto-advancing + // to the next hourly timer while waiting for a worker transition. + for _ in 0..1000 { + if store.liveness_attempt_count_for_tests() >= attempts { + return; + } + tokio::task::yield_now().await; + } + panic!("liveness worker did not reach {attempts} attempts"); +} + +#[tokio::test(start_paused = true)] +async fn healthy_idle_writer_releases_without_appending() { + let fixture = TestIdleWriter::create(1).await; + fixture.assert_no_payload_or_release(); + fixture.stream.release_writer().await.unwrap(); + assert_eq!(fixture.authority.releases.load(Ordering::Acquire), 1); + assert!(fixture.session.is_released()); + assert_eq!(fixture.store.liveness_attempt_count_for_tests(), 0); + assert_eq!(fixture.store.chunk_write_count(), 0); +} + +#[tokio::test(start_paused = true)] +async fn failed_idle_liveness_then_revocation_stalls_release_before_owner_release() { + let fixture = TestIdleWriter::create(2).await; + let original_deadline = fixture.authority.deadline; + enter_failed_liveness_maintenance(&fixture).await; + // The renewal task performs only three attempts before notifying the + // worker. With no successor or next hourly tick, attempt four proves the + // real worker is in maintain(), sleeping after its first failed renewal. + assert_eq!(fixture.store.liveness_attempt_count_for_tests(), 4); + fixture.assert_no_payload_or_release(); + fixture.authority.fenced.store(true, Ordering::Release); + let stream = fixture.stream.clone(); + let release = tokio::spawn(async move { stream.release_writer().await }); + tokio::task::yield_now().await; + assert!(!release.is_finished(), "maintain must still own the worker"); + tokio::time::advance(Duration::from_millis(100)).await; + for _ in 0..1000 { + if release.is_finished() { + break; + } + tokio::task::yield_now().await; + } + assert!( + release.is_finished(), + "release command did not complete after fencing" + ); + assert_eq!(release.await.unwrap(), Err(StreamError::WriteStalled)); + fixture.assert_no_payload_or_release(); + assert_eq!(fixture.authority.deadline, original_deadline); + assert_eq!(fixture.store.liveness_attempt_count_for_tests(), 4); +} + +#[tokio::test(start_paused = true)] +async fn transient_idle_liveness_failure_recovers_before_release_with_live_authority() { + let fixture = TestIdleWriter::create(3).await; + enter_failed_liveness_maintenance(&fixture).await; + fixture.store.fail_next_renewals(0); + let stream = fixture.stream.clone(); + let release = tokio::spawn(async move { stream.release_writer().await }); + tokio::task::yield_now().await; + assert!(!release.is_finished(), "maintain must still own the worker"); + tokio::time::advance(Duration::from_millis(100)).await; + for _ in 0..1000 { + if release.is_finished() { + break; + } + tokio::task::yield_now().await; + } + assert!( + release.is_finished(), + "release did not complete after liveness recovered" + ); + release.await.unwrap().unwrap(); + assert_eq!(fixture.store.liveness_attempt_count_for_tests(), 5); + assert_eq!(fixture.authority.releases.load(Ordering::Acquire), 1); + assert!(fixture.session.is_released()); + assert!(!fixture.authority.fenced.load(Ordering::Acquire)); + assert_eq!(fixture.store.chunk_write_count(), 0); + assert_eq!(fixture.store.cursor_advance_count(), 0); +} diff --git a/lib/crowdb-kv-client/src/binding/chunk_slots/publication/epochs.rs b/lib/crowdb-kv-client/src/binding/chunk_slots/publication/epochs.rs index 5b2371095..d5f0b475b 100644 --- a/lib/crowdb-kv-client/src/binding/chunk_slots/publication/epochs.rs +++ b/lib/crowdb-kv-client/src/binding/chunk_slots/publication/epochs.rs @@ -44,7 +44,9 @@ impl ChunkSlotMapClient { /// Advance this instance's owned slot epochs once before a restarted process serves. /// /// # Errors - /// Returns a publication conflict or a missing/corrupt dynamic assignment. + /// Recomputes the current assignment after `CasBusy` or `CasFailed`, up to + /// 32 attempts. Other errors, including an unreconciled `OutcomeUnknown`, + /// stop immediately. Exhausted contention returns `CasBusy` to the caller. pub async fn regrant_service_epochs(&self, instance_id: u64) -> Result<()> { for _ in 0..32 { match self.regrant_service_epochs_once(instance_id).await { @@ -74,13 +76,29 @@ impl ChunkSlotMapClient { /// No old-write draining or data-group network checks participate in publication. /// /// # Errors - /// Rejects stale publication, epoch rollback/reuse and ambiguous conflicting outcomes. + /// Returns `CasBusy` when same-head contention cannot yet be reconciled, + /// `CasFailed` for a stale source, and `OutcomeUnknown` for an unconfirmed + /// submitted outcome. Callers must reconcile the complete map before + /// deciding whether to resubmit the same intent; this method never rebases + /// a fixed publication or silently retries its mutation. Epoch rollback + /// and reuse are rejected. pub async fn publish_service_epochs(&self, next: &ChunkSlotMap) -> Result<()> { match self.publish_service_epochs_once(next).await { Err(error @ (Error::OutcomeUnknown | Error::CasBusy | Error::CasFailed { .. })) => { // A competing identical publisher can commit during preparation, // before this caller submits its CAS. Reconcile that boundary too. - let current = self.read_service_snapshot().await?; + let current = match self.read_service_snapshot().await { + Ok(current) => current, + Err(reconcile_error) if matches!(&error, Error::OutcomeUnknown) => { + tracing::warn!( + generation = next.head().generation, + %reconcile_error, + "epoch publication outcome remains unknown; reconcile the complete map before retrying" + ); + return Err(error); + } + Err(reconcile_error) => return Err(reconcile_error), + }; if current.authority().head() == next.head() && current .authority() diff --git a/lib/crowdb-kv-client/tests/chunk_slot_publication_test.rs b/lib/crowdb-kv-client/tests/chunk_slot_publication_test.rs new file mode 100644 index 000000000..3fdd5b416 --- /dev/null +++ b/lib/crowdb-kv-client/tests/chunk_slot_publication_test.rs @@ -0,0 +1,147 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/chunk_slot_publication.rs"] +mod fixture; +#[path = "common/publication_peer.rs"] +mod peer; + +use crowdb_kv::cluster::group::PxGroup; +use crowdb_kv_client::{ChunkServiceSnapshot, ChunkSlotMapClient, ClientConfig, CrowdbKvClient, Error}; +use crowdb_protocol::{ + chunk_slot::{ChunkSlot, ChunkSlotAuthority, ChunkSlotMap}, + key::{ChunkSlotMapHeadKey, TextKey}, +}; +use fixture::TestChunkSlotPublication; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; + +#[tokio::test] +async fn ambiguous_publication_does_not_rebase_after_busy_reconciliation() { + let peer = peer::TestPublicationPeer::start().await; + let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![]))); + kv.seed_leader(0, 0, peer.endpoint.clone()); + let maps = ChunkSlotMapClient::new(kv); + let before = maps.read_service_snapshot().await.unwrap(); + let slots: Vec<_> = ChunkSlot::all() + .filter(|slot| before.service().owner(*slot) == 1) + .map(|slot| (slot, 1)) + .collect(); + let expected = before.authority().reassign(&slots).unwrap(); + let result = maps.regrant_service_epochs(1).await; + assert!( + matches!(result, Err(Error::OutcomeUnknown)), + "ambiguous publication must not become retryable: {result:?}; submissions={}; generation={}", + peer.submissions.load(Ordering::Acquire), + peer.committed_generation.load(Ordering::Acquire) + ); + assert_eq!(peer.submissions.load(Ordering::Acquire), 1); + assert_complete(&maps.read_service_snapshot().await.unwrap(), &expected); +} + +async fn wait_for_head_admission(group: &PxGroup) { + let key = ChunkSlotMapHeadKey::Service.to_path(); + tokio::time::timeout(Duration::from_secs(1), async { + while !group.conditional_write_pending_for_tests(key.as_bytes()) { + tokio::task::yield_now().await; + } + }) + .await + .expect("publication must reach the held head admission"); +} + +fn assert_complete(actual: &ChunkServiceSnapshot, expected: &ChunkSlotMap) { + assert_eq!(actual.authority().head(), expected.head()); + assert_eq!(actual.service().head().generation, expected.head().generation); + for slot in ChunkSlot::all() { + assert_eq!(actual.authority().owner(slot), expected.owner(slot)); + assert_eq!(actual.service().owner(slot), expected.owner(slot).instance_id()); + } +} + +#[tokio::test] +async fn fixed_intent_caller_observes_busy_then_reconciles_before_explicit_republication() { + let test = TestChunkSlotPublication::start().await; + let maps = ChunkSlotMapClient::new(Arc::clone(&test.kv)); + let before = maps.read_service_snapshot().await.unwrap(); + let storage = maps.read_storage().await.unwrap(); + let revision = test.head_revision().await; + let next = before + .authority() + .reassign(&[(ChunkSlot::try_from(0).unwrap(), 3)]) + .unwrap(); + let group = test.store.get_group(0).unwrap(); + let permits = group.try_acquire_all_inflight_permits(); + let kv = Arc::clone(&test.kv); + let pending = next.clone(); + let first = + tokio::spawn(async move { ChunkSlotMapClient::new(kv).publish_service_epochs(&pending).await }); + wait_for_head_admission(&group).await; + + assert!(matches!( + maps.publish_service_epochs(&next).await, + Err(Error::CasBusy) + )); + assert!(!first.is_finished()); + assert_complete(&maps.read_service_snapshot().await.unwrap(), before.authority()); + assert_eq!(test.head_revision().await, revision); + assert_eq!(maps.read_storage().await.unwrap().bindings(), storage.bindings()); + + drop(permits); + first.await.unwrap().unwrap(); + let committed = maps.read_service_snapshot().await.unwrap(); + assert_complete(&committed, &next); + assert_eq!(maps.read_storage().await.unwrap().bindings(), storage.bindings()); + let committed_revision = test.head_revision().await; + + // The caller now has a complete committed witness for its original intent. + maps.publish_service_epochs(&next).await.unwrap(); + assert_complete(&maps.read_service_snapshot().await.unwrap(), &next); + assert_eq!(test.head_revision().await, committed_revision); +} + +#[tokio::test] +async fn restart_regrant_exhausts_busy_policy_then_recomputes_only_current_owned_slots() { + let test = TestChunkSlotPublication::start().await; + let maps = ChunkSlotMapClient::new(Arc::clone(&test.kv)); + let before = maps.read_service_snapshot().await.unwrap(); + let storage = maps.read_storage().await.unwrap(); + let revision = test.head_revision().await; + let next = before + .authority() + .reassign(&[(ChunkSlot::try_from(0).unwrap(), 3)]) + .unwrap(); + let group = test.store.get_group(0).unwrap(); + let permits = group.try_acquire_all_inflight_permits(); + let kv = Arc::clone(&test.kv); + let pending = next.clone(); + let first = + tokio::spawn(async move { ChunkSlotMapClient::new(kv).publish_service_epochs(&pending).await }); + wait_for_head_admission(&group).await; + + assert!(matches!( + maps.regrant_service_epochs(2).await, + Err(Error::CasBusy) + )); + assert!(!first.is_finished()); + assert_complete(&maps.read_service_snapshot().await.unwrap(), before.authority()); + assert_eq!(test.head_revision().await, revision); + assert_eq!(maps.read_storage().await.unwrap().bindings(), storage.bindings()); + + drop(permits); + first.await.unwrap().unwrap(); + assert_complete(&maps.read_service_snapshot().await.unwrap(), &next); + assert_eq!(maps.read_storage().await.unwrap().bindings(), storage.bindings()); + let slots: Vec<_> = ChunkSlot::all() + .filter(|slot| next.owner(*slot).instance_id() == 2) + .map(|slot| (slot, 2)) + .collect(); + let expected = next.reassign(&slots).unwrap(); + + // Restart intent is instance-scoped; it deliberately uses the current map. + maps.regrant_service_epochs(2).await.unwrap(); + assert_complete(&maps.read_service_snapshot().await.unwrap(), &expected); + assert_eq!(maps.read_storage().await.unwrap().bindings(), storage.bindings()); +} diff --git a/lib/crowdb-kv-client/tests/common/chunk_slot_publication.rs b/lib/crowdb-kv-client/tests/common/chunk_slot_publication.rs new file mode 100644 index 000000000..d16bed7d0 --- /dev/null +++ b/lib/crowdb-kv-client/tests/common/chunk_slot_publication.rs @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_kv::cluster::{ + group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +}; +use crowdb_kv_client::{ChunkSlotMapClient, ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_protocol::{ + chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}, + key::{ChunkSlotMapHeadKey, TextKey}, +}; + +pub struct TestChunkSlotPublication { + pub store: Arc, + pub kv: Arc, +} + +impl TestChunkSlotPublication { + pub async fn start() -> Self { + let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); + for group in [0, 1] { + store.add_group(PxGroup::new( + group, + PxLocalReplica::new(1, PxLocalReplicaRole::Leader), + )); + } + store.start().await.unwrap(); + let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![]))); + for group in [0, 1] { + kv.seed_leader(0, group, store.listen_addr().unwrap().to_string()); + } + let maps = ChunkSlotMapClient::new(Arc::clone(&kv)); + maps.initialize_layout(&ChunkSlotBootstrap { + service_instances: vec![1, 2], + storage_groups: vec![ChunkStorageGroup { + store_id: 0, + group_id: 1, + }], + }) + .await + .unwrap(); + maps.initialize_service_epochs().await.unwrap(); + Self { store, kv } + } + + pub async fn head_revision(&self) -> u64 { + let key = ChunkSlotMapHeadKey::Service.to_path(); + match self + .kv + .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) + .await + .unwrap() + { + GetOutcome::Found { revision, .. } => revision, + GetOutcome::NotFound => panic!("service head disappeared"), + } + } +} + +impl Drop for TestChunkSlotPublication { + fn drop(&mut self) { + self.store.stop(); + } +} diff --git a/lib/crowdb-kv-client/tests/common/publication_peer.rs b/lib/crowdb-kv-client/tests/common/publication_peer.rs new file mode 100644 index 000000000..a099c8c96 --- /dev/null +++ b/lib/crowdb-kv-client/tests/common/publication_peer.rs @@ -0,0 +1,360 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::{ + chunk_slot::{ + ChunkServiceIncarnation, ChunkSlotAuthority, ChunkSlotBinding, ChunkSlotBootstrap, ChunkStorageGroup, + }, + fb::FBMsgType, + key::{ + ChunkServiceAuthorityKey, ChunkServiceSlotsKey, ChunkSlotMapHeadKey, ChunkStorageSlotsKey, TextKey, + }, + kv_client_fb::{ + FBKvBatchWriteRequest, FBKvClientRetCode, FBKvGetRequest, FBKvResponse, FBKvResponseArgs, + FBKvScanItem, FBKvScanItemArgs, FBKvScanRequest, FBKvScanResponse, FBKvScanResponseArgs, + }, +}; +use flatbuffers::FlatBufferBuilder; +use std::{ + collections::BTreeMap, + sync::{ + atomic::{AtomicU64, AtomicUsize, Ordering}, + Arc, + }, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + sync::{mpsc, oneshot}, + task::{JoinHandle, JoinSet}, +}; + +pub struct TestPublicationPeer { + pub endpoint: String, + pub submissions: Arc, + pub committed_generation: Arc, + worker: JoinHandle<()>, +} + +struct Request { + kind: u16, + control: Vec, + response: oneshot::Sender>, +} + +struct Records { + values: BTreeMap, (u64, Vec)>, + revision: u64, + moving_head_reads: usize, + submissions: Arc, + committed_generation: Arc, +} + +impl TestPublicationPeer { + pub async fn start() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = listener.local_addr().unwrap().to_string(); + let submissions = Arc::new(AtomicUsize::new(0)); + let committed_generation = Arc::new(AtomicU64::new(2)); + let mut records = Records::new(submissions.clone(), committed_generation.clone()); + let (tx, mut rx) = mpsc::channel::(16); + let worker = tokio::spawn(async move { + let mut connections = JoinSet::new(); + loop { + tokio::select! { + accepted=listener.accept()=>{ + let (stream,_)=accepted.unwrap(); + connections.spawn(serve(stream,tx.clone())); + } + Some(request)=rx.recv()=>{ + let response=records.reply(request.kind,&request.control); + let _=request.response.send(response); + } + completed=connections.join_next(),if !connections.is_empty()=>{ + completed.unwrap().unwrap(); + } + } + } + }); + Self { + endpoint, + submissions, + committed_generation, + worker, + } + } +} + +impl Drop for TestPublicationPeer { + fn drop(&mut self) { + self.worker.abort(); + } +} + +impl Records { + fn new(submissions: Arc, committed_generation: Arc) -> Self { + let bootstrap = ChunkSlotBootstrap { + service_instances: vec![1, 2], + storage_groups: vec![ChunkStorageGroup { + store_id: 0, + group_id: 1, + }], + }; + let service = bootstrap.service_map().unwrap(); + let storage = bootstrap.storage_map().unwrap(); + let mut values = BTreeMap::new(); + let mut head = service.head().clone(); + head.generation = 2; + for binding in service.bindings() { + let mut binding = binding.clone(); + binding.generation = 2; + values.insert( + ChunkServiceSlotsKey { + instance_id: binding.owner, + } + .to_path() + .into_bytes(), + (100, serde_json::to_vec(&binding).unwrap()), + ); + let authority = ChunkSlotAuthority::new( + binding.owner, + ChunkServiceIncarnation::try_from([1; 16]).unwrap(), + 1, + ) + .unwrap(); + let binding = ChunkSlotBinding { + generation: 2, + owner: authority, + slots: binding.slots, + }; + values.insert( + ChunkServiceAuthorityKey { authority }.to_path().into_bytes(), + (100, serde_json::to_vec(&binding).unwrap()), + ); + } + for key in [ChunkSlotMapHeadKey::Service, ChunkSlotMapHeadKey::Authority] { + values.insert( + key.to_path().into_bytes(), + (100, serde_json::to_vec(&head).unwrap()), + ); + } + for binding in storage.bindings() { + values.insert( + ChunkStorageSlotsKey { + store_id: binding.owner.store_id, + group_id: binding.owner.group_id, + } + .to_path() + .into_bytes(), + (100, serde_json::to_vec(binding).unwrap()), + ); + } + values.insert( + ChunkSlotMapHeadKey::Storage.to_path().into_bytes(), + (100, serde_json::to_vec(storage.head()).unwrap()), + ); + Self { + values, + revision: 100, + moving_head_reads: 0, + submissions, + committed_generation, + } + } + + fn reply(&mut self, kind: u16, control: &[u8]) -> Vec { + if kind == FBMsgType::EKvBatchWriteRequest.0 as u16 { + return self.batch(control); + } + if kind == FBMsgType::EKvGetRequest.0 as u16 { + let request = flatbuffers::root::(control).unwrap(); + assert_eq!(request.group_id(), 0); + let key = request.key().unwrap().bytes(); + if key == ChunkSlotMapHeadKey::Service.to_path().as_bytes() && self.moving_head_reads > 0 { + // A competing cohort reservation can rewrite an unchanged service head. + self.moving_head_reads -= 1; + self.revision += 1; + self.values.get_mut(key).unwrap().0 = self.revision; + } + let found = self.values.get(key); + return point( + request.id(), + true, + self.revision, + found, + FBKvClientRetCode::Success, + ); + } + assert_eq!(kind, FBMsgType::EKvScanRequest.0 as u16); + let request = flatbuffers::root::(control).unwrap(); + assert_eq!(request.group_id(), 0); + let cutoff = if request.scan_cutoff() == 0 { + self.revision + } else { + request.scan_cutoff() + }; + let prefix = request.prefix().unwrap().bytes(); + let after = request.start_after().map_or(&[][..], |key| key.bytes()); + let rows: Vec<_> = self + .values + .iter() + .filter(|(key, (revision, _))| { + key.starts_with(prefix) && key.as_slice() > after && *revision <= cutoff + }) + .collect(); + let mut builder = FlatBufferBuilder::new(); + let items: Vec<_> = rows + .into_iter() + .map(|(key, (revision, value))| { + let key = builder.create_vector(key); + let value = builder.create_vector(value); + FBKvScanItem::create( + &mut builder, + &FBKvScanItemArgs { + key: Some(key), + value: Some(value), + commit_slot: *revision, + }, + ) + }) + .collect(); + let items = builder.create_vector(&items); + let response = FBKvScanResponse::create( + &mut builder, + &FBKvScanResponseArgs { + id: request.id(), + version: 1, + ok: true, + items: Some(items), + read_slot: self.revision, + scan_cutoff: cutoff, + ..Default::default() + }, + ); + builder.finish(response, None); + frame(FBMsgType::EKvScanResponse.0 as u16, builder.finished_data()) + } + + fn batch(&mut self, control: &[u8]) -> Vec { + let request = flatbuffers::root::(control).unwrap(); + assert_eq!(request.group_id(), 0); + let precondition = request.precondition().unwrap(); + let key = precondition.key().unwrap().bytes(); + assert_eq!(key, ChunkSlotMapHeadKey::Service.to_path().as_bytes()); + assert_eq!(precondition.expected_revision(), self.values[key].0); + self.revision += 1; + for item in request.items().unwrap() { + let key = item.key().unwrap().bytes().to_vec(); + if item.is_delete() { + self.values.remove(&key); + } else { + self.values + .insert(key, (self.revision, item.value().unwrap().bytes().to_vec())); + } + } + let first = self.submissions.fetch_add(1, Ordering::AcqRel) == 0; + let head = &self.values[ChunkSlotMapHeadKey::Service.to_path().as_bytes()].1; + self.committed_generation.store( + serde_json::from_slice::(head).unwrap()["generation"] + .as_u64() + .unwrap(), + Ordering::Release, + ); + if first { + // The mutation committed, but its caller received no confirmed outcome. + // Nine head changes exhaust read_service's four consistent-read attempts. + self.moving_head_reads = 9; + } + point( + request.id(), + !first, + self.revision, + None, + if first { + FBKvClientRetCode::OutcomeUnknown + } else { + FBKvClientRetCode::Success + }, + ) + } +} + +fn point( + id: u64, + ok: bool, + read_slot: u64, + found: Option<&(u64, Vec)>, + code: FBKvClientRetCode, +) -> Vec { + let mut builder = FlatBufferBuilder::new(); + let value = found.map(|(_, value)| builder.create_vector(value)); + let response = FBKvResponse::create( + &mut builder, + &FBKvResponseArgs { + id, + version: 1, + ok, + revision: found.map_or(read_slot, |(revision, _)| *revision), + value, + not_found: ok && found.is_none(), + ret_code: code, + read_slot, + ..Default::default() + }, + ); + builder.finish(response, None); + frame(FBMsgType::EKvResponse.0 as u16, builder.finished_data()) +} + +fn frame(kind: u16, control: &[u8]) -> Vec { + let mut bytes = Vec::with_capacity(control.len() + 14); + bytes.extend_from_slice(&0xca70_u16.to_le_bytes()); + bytes.extend_from_slice(&kind.to_le_bytes()); + bytes.extend_from_slice(&u32::try_from(control.len()).unwrap().to_le_bytes()); + bytes.extend_from_slice(&0_u32.to_le_bytes()); + bytes.extend_from_slice(&[14, 0]); + bytes.extend_from_slice(control); + bytes +} + +async fn serve(mut stream: TcpStream, tx: mpsc::Sender) { + loop { + let mut header = [0; 14]; + if stream.read_exact(&mut header).await.is_err() { + return; + } + assert_eq!(u16::from_le_bytes(header[..2].try_into().unwrap()), 0xca70); + assert_eq!(&header[8..12], &[0; 4]); + let kind = u16::from_le_bytes(header[2..4].try_into().unwrap()); + let length = u32::from_le_bytes(header[4..8].try_into().unwrap()) as usize; + assert!(length <= 65536); + let mut control = vec![0; length]; + stream.read_exact(&mut control).await.unwrap(); + let reply = if kind == 0xfffe { + let mut capabilities = vec![0; 160]; + capabilities[..4].copy_from_slice(&[0x44, 0x52, 4, 0]); + capabilities[4..12].copy_from_slice(&1_u64.to_le_bytes()); + frame(kind, &capabilities) + } else { + let (response, rx) = oneshot::channel(); + if tx + .send(Request { + kind, + control, + response, + }) + .await + .is_err() + { + return; + } + let Ok(response) = rx.await else { + return; + }; + response + }; + if stream.write_all(&reply).await.is_err() { + return; + } + } +} From 15945165911410c369b6211e007122d06a1d028e Mon Sep 17 00:00:00 2001 From: Gian Date: Wed, 4 Nov 2026 20:25:40 +0800 Subject: [PATCH 22/24] Fix CI disk exhaustion and lifecycle test races --- .github/actions/compilation-cache/action.yml | 28 +-- .github/workflows/ci.yml | 11 ++ doc/working/plan-ci-stability-review.md | 48 +++--- .../e2e/flows/13-node-service-plans.spec.ts | 14 +- .../e2e/flows/55-chunk-kv-catalog.spec.ts | 10 +- .../e2e/flows/92-three-node-data.spec.ts | 18 +- e2e-test/scenarios.py | 2 +- e2e-test/tests/console/native_browser.rs | 7 +- e2e-test/tests/kv/reconfiguration.rs | 161 +++++++++++++++--- e2e-test/tests/test_dispatch_test.py | 21 ++- 10 files changed, 251 insertions(+), 69 deletions(-) diff --git a/.github/actions/compilation-cache/action.yml b/.github/actions/compilation-cache/action.yml index 37cff2bba..0b91fd841 100644 --- a/.github/actions/compilation-cache/action.yml +++ b/.github/actions/compilation-cache/action.yml @@ -5,23 +5,31 @@ inputs: description: Compilation owner whose cache should be reused required: false default: ${{ github.job }} + family: + description: Compilation cache family, including fallback isolation + required: false + default: release-ci-v1 + directories: + description: Compilation output directories owned by this job + required: false + default: | + target/debug + target/release + sdk/target/release + sdk/target/sdist/release + target/iceberg-sdk/release + app/crowdb-diskio/build + lib/crowdb-tree/build + lib/crowdb-rpc/build runs: using: composite steps: - uses: Swatinem/rust-cache@v2 with: - shared-key: release-ci-v1-${{ hashFiles('pixi.lock') }} + shared-key: ${{ inputs.family }}-${{ hashFiles('pixi.lock') }} key: ${{ inputs.key }} cache-targets: false - cache-directories: | - target/debug - target/release - sdk/target/release - sdk/target/sdist/release - target/iceberg-sdk/release - app/crowdb-diskio/build - lib/crowdb-tree/build - lib/crowdb-rpc/build + cache-directories: ${{ inputs.directories }} cmd-format: "pixi run {0}" cache-workspace-crates: true env-vars: "CARGO CC CXX CFLAGS CXXFLAGS CMAKE RUST CROWDB_KEEP_DEBUG_INFO" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e78d2ec85..2eb3cb47d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -208,6 +208,11 @@ jobs: AccessTests: timeout-minutes: 90 runs-on: ubuntu-24.04 + env: + # Keep function symbols and panic locations without duplicating native + # DWARF across hundreds of linked test executables. + CARGO_PROFILE_TEST_STRIP: debuginfo + CARGO_BUILD_JOBS: "4" steps: - uses: actions/checkout@v4 with: @@ -223,6 +228,12 @@ jobs: with: pixi-version: v0.77.0 - uses: ./.github/actions/compilation-cache + with: + family: access-ci-v2 + directories: | + target/debug + sdk/target/release + sdk/target/sdist/release - name: Clean env run: pixi run clean --runtime-only - name: Run access component tests diff --git a/doc/working/plan-ci-stability-review.md b/doc/working/plan-ci-stability-review.md index 78de0755f..77b0b6e92 100644 --- a/doc/working/plan-ci-stability-review.md +++ b/doc/working/plan-ci-stability-review.md @@ -3,40 +3,30 @@ # CI Stability Review Plan -The three requested CI repairs and local verification finished on 2026-11-04. -The user authorized pushing the completed changes on 2026-11-04. +Run: https://github.com/buzzcrow/crowdb/actions/runs/41816831055 -- ContainerKV: 50 fixtures and 58 client phases pass, including two-slot isolation. - Explicit CAS conflicts remain visible; an unknown publication outcome can no - longer become a retryable conflict through a failed reconciliation read. -- Deployment services: 33 fixtures and 41 client phases pass. Strict bucket-row - classification fixes the reproduced multipart metadata collision. The original - restart HTTP 500 lacks its offending row, so historical attribution stays qualified. -- Console native-data: the original ordered 14-case baseline passes. The fixed - image passes the original browser case and all nine affected node cases; retain - five Dataset baseline passes whose relevant inputs did not change. This is - incremental coverage, not a claim that all 14 ran on the fixed image. - Complete plans keep observing current processes. All 381 UI tests and types pass. +All four failed jobs were repaired and locally verified on 2026-11-04: +AccessTests completes every original stage; ContainerKV passes 50 fixtures and +58 client phases with isolation; Console native operations passes 19 owners; +Console native data passes all 14 original owners. Rust fmt, all-target clippy, +314 fixture controls and CI task coverage pass. Original failures remain in the +evidence; this records local verification, not a later remote CI result. + +As requested, this persistent plan retains only unfinished follow-up work. +Completed CI execution and review tasks have been removed. CI evidence from repository root: -`.crowdb-runtime/artifacts/ci/new-failures-20261104/local-verification/`. -`console-fixed-coverage.json` records exact case membership and image qualification. -Original failures and repair RED/GREEN evidence remain in the three job directories. +`.crowdb-runtime/artifacts/ci/run-41816831055/`. ## Remaining task: deferred historical attribution -- [ ] **P13 first aggregate WAL release error**: Identify its exact historical - partition/phase and durable release outcome if further attribution is required. - Six failing streams map to exact partitions, but the saved aggregate error does - not identify the first branch. A real-worker reproduction establishes that - idle liveness failure followed by authority revocation returns `WriteStalled` - before owner release with zero payload IO; healthy idle release and transient - recovery both pass. This warning alone does not establish an unfinished payload - write or corruption. No production fencing change is justified by that evidence. - The next investigation needs typed partition/stream/epoch/lifecycle/release-phase - and first stalled cause in a local server reproduction. Preserve lease - exclusion and replay. The three controls, fmt and focused clippy pass. +- [ ] **P13 first aggregate WAL release error (deferred)**: identify its exact + historical partition/phase and durable release outcome if further attribution + is required. Saved logs map six failing streams but do not identify the first + aggregate branch. A local server reproduction needs typed partition, stream, + epoch, lifecycle, release phase and first stalled cause. Preserve lease + exclusion and replay. Existing real-worker controls pass and establish + `WriteStalled` before release with zero payload IO after authority revocation; + they do not justify a production fencing change. Evidence from repository root: `.crowdb-runtime/artifacts/physical-nodes/physical-20261031T194149Z/repair/readiness-probe-budget-2s/wal-drain-research-20261104/findings.md`. - -Completed execution, probe review and resource-closure tasks have been removed. diff --git a/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts b/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts index 86b9fc18b..20864a7b2 100644 --- a/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts @@ -3,7 +3,7 @@ import type { APIRequestContext } from '@playwright/test'; import { test, expect } from '../fixtures/realBackend'; import { diskMenu } from '../fixtures/capacity'; -import { nativeProcessAlive } from '../fixtures/nativeProcess'; +import { nativeProcessAlive, nativeServices } from '../fixtures/nativeProcess'; const kinds = ['paxos-kv', 'diskdb', 'chunkdb', 'diskio', 'chunk-kv', 'access-server']; async function json(request: APIRequestContext, path: string) { @@ -70,6 +70,18 @@ test('native plans: fixed ports isolate three nodes and failed DiskDB retries pr const retained = await json(request, '/api/service-plans'); expect(retained[1].overrides).toEqual(before[1].overrides); expect(retained[1].steps.diskdb).toEqual(before[1].steps.diskdb); + // Keep the failed registration for explicit Retry, but cancel its running + // intent before freeing the port so monitor recovery cannot win this phase. + const stopped = await request.post('/api/services/diskdb-1/stop'); + expect(stopped.ok(), await stopped.text()).toBe(true); + expect(await stopped.json()).toMatchObject({ id: 'diskdb-1', pid: null, removed: false }); + await expect.poll(async () => (await nativeServices(0))['diskdb-1']?.pid, { intervals: [100] }).toBeNull(); + const registered = (await json(request, '/api/servers')).filter((server: { id: string }) => server.id === 'diskdb-1'); + expect(registered).toHaveLength(1); + expect(registered[0]).toMatchObject({ id: 'diskdb-1', node_id: 1, service_type: 'diskdb' }); + expect(registered[0].pid).toBeFalsy(); + expect((await json(request, '/api/service-plans'))[1].steps.diskdb.state).toBe('failed'); + expect(await nativeProcessAlive(0, guard.pid)).toBe(true); const removed = await request.delete(`/api/services/${guard.id}`); expect(removed.ok(), await removed.text()).toBe(true); await expect.poll(() => nativeProcessAlive(0, guard.pid), { intervals: [100] }).toBe(false); diff --git a/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts b/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts index 14437a2f0..1769b114b 100644 --- a/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts @@ -188,11 +188,15 @@ async function refreshSelectedJournal(page: Page, target: JournalRuntimeTarget, const sameSelection = await properties.count() === 1 && (await property('Split ID')) === target.id && (await property('Owner epoch')) === target.epoch && (await property('Generation')) === target.generation; const select = () => page.getByLabel('Partition range map').getByRole('button', { name: `Partition ${target.id}`, exact: true }).click({ timeout: remaining() }); - if (sameSelection) await select(); - else await step('journal automatic selection response', () => journalRuntimeResponse(page, target, select, remaining)); const refreshRuntime = page.getByRole('region', { name: 'Partition runtime', exact: true }).getByRole('button', { name: 'Refresh runtime', exact: true }); + if (!sameSelection) { + const observed = await step('journal automatic selection response', () => journalRuntimeResponse(page, target, select, remaining)); + // A changed identity starts its own observation. Preserve that response, + // including a catalog conflict, for the caller's coherence checks. + return { observed, refreshRuntime }; + } await expect(refreshRuntime).toBeEnabled({ timeout: remaining() }); - // Selecting the same partition leaves its identity unchanged. Explicitly + // The same partition identity does not trigger a new observation. Explicitly // refresh runtime so owner restart is observed even without a new catalog. const observed = await step('journal runtime DOM refresh', () => journalRuntimeResponse(page, target, () => refreshRuntime.click({ timeout: remaining() }), remaining)); diff --git a/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts b/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts index 9429fcc02..d1b0f0a4e 100644 --- a/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts @@ -12,6 +12,8 @@ import { disklessExpansion, verifyDisklessSlotBalance } from '../fixtures/diskle import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; import { resolve } from 'node:path'; +import { parseIcebergJson } from '../../src/iceberg/json'; +import type { TableLoad } from '../../src/iceberg/types'; import type { APIRequestContext, Page } from '@playwright/test'; @@ -222,7 +224,21 @@ async function icebergRoundTrip({ page, baseURL }: UIContext) { execution.child.stdin!.end(JSON.stringify(writerConfig)); const result = await execution; console.log(result.stdout); if (result.stderr) console.log(result.stderr); - await page.getByRole('button', { name: 'Refresh table', exact: true }).click(); + await step('three-node: refreshed Iceberg metadata reaches the DOM', async () => { + const refreshed = page.waitForResponse(response => response.request().method() === 'GET' + && new URL(response.url()).pathname === '/api/access/iceberg/v1/namespaces/ui_three_node/tables/events'); + const refresh = page.getByRole('button', { name: 'Refresh table', exact: true }); + await refresh.click(); + const response = await refreshed; + expect(response.status(), await response.text()).toBe(200); + const loaded = parseIcebergJson(await response.text()) as TableLoad; + const current = String(loaded.metadata['current-snapshot-id']); + const snapshot = loaded.metadata.snapshots?.find(snapshot => String(snapshot['snapshot-id']) === current); + expect(snapshot?.summary).toMatchObject({ 'total-records': '3' }); + await expect(page.getByRole('table', { name: 'Table snapshots', exact: true }).getByRole('row') + .filter({ has: page.getByRole('button', { name: current, exact: true }) })).toContainText('Current'); + await expect(refresh).toBeEnabled(); + }); await expect(page.getByRole('button', { name: 'Inspect current snapshot', exact: true })).toBeVisible(); await page.getByRole('button', { name: 'Inspect current snapshot', exact: true }).click(); await expect(page.getByRole('table', { name: 'Manifest list records' })).toContainText('avro'); diff --git a/e2e-test/scenarios.py b/e2e-test/scenarios.py index f772710c9..30699c20e 100644 --- a/e2e-test/scenarios.py +++ b/e2e-test/scenarios.py @@ -158,7 +158,7 @@ class Case: Case("kv.recovery", "kv::snapshot::snapshot_import_precedes_wiring_and_catches_up_tail", (0, 1, 2), groups_per_node=((1,), (1,), ())), Case("kv.membership", "kv::reconfiguration::quorum_expands_and_shrinks_without_losing_writes", - (0, 1, 2, 3, 4)), + (0, 1, 2, 3, 4), groups_per_node=((1,), (1,), (1,), (), ())), Case("kv.routing", "kv::cluster::topology_and_follower_hint", (10, 20, 30), True), Case("kv.routing", "kv::cluster::dynamic_groups_isolate_data", (0, 1, 2), True), Case("kv.management", "kv::management::management_lifecycle"), diff --git a/e2e-test/tests/console/native_browser.rs b/e2e-test/tests/console/native_browser.rs index 33d56af98..80b2a4c40 100644 --- a/e2e-test/tests/console/native_browser.rs +++ b/e2e-test/tests/console/native_browser.rs @@ -135,7 +135,12 @@ async fn browse_diskdb() { #[tokio::test] #[ignore = "requires normal storage and owned container PID observations"] async fn auxiliary_process_menus() { - browse_storage().await; + let context = TestNativeContext::connect().await; + super::native_storage::start(&context).await; + // Finish initial placement before testing one service outage at a time. + super::native_balance::settle_existing(&context).await; + let result = crate::common::control::native_browser().await; + assert_eq!(result["passed"], true); } #[tokio::test] diff --git a/e2e-test/tests/kv/reconfiguration.rs b/e2e-test/tests/kv/reconfiguration.rs index 34e3b07b2..847aeceae 100644 --- a/e2e-test/tests/kv/reconfiguration.rs +++ b/e2e-test/tests/kv/reconfiguration.rs @@ -3,7 +3,7 @@ use super::cluster::{leader, topology}; use crate::common::management::{addresses, client, origin, request}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, KvRpcTransport, ReadMode}; use reqwest::{Client, Method}; use serde_json::{json, Value}; use std::time::{Duration, Instant}; @@ -32,37 +32,145 @@ pub(super) async fn wire_initial(http: &Client, ips: &[String]) { } } -async fn expand(http: &Client, ips: &[String]) { - for (index, ip) in ips.iter().enumerate() { - let peers = if index < 3 { 3..5 } else { 0..5 }; - for peer in peers.filter(|peer| *peer != index) { - request( +async fn membership(http: &Client, ip: &str, method: Method, path: &str, body: Value, epoch: u64) { + let response = http + .request(method, format!("{}{path}", origin(ip))) + .header("x-crowdb-membership-epoch", epoch.to_string()) + .json(&body) + .send() + .await + .unwrap(); + let status = response.status(); + let text = response.text().await.unwrap(); + assert_eq!(status.as_u16(), 200, "{path}: {text}"); +} + +async fn applied(http: &Client, ip: &str) -> u64 { + topology(http, ip).await["stores"][0]["groups"][0]["read_state"]["contiguous_applied"] + .as_u64() + .expect("group applied frontier") +} + +async fn imported_value(ip: &str, required: u64) { + let response = KvRpcTransport::new() + .send_get( + &format!("http://{ip}:7001"), + b"k1", + 9201, + 9201, + 1, + crowdb_kv::rpc::ReadMode::MinSlot, + required, + ) + .await + .unwrap(); + assert!(response.ok, "snapshot local read failed: {}", response.error); + assert!(!response.not_found, "snapshot omitted acknowledged k1"); + assert_eq!(response.value.as_ref(), b"v1"); +} + +async fn add_member(http: &Client, ips: &[String], index: usize, epoch: u64) { + let source = leader(http, &ips[..index], 1).await; + let required = applied(http, &ips[source]).await; + request( + http, + &origin(&ips[index]), + Method::POST, + &format!("/stores/{index}/groups/1/join"), + json!({"replica_id": index + 1, "peer_endpoint": format!("{}:7001", ips[source])}), + 201, + ) + .await; + // Snapshot state must exist before this replica participates in any quorum. + assert!(applied(http, &ips[index]).await >= required); + imported_value(&ips[index], required).await; + for (peer, ip) in ips[..index].iter().enumerate() { + request( + http, + &origin(ip), + Method::POST, + &format!("/stores/{peer}/groups/1/remotes"), + json!([{"replica_id": index + 1, "endpoint": format!("{}:7001", ips[index]), "voting": false}]), + 200, + ) + .await; + } + let remotes: Vec<_> = ips[..index] + .iter() + .enumerate() + .map(|(peer, ip)| json!({"replica_id": peer + 1, "endpoint": format!("{ip}:7001"), "voting": true})) + .collect(); + membership( + http, + &ips[index], + Method::POST, + &format!("/stores/{index}/groups/1/remotes"), + json!(remotes), + epoch, + ) + .await; + let required = applied(http, &ips[source]).await; + let deadline = Instant::now() + Duration::from_secs(10); + while applied(http, &ips[index]).await < required { + assert!( + Instant::now() < deadline, + "joining member did not catch up before promotion" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + for (peer, ip) in ips[..index].iter().enumerate() { + membership( + http, + ip, + Method::POST, + &format!("/stores/{peer}/groups/1/remotes"), + json!([{"replica_id": index + 1, "endpoint": format!("{}:7001", ips[index]), "voting": true}]), + epoch, + ) + .await; + } + leader(http, &ips[..=index], 1).await; +} + +async fn shrink(http: &Client, ips: &[String], mut epoch: u64) { + for removed in 0..2 { + let current = leader(http, &ips[removed..], 1).await + removed; + if current == removed { + let stepped = request( http, - &origin(ip), + &origin(&ips[removed]), Method::POST, - &format!("/stores/{index}/groups/1/remotes"), - json!([{"replica_id": peer + 1, "endpoint": format!("{}:7001", ips[peer])}]), + &format!("/stores/{removed}/groups/1/step-down?sync=true"), + json!({"reason": "decommission"}), 200, ) .await; + assert_eq!(stepped["accepted"], true); } - } -} - -async fn shrink(http: &Client, ips: &[String]) { - for (index, ip) in ips.iter().enumerate() { - let removed = if index < 2 { 2..5 } else { 0..2 }; - for peer in removed { - request( + epoch += 1; + // Retire while its old epoch is still current. The remaining members + // retain the old quorum until the single-member removal reaches them. + membership( + http, + &ips[removed], + Method::DELETE, + &format!("/stores/{removed}/groups/1"), + Value::Null, + epoch, + ) + .await; + for (index, ip) in ips.iter().enumerate().skip(removed + 1) { + membership( http, - &origin(ip), + ip, Method::DELETE, - &format!("/stores/{index}/groups/1/remotes/{}", peer + 1), + &format!("/stores/{index}/groups/1/remotes/{}", removed + 1), Value::Null, - 200, + epoch, ) .await; } + leader(http, &ips[removed + 1..], 1).await; } } @@ -90,15 +198,24 @@ async fn quorum_expands_and_shrinks_without_losing_writes() { let initial = kv(&ips[..3]); initial.put(0, 1, b"k1", b"v1", None).await.unwrap(); exact(&initial, 0, b"k1", b"v1").await; - expand(&http, &ips).await; + eprintln!("reconfiguration: acknowledged k1 before expansion"); + // First wiring bootstraps the original members at epoch zero. + let mut epoch = 0; + for index in 3..5 { + epoch += 1; + add_member(&http, &ips, index, epoch).await; + } leader(&http, &ips, 1).await; let expanded = kv(&ips); exact(&expanded, 0, b"k1", b"v1").await; expanded.put(0, 1, b"k2", b"v2", None).await.unwrap(); - shrink(&http, &ips).await; + eprintln!("reconfiguration: expansion preserves k1; acknowledged k2 before shrink"); + shrink(&http, &ips, epoch).await; leader(&http, &ips[2..], 1).await; let surviving = kv(&ips[2..]); + exact(&surviving, 2, b"k1", b"v1").await; exact(&surviving, 2, b"k2", b"v2").await; + eprintln!("reconfiguration: shrink preserves k1 and k2"); surviving.delete(2, 1, b"k1", None).await.unwrap(); assert!(matches!( surviving diff --git a/e2e-test/tests/test_dispatch_test.py b/e2e-test/tests/test_dispatch_test.py index 8aa9999ad..217c28e5d 100644 --- a/e2e-test/tests/test_dispatch_test.py +++ b/e2e-test/tests/test_dispatch_test.py @@ -52,7 +52,9 @@ def test_compilation_cache_excludes_rootful_oci_state_and_runtime_artifacts(self self.assertIn('key: ${{ inputs.key }}', cache) self.assertIn('cmd-format: "pixi run {0}"', cache) self.assertIn('cache-workspace-crates: true', cache) - directories = cache.split('cache-directories: |', 1)[1].split('cmd-format:', 1)[0].split() + self.assertIn('cache-directories: ${{ inputs.directories }}', cache) + defaults = cache.split(' directories:\n', 1)[1].split('runs:', 1)[0] + directories = defaults.split(' default: |\n', 1)[1].split() self.assertEqual(set(directories), { 'target/debug', 'target/release', 'sdk/target/release', 'sdk/target/sdist/release', 'target/iceberg-sdk/release', 'app/crowdb-diskio/build', 'lib/crowdb-tree/build', 'lib/crowdb-rpc/build'}) @@ -65,6 +67,23 @@ def test_compilation_cache_excludes_rootful_oci_state_and_runtime_artifacts(self 'ContainerConsole', 'ContainerStorage', 'ContainerDeployment'): self.assertNotIn('uses: ./.github/actions/compilation-cache', self.job(job), job) + def test_access_cache_and_test_profile_bound_disk_without_narrowing_tasks(self): + cache = (ROOT / '.github/actions/compilation-cache/action.yml').read_text() + self.assertIn('default: release-ci-v1', cache) + self.assertIn("shared-key: ${{ inputs.family }}-${{ hashFiles('pixi.lock') }}", cache) + access = self.job('AccessTests') + self.assertRegex(access, r'(?m)^ CARGO_PROFILE_TEST_STRIP: debuginfo$') + self.assertRegex(access, r'(?m)^ CARGO_BUILD_JOBS: "4"$') + self.assertNotRegex(self.workflow, r'(?m)^ CARGO_PROFILE_TEST_STRIP:') + self.assertIn('family: access-ci-v2', access) + directories = re.search(r' directories: \|\n((?: [^\n]+\n)+)', access) + self.assertIsNotNone(directories) + self.assertEqual(directories[1].split(), [ + 'target/debug', 'sdk/target/release', 'sdk/target/sdist/release']) + self.assertEqual(re.findall(r'^ pixi run ([\w-]+)', access, re.M), [ + 'test-access', 'test-access-sdk', 'check-dataset-sdk-boundary', + 'sdk-fmt-check', 'sdk-lint', 'test-dataset-sdk', 'test-dataset-sdk-python']) + def test_only_binary_consumers_wait_for_shared_builds(self): for name in ('Lint', 'UITests', 'CppTests', 'UnitTests', 'StorageTests', 'AccessTests', 'ConsoleTests', 'ContainerImage', 'BuildClients'): From 84d2c30042f312a2898568147d1f1d561cae9415 Mon Sep 17 00:00:00 2001 From: Gian Date: Wed, 4 Nov 2026 21:42:56 +0800 Subject: [PATCH 23/24] Separate Dataset setup and browser CI budgets --- doc/working/plan-ci-stability-review.md | 10 +++++ e2e-test/scenarios.py | 4 ++ e2e-test/tests/console/browser_dataset.rs | 28 +++++++++----- e2e-test/tests/runner_progression_test.py | 47 ++++++++++++++++++++++- 4 files changed, 78 insertions(+), 11 deletions(-) diff --git a/doc/working/plan-ci-stability-review.md b/doc/working/plan-ci-stability-review.md index 77b0b6e92..9df6b1bdc 100644 --- a/doc/working/plan-ci-stability-review.md +++ b/doc/working/plan-ci-stability-review.md @@ -18,6 +18,16 @@ Completed CI execution and review tasks have been removed. CI evidence from repository root: `.crowdb-runtime/artifacts/ci/run-41816831055/`. +The Dataset import follow-up in +[run 41829384144](https://github.com/buzzcrow/crowdb/actions/runs/41829384144/job/126583192879) +is repaired and locally verified: preparation and browser execution each retain +360 seconds in the same fixture, instead of sharing one client budget. All five +Dataset owners pass at CI concurrency, including full production placement, +512 records, 12 partitions and three owners. Original browser assertions remain +unchanged. All 316 runner controls, Rust format, clippy and CI coverage pass. +Evidence from repository root: +`.crowdb-runtime/artifacts/ci/job-126583192879/`. + ## Remaining task: deferred historical attribution - [ ] **P13 first aggregate WAL release error (deferred)**: identify its exact diff --git a/e2e-test/scenarios.py b/e2e-test/scenarios.py index 30699c20e..19edccc1b 100644 --- a/e2e-test/scenarios.py +++ b/e2e-test/scenarios.py @@ -846,6 +846,10 @@ def execute(case, owned=None): cluster.crash_restart() cluster.client(bundle, "verify", case.name) else: + if case.scope == "console.native.browser.dataset": + # Production placement and browser execution each retain the + # case's bounded client budget in the same owned fixture. + cluster.client(bundle, "prepare", case.name) cluster.client(bundle, case.name.rsplit("::", 1)[-1], case.name) def stable_network(case): diff --git a/e2e-test/tests/console/browser_dataset.rs b/e2e-test/tests/console/browser_dataset.rs index 6d1e7f532..6d5e68317 100644 --- a/e2e-test/tests/console/browser_dataset.rs +++ b/e2e-test/tests/console/browser_dataset.rs @@ -1,37 +1,45 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -async fn run() { +async fn run(browser_phase: &str) { + let phase = std::env::var("CROWDB_E2E_PHASE").expect("owned Dataset client phase"); + assert!( + phase == "prepare" || phase == browser_phase, + "unexpected Dataset phase: {phase}" + ); let context = super::native_context::TestNativeContext::connect().await; - super::native_storage::start(&context).await; - // The first split leaves the normal partition-count policy unsatisfied. - // Finish bootstrap placement before the bounded Dataset reads begin. - super::native_balance::settle(&context).await; + if phase == "prepare" { + super::native_storage::start(&context).await; + // Finish the full production placement policy in the preparation phase. + super::native_balance::settle(&context).await; + return; + } + super::native_balance::settle_existing(&context).await; let result = crate::common::control::native_browser().await; assert_eq!(result["passed"], true); } #[tokio::test] #[ignore = "requires native Dataset and installed browser"] async fn browse() { - run().await; + run("browse").await; } #[tokio::test] #[ignore = "requires native Dataset and installed browser"] async fn import() { - run().await; + run("import").await; } #[tokio::test] #[ignore = "requires native Dataset and installed browser"] async fn split_revisions() { - run().await; + run("split_revisions").await; } #[tokio::test] #[ignore = "requires native Dataset and installed browser"] async fn recovery() { - run().await; + run("recovery").await; } #[tokio::test] #[ignore = "requires native Dataset and installed browser"] async fn capabilities() { - run().await; + run("capabilities").await; } diff --git a/e2e-test/tests/runner_progression_test.py b/e2e-test/tests/runner_progression_test.py index 78183eb6a..032b1230e 100644 --- a/e2e-test/tests/runner_progression_test.py +++ b/e2e-test/tests/runner_progression_test.py @@ -7,10 +7,55 @@ from pathlib import Path from unittest.mock import patch -from scenarios import Case, run_cases +from scenarios import Case, run_cases, select_cases class RunnerProgressionTest(unittest.TestCase): + def run_dataset_phases(self, fail_prepare=False): + events = [] + cases = select_cases(layer='console', scope='console.native.browser.dataset') + with tempfile.TemporaryDirectory() as temporary: + class Fixture: + def __init__(self, case): + self.name = case.name + self.artifacts = Path(temporary) / self.name + self.artifacts.mkdir() + + def __enter__(self): + return self + + def __exit__(self, *_): + events.append((self.name, 'close', self.client_timeout)) + + def client(self, _bundle, phase, name): + self_owner.assertEqual(name, self.name) + events.append((name, phase, self.client_timeout)) + if fail_prepare and phase == 'prepare': + raise ValueError('Dataset preparation failed') + + self_owner = self + with patch('scenarios.select_cases', return_value=cases), \ + patch('scenarios.allocate_cluster', side_effect=lambda case, *_: Fixture(case)): + if fail_prepare: + with self.assertRaisesRegex(ValueError, 'Dataset preparation failed'): + run_cases('image', temporary, 'bundle', 1, fail_fast=True) + else: + run_cases('image', temporary, 'bundle', 1) + return cases, events + + def test_dataset_preparation_and_browser_keep_separate_original_budgets(self): + cases, events = self.run_dataset_phases() + self.assertEqual(len(cases), 5) + self.assertEqual(events, [ + (case.name, phase, 360) + for case in cases + for phase in ('prepare', case.name.rsplit('::', 1)[-1], 'close') + ]) + + def test_dataset_preparation_failure_skips_browser_and_closes_fixture(self): + cases, events = self.run_dataset_phases(fail_prepare=True) + self.assertEqual(events, [(cases[0].name, phase, 360) for phase in ('prepare', 'close')]) + def test_ready_cli_executes_before_a_peer_finishes_starting(self): executed = threading.Event() events = [] From 5d3919fed6c11de2b705b6ab985071f0f7b4060a Mon Sep 17 00:00:00 2001 From: Gian Date: Wed, 4 Nov 2026 23:14:15 +0800 Subject: [PATCH 24/24] Fix asynchronous CI assertions and retain Iceberg command diagnostics --- .../crowdb-monitor/src/bootstrap/iceberg.rs | 11 ++- .../src/bootstrap/iceberg/command_failure.rs | 36 ++++++++ .../tests/iceberg_command_failure_test.rs | 90 +++++++++++++++++++ .../e2e/flows/13-node-service-plans.spec.ts | 3 +- .../integration/chunk_page_store_test.cpp | 11 +-- 5 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 container/crowdb-monitor/src/bootstrap/iceberg/command_failure.rs create mode 100644 container/crowdb-monitor/tests/iceberg_command_failure_test.rs diff --git a/container/crowdb-monitor/src/bootstrap/iceberg.rs b/container/crowdb-monitor/src/bootstrap/iceberg.rs index b35f025e0..0a16f4b60 100644 --- a/container/crowdb-monitor/src/bootstrap/iceberg.rs +++ b/container/crowdb-monitor/src/bootstrap/iceberg.rs @@ -11,6 +11,7 @@ use crate::{ MonitorLogError, ServerCredentials, }; +mod command_failure; mod console; mod shared_console; @@ -28,6 +29,12 @@ pub enum IcebergBootstrapError { Conflict(&'static str), #[error("Iceberg management command failed: {0}")] Command(&'static str), + #[error("Iceberg management command {action} failed ({status}): {diagnostic}")] + CommandExit { + action: String, + status: String, + diagnostic: String, + }, #[error("Iceberg management process failed: {0}")] Io(#[from] std::io::Error), #[error("Iceberg bootstrap manifest failed: {0}")] @@ -319,9 +326,7 @@ impl ManagementCommand<'_> { .await .map_err(|_| IcebergBootstrapError::Command("management command timed out"))??; if !output.status.success() { - return Err(IcebergBootstrapError::Command( - "management command exited unsuccessfully", - )); + return Err(command_failure::describe(arguments, &output, self.credentials)); } Ok(output.stdout) } diff --git a/container/crowdb-monitor/src/bootstrap/iceberg/command_failure.rs b/container/crowdb-monitor/src/bootstrap/iceberg/command_failure.rs new file mode 100644 index 000000000..8e4cdf4b3 --- /dev/null +++ b/container/crowdb-monitor/src/bootstrap/iceberg/command_failure.rs @@ -0,0 +1,36 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::IcebergBootstrapError; +use crate::ServerCredentials; +use std::process::Output; + +pub(super) fn describe( + arguments: &[&str], + output: &Output, + credentials: &ServerCredentials, +) -> IcebergBootstrapError { + let mut diagnostic = String::from_utf8_lossy(&output.stderr).into_owned(); + // Redact before truncation so a boundary cannot expose part of a credential. + for line in credentials.server_env().lines() { + if let Some((_, value)) = line.split_once('=') { + if !value.is_empty() { + diagnostic = diagnostic.replace(value, ""); + } + } + } + let diagnostic: String = diagnostic + .chars() + .take(1024) + .map(|character| if character.is_control() { ' ' } else { character }) + .collect(); + IcebergBootstrapError::CommandExit { + action: arguments.first().copied().unwrap_or("unknown").to_owned(), + status: output.status.to_string(), + diagnostic: if diagnostic.trim().is_empty() { + "no stderr was produced".into() + } else { + diagnostic + }, + } +} diff --git a/container/crowdb-monitor/tests/iceberg_command_failure_test.rs b/container/crowdb-monitor/tests/iceberg_command_failure_test.rs new file mode 100644 index 000000000..671b7115d --- /dev/null +++ b/container/crowdb-monitor/tests/iceberg_command_failure_test.rs @@ -0,0 +1,90 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_monitor::{IcebergBootstrap, IcebergBootstrapError, ServerCredentials}; +use std::{fs, os::unix::fs::PermissionsExt, path::Path}; + +async fn failure(root: &Path, script: &str, credentials: &ServerCredentials) -> IcebergBootstrapError { + let program = root.join("management"); + fs::write(&program, script).unwrap(); + fs::set_permissions(&program, fs::Permissions::from_mode(0o700)).unwrap(); + IcebergBootstrap::ensure_console_catalog(root, &program, "127.0.0.1:1", credentials) + .await + .unwrap_err() +} + +#[tokio::test] +async fn command_failure_reports_action_exit_status_and_redacts_every_server_credential() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("iceberg-command-failure"); + let credentials = ServerCredentials::load_or_create(root.path()).unwrap(); + let script = r#"#!/bin/sh +printf 'catalog owner unavailable\n%s\n%s\n%s\n%s\n%s\n' \ + "$CROWDB_S3_MASTER_KEY" "$CROWDB_ICEBERG_READ_TOKEN" "$CROWDB_ICEBERG_WRITE_TOKEN" \ + "$CROWDB_ICEBERG_MANAGE_TOKEN" "$CROWDB_ICEBERG_CLEAR_TOKEN" >&2 +exit 17 +"#; + let error = failure(root.path(), script, &credentials).await; + let message = error.to_string(); + let IcebergBootstrapError::CommandExit { + action, + status, + diagnostic, + } = error + else { + panic!("expected a command exit diagnostic: {message}"); + }; + assert_eq!(action, "inspect"); + assert_eq!(status, "exit status: 17"); + assert!(diagnostic.contains("catalog owner unavailable")); + assert!(diagnostic.contains("")); + assert!(!diagnostic.chars().any(char::is_control)); + for line in credentials.server_env().lines() { + let (_, value) = line.split_once('=').unwrap(); + assert!(!message.contains(value), "a server credential escaped redaction"); + } +} + +#[tokio::test] +async fn oversized_multibyte_stderr_is_bounded_without_splitting_utf8() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("iceberg-command-bound"); + let credentials = ServerCredentials::load_or_create(root.path()).unwrap(); + let error = failure( + root.path(), + "#!/usr/bin/env python3\nimport sys\nsys.stderr.write('界' * 4096)\nsys.exit(3)\n", + &credentials, + ) + .await; + let IcebergBootstrapError::CommandExit { diagnostic, .. } = error else { + panic!("expected a command exit diagnostic"); + }; + assert_eq!(diagnostic, "界".repeat(1024)); +} + +#[tokio::test] +async fn a_signal_exit_remains_distinguishable_from_an_application_error() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("iceberg-command-signal"); + let credentials = ServerCredentials::load_or_create(root.path()).unwrap(); + let error = failure(root.path(), "#!/bin/sh\nkill -TERM $$\n", &credentials).await; + let IcebergBootstrapError::CommandExit { + status, diagnostic, .. + } = error + else { + panic!("expected a command exit diagnostic"); + }; + assert!(status.contains("signal: 15"), "{status}"); + assert_eq!(diagnostic, "no stderr was produced"); +} + +#[tokio::test] +async fn a_credential_crossing_the_diagnostic_bound_is_redacted_before_truncation() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("iceberg-command-boundary"); + let credentials = ServerCredentials::load_or_create(root.path()).unwrap(); + let padding = "x".repeat(1020); + let script = + format!("#!/bin/sh\nprintf '%s%s' '{padding}' \"$CROWDB_ICEBERG_MANAGE_TOKEN\" >&2\nexit 1\n"); + let error = failure(root.path(), &script, &credentials).await; + let IcebergBootstrapError::CommandExit { diagnostic, .. } = error else { + panic!("expected a command exit diagnostic"); + }; + assert_eq!(diagnostic, format!("{padding} (await nativeServices(0))['diskdb-1']?.pid, { intervals: [100] }).toBeNull(); + // Monitor status lists owned children; Stop removes the child but retains registration. + await expect.poll(async () => Object.hasOwn(await nativeServices(0), 'diskdb-1'), { intervals: [100] }).toBe(false); const registered = (await json(request, '/api/servers')).filter((server: { id: string }) => server.id === 'diskdb-1'); expect(registered).toHaveLength(1); expect(registered[0]).toMatchObject({ id: 'diskdb-1', node_id: 1, service_type: 'diskdb' }); diff --git a/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp b/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp index 86de65b90..5d7d84dcf 100644 --- a/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp +++ b/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp @@ -176,7 +176,7 @@ class BlockingWriteTransport final : public ChunkTransport uint32_t observed = max_active_writes_.load(std::memory_order_relaxed); while (observed < active && !max_active_writes_.compare_exchange_weak(observed, active)) { } - active_writes_.notify_all(); + max_active_writes_.notify_all(); if (hold_writes_.load(std::memory_order_acquire)) { release_writes_.wait(false, std::memory_order_acquire); } @@ -213,10 +213,11 @@ class BlockingWriteTransport final : public ChunkTransport void wait_for_concurrent_writes(uint32_t count) { - uint32_t active = active_writes_.load(std::memory_order_acquire); - while (active < count) { - active_writes_.wait(active, std::memory_order_acquire); - active = active_writes_.load(std::memory_order_acquire); + // An active count is visible before its corresponding maximum is published. + uint32_t observed = max_active_writes_.load(std::memory_order_acquire); + while (observed < count) { + max_active_writes_.wait(observed, std::memory_order_acquire); + observed = max_active_writes_.load(std::memory_order_acquire); } }